From 9b5ee27de2be4a371647c92488804dba40d4d9d9 Mon Sep 17 00:00:00 2001 From: Simon Parten Date: Mon, 26 Oct 2026 14:35:14 +0100 Subject: [PATCH] Read CSV and JSON files as UTF-8 regardless of platform charset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Source.fromFile / fromResource / fromURL fell back to the JVM default charset. On Windows with JDK 17 and earlier that is windows-1252, so non-ASCII text was garbled and a UTF-8 BOM arrived as "", which the BOM strip never matched. Pass Codec.UTF8 explicitly at every read, both in the compile-time macros and the runtime iterators. Co-Authored-By: Claude Opus 5.5 (1M context) --- scautable/src-jvm/JsonTable.scala | 7 ++++--- scautable/src/csv.scala | 15 +++++++------- .../test/resources/bom_utf8_non_ascii.csv | 3 +++ scautable/test/src-jvm/BomSuite.scala | 20 +++++++++++++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 scautable/test/resources/bom_utf8_non_ascii.csv diff --git a/scautable/src-jvm/JsonTable.scala b/scautable/src-jvm/JsonTable.scala index abd8a9f40..8c4ea400f 100644 --- a/scautable/src-jvm/JsonTable.scala +++ b/scautable/src-jvm/JsonTable.scala @@ -1,5 +1,6 @@ package io.github.quafadas.scautable.json +import scala.io.Codec import scala.io.Source import scala.quoted.* @@ -132,7 +133,7 @@ object JsonTable: end if // Use streaming for compile-time type inference - val source = Source.fromURL(resourcePath) + val source = Source.fromURL(resourcePath)(using Codec.UTF8) try val objects = StreamingJsonParser.parseArrayFromSource(source) processJsonArrayFromFile(objects, pathExpr, typeInferrerExpr, isResource = true) @@ -145,7 +146,7 @@ object JsonTable: val path = pathExpr.valueOrAbort val source = - try Source.fromFile(path) + try Source.fromFile(path)(using Codec.UTF8) catch case e: Exception => report.throwError(s"Failed to read file at path $path: ${e.getMessage}") @@ -174,7 +175,7 @@ object JsonTable: "This method saves the JSON to a local temp file and opens it. There may be performance implications - it is recommended to use one of the other methods where possible." ) - val source = Source.fromURL(urlExpr.valueOrAbort) + val source = Source.fromURL(urlExpr.valueOrAbort)(using Codec.UTF8) val tmpPath = java.nio.file.Files.createTempFile("temp_json_", ".json") java.nio.file.Files.writeString(tmpPath, source.mkString) diff --git a/scautable/src/csv.scala b/scautable/src/csv.scala index 390884689..7837bb4d5 100644 --- a/scautable/src/csv.scala +++ b/scautable/src/csv.scala @@ -1,6 +1,7 @@ package io.github.quafadas.scautable import scala.NamedTuple.* +import scala.io.Codec import scala.io.Source import scala.quoted.* import java.nio.file.{Files, Path, Paths} @@ -365,14 +366,14 @@ object CSV: end openSourceWithFallback private[scautable] def openSourceWithFallback(absolutePath: String, rootRelativePath: String, resourceName: String, useFallback: Boolean): Source = - if !useFallback then Source.fromFile(absolutePath) + if !useFallback then Source.fromFile(absolutePath)(using Codec.UTF8) else val candidateResources = Seq(rootRelativePath, resourceName).map(_.replace('\\', '/')).distinct - Try(Source.fromFile(absolutePath)) - .orElse(Try(Source.fromFile(rootRelativePath))) + Try(Source.fromFile(absolutePath)(using Codec.UTF8)) + .orElse(Try(Source.fromFile(rootRelativePath)(using Codec.UTF8))) .orElse( candidateResources.iterator - .map(resource => Try(Source.fromResource(resource))) + .map(resource => Try(Source.fromResource(resource)(using Codec.UTF8))) .collectFirst { case scala.util.Success(source) => source } .map(scala.util.Success(_)) .getOrElse(scala.util.Failure(new RuntimeException("Resource lookup failed"))) @@ -417,7 +418,7 @@ object CSV: val denseRowMajorType: Option[TypeRepr] = CSV.extractDenseArrayType(readAsTerm, "ArrayDenseRowMajor") val source = scala.util - .Try(Source.fromFile(pathChain.absolutePath)) + .Try(Source.fromFile(pathChain.absolutePath)(using Codec.UTF8)) .getOrElse( report.errorAndAbort( s"scautable: could not read a CSV at '${pathChain.absolutePath}' while compiling. The file has to exist at compile time so its columns can be typed.${pathChain.anchorHint}" @@ -652,7 +653,7 @@ object CSV: "This method saves the CSV to a local temp file and opens it. There may be performance implications - it is recommended to use one of the other methods where possible." ) - val source = Source.fromURL(pathExpr.valueOrAbort) + val source = Source.fromURL(pathExpr.valueOrAbort)(using Codec.UTF8) val tmpPath = java.nio.file.Files.createTempFile("temp_csv_", ".csv") java.nio.file.Files.writeString(tmpPath, source.mkString) readHeaderlineAsCsv(runtimePathChain(tmpPath, Paths.get(".").toAbsolutePath.normalize, useFallback = false), optsExpr) @@ -955,7 +956,7 @@ object CSV: */ private inline def fromTyped[K <: Tuple, V <: Tuple](headers: HeaderOptions): PlatformPath => CsvIterator[K, V] = (path: PlatformPath) => - val source = scala.io.Source.fromFile(path.platformPathString) + val source = scala.io.Source.fromFile(path.platformPathString)(using scala.io.Codec.UTF8) // Every check below can throw, and each one used to strand the handle it had just opened. try val (hdrs, iterator) = source.getLines().headers(headers) diff --git a/scautable/test/resources/bom_utf8_non_ascii.csv b/scautable/test/resources/bom_utf8_non_ascii.csv new file mode 100644 index 000000000..176936967 --- /dev/null +++ b/scautable/test/resources/bom_utf8_non_ascii.csv @@ -0,0 +1,3 @@ +Stadt,Größe,Notiz +Zürich,1.5,café +São Paulo,2.0,日本語 diff --git a/scautable/test/src-jvm/BomSuite.scala b/scautable/test/src-jvm/BomSuite.scala index 51273dc8f..2a23a272f 100644 --- a/scautable/test/src-jvm/BomSuite.scala +++ b/scautable/test/src-jvm/BomSuite.scala @@ -47,6 +47,26 @@ class BomSuite extends munit.FunSuite: assertEquals(rows.map(_.col1), List(1, 3)) } + // Read with an explicit UTF-8 codec, so this holds even where the platform default charset is not UTF-8 (Windows on JDK 17 and earlier). Decoded as windows-1252, the BOM + // would arrive as "" and never be stripped, and the non-ASCII text would be garbled. + test("BOM-prefixed CSV with non-ASCII headers and values decodes as UTF-8") { + val csv: CsvIterator[("Stadt", "Größe", "Notiz"), (String, Double, String)] = CSV.resource("bom_utf8_non_ascii.csv") + + assertEquals(csv.headers, Seq("Stadt", "Größe", "Notiz")) + val rows = csv.toList + assertEquals(rows.map(_.Stadt), List("Zürich", "São Paulo")) + assertEquals(rows.map(_.Notiz), List("café", "日本語")) + } + + test("BOM-prefixed CSV with non-ASCII text via runtime fromTyped") { + val reader = CSV.fromTyped[("Stadt", "Größe", "Notiz"), (String, Double, String)] + val path = os.Path(this.getClass.getClassLoader.getResource("bom_utf8_non_ascii.csv").getPath) + + val rows = reader(path).toList + assertEquals(rows.map(_.Stadt), List("Zürich", "São Paulo")) + assertEquals(rows.map(_.Notiz), List("café", "日本語")) + } + test("CSV without a BOM is unchanged") { val csv: CsvIterator[("col1", "col2", "col3"), (Int, Int, Int)] = CSV.resource("simple.csv") assertEquals(csv.headers, Seq("col1", "col2", "col3"))