Read CSV and JSON files as UTF-8 regardless of platform charset

scautable/src-jvm/JsonTable.scala

@@ -1,5 +1,6 @@
 package io.github.quafadas.scautable.json
 
+import scala.io.Codec
 import scala.io.Source
 import scala.quoted.*
 
@@ -132,7 +133,7 @@ object JsonTable:
     end if
 
     // Use streaming for compile-time type inference
-    val source = Source.fromURL(resourcePath)
+    val source = Source.fromURL(resourcePath)(using Codec.UTF8)
     try
       val objects = StreamingJsonParser.parseArrayFromSource(source)
       processJsonArrayFromFile(objects, pathExpr, typeInferrerExpr, isResource = true)
@@ -145,7 +146,7 @@ object JsonTable:
 
     val path = pathExpr.valueOrAbort
     val source =
-      try Source.fromFile(path)
+      try Source.fromFile(path)(using Codec.UTF8)
       catch
         case e: Exception =>
           report.throwError(s"Failed to read file at path $path: ${e.getMessage}")
@@ -174,7 +175,7 @@ object JsonTable:
       "This method saves the JSON to a local temp file and opens it. There may be performance implications - it is recommended to use one of the other methods where possible."
     )
 
-    val source = Source.fromURL(urlExpr.valueOrAbort)
+    val source = Source.fromURL(urlExpr.valueOrAbort)(using Codec.UTF8)
     val tmpPath = java.nio.file.Files.createTempFile("temp_json_", ".json")
     java.nio.file.Files.writeString(tmpPath, source.mkString)
 

scautable/src/csv.scala

@@ -1,6 +1,7 @@
 package io.github.quafadas.scautable
 
 import scala.NamedTuple.*
+import scala.io.Codec
 import scala.io.Source
 import scala.quoted.*
 import java.nio.file.{Files, Path, Paths}
@@ -365,14 +366,14 @@ object CSV:
   end openSourceWithFallback
 
   private[scautable] def openSourceWithFallback(absolutePath: String, rootRelativePath: String, resourceName: String, useFallback: Boolean): Source =
-    if !useFallback then Source.fromFile(absolutePath)
+    if !useFallback then Source.fromFile(absolutePath)(using Codec.UTF8)
     else
       val candidateResources = Seq(rootRelativePath, resourceName).map(_.replace('\\', '/')).distinct
-      Try(Source.fromFile(absolutePath))
-        .orElse(Try(Source.fromFile(rootRelativePath)))
+      Try(Source.fromFile(absolutePath)(using Codec.UTF8))
+        .orElse(Try(Source.fromFile(rootRelativePath)(using Codec.UTF8)))
         .orElse(
           candidateResources.iterator
-            .map(resource => Try(Source.fromResource(resource)))
+            .map(resource => Try(Source.fromResource(resource)(using Codec.UTF8)))
             .collectFirst { case scala.util.Success(source) => source }
             .map(scala.util.Success(_))
             .getOrElse(scala.util.Failure(new RuntimeException("Resource lookup failed")))
@@ -417,7 +418,7 @@ object CSV:
     val denseRowMajorType: Option[TypeRepr] = CSV.extractDenseArrayType(readAsTerm, "ArrayDenseRowMajor")
 
     val source = scala.util
-      .Try(Source.fromFile(pathChain.absolutePath))
+      .Try(Source.fromFile(pathChain.absolutePath)(using Codec.UTF8))
       .getOrElse(
         report.errorAndAbort(
           s"scautable: could not read a CSV at '${pathChain.absolutePath}' while compiling. The file has to exist at compile time so its columns can be typed.${pathChain.anchorHint}"
@@ -652,7 +653,7 @@ object CSV:
       "This method saves the CSV to a local temp file and opens it. There may be performance implications - it is recommended to use one of the other methods where possible."
     )
 
-    val source = Source.fromURL(pathExpr.valueOrAbort)
+    val source = Source.fromURL(pathExpr.valueOrAbort)(using Codec.UTF8)
     val tmpPath = java.nio.file.Files.createTempFile("temp_csv_", ".csv")
     java.nio.file.Files.writeString(tmpPath, source.mkString)
     readHeaderlineAsCsv(runtimePathChain(tmpPath, Paths.get(".").toAbsolutePath.normalize, useFallback = false), optsExpr)
@@ -955,7 +956,7 @@ object CSV:
     */
   private inline def fromTyped[K <: Tuple, V <: Tuple](headers: HeaderOptions): PlatformPath => CsvIterator[K, V] =
     (path: PlatformPath) =>
-      val source = scala.io.Source.fromFile(path.platformPathString)
+      val source = scala.io.Source.fromFile(path.platformPathString)(using scala.io.Codec.UTF8)
       // Every check below can throw, and each one used to strand the handle it had just opened.
       try
         val (hdrs, iterator) = source.getLines().headers(headers)

scautable/test/resources/bom_utf8_non_ascii.csv

@@ -0,0 +1,3 @@
+Stadt,Größe,Notiz
+Zürich,1.5,café
+São Paulo,2.0,日本語

scautable/test/src-jvm/BomSuite.scala

@@ -47,6 +47,26 @@ class BomSuite extends munit.FunSuite:
     assertEquals(rows.map(_.col1), List(1, 3))
   }
 
+  // Read with an explicit UTF-8 codec, so this holds even where the platform default charset is not UTF-8 (Windows on JDK 17 and earlier). Decoded as windows-1252, the BOM
+  // would arrive as "" and never be stripped, and the non-ASCII text would be garbled.
+  test("BOM-prefixed CSV with non-ASCII headers and values decodes as UTF-8") {
+    val csv: CsvIterator[("Stadt", "Größe", "Notiz"), (String, Double, String)] = CSV.resource("bom_utf8_non_ascii.csv")
+
+    assertEquals(csv.headers, Seq("Stadt", "Größe", "Notiz"))
+    val rows = csv.toList
+    assertEquals(rows.map(_.Stadt), List("Zürich", "São Paulo"))
+    assertEquals(rows.map(_.Notiz), List("café", "日本語"))
+  }
+
+  test("BOM-prefixed CSV with non-ASCII text via runtime fromTyped") {
+    val reader = CSV.fromTyped[("Stadt", "Größe", "Notiz"), (String, Double, String)]
+    val path = os.Path(this.getClass.getClassLoader.getResource("bom_utf8_non_ascii.csv").getPath)
+
+    val rows = reader(path).toList
+    assertEquals(rows.map(_.Stadt), List("Zürich", "São Paulo"))
+    assertEquals(rows.map(_.Notiz), List("café", "日本語"))
+  }
+
   test("CSV without a BOM is unchanged") {
     val csv: CsvIterator[("col1", "col2", "col3"), (Int, Int, Int)] = CSV.resource("simple.csv")
     assertEquals(csv.headers, Seq("col1", "col2", "col3"))