fix(evaluation): Correct CSV encoding issues

backend/app/services/evaluations/validators.py

@@ -1,5 +1,6 @@
 """Validation utilities for evaluation datasets."""
 
+import codecs
 import csv
 import io
 import logging
@@ -20,6 +21,19 @@
     "text/plain",
 }
 
+# Checked longest-first: the UTF-32-LE BOM starts with the UTF-16-LE BOM,
+# so probing UTF-16 first would misread a UTF-32 file.
+BOM_ENCODINGS = (
+    (codecs.BOM_UTF32_LE, "utf-32"),
+    (codecs.BOM_UTF32_BE, "utf-32"),
+    (codecs.BOM_UTF16_LE, "utf-16"),
+    (codecs.BOM_UTF16_BE, "utf-16"),
+)
+
+# cp1252 before latin-1: it maps 0x80-0x9F to the smart quotes, apostrophes and
+# dashes Windows Excel writes there, which latin-1 would turn into C1 controls.
+FALLBACK_ENCODINGS = ("utf-8-sig", "cp1252", "latin-1")
+
 
 def sanitize_dataset_name(name: str) -> str:
     """
@@ -114,6 +128,35 @@ async def validate_csv_file(file: UploadFile) -> bytes:
     return await file.read()
 
 
+def decode_csv_bytes(csv_content: bytes) -> str:
+    """Decode uploaded CSV bytes, tolerating the encodings spreadsheet apps emit.
+
+    Handles UTF-8, UTF-8 with a BOM (Excel's "CSV UTF-8" export), UTF-16/UTF-32
+    with a BOM (Excel's "Unicode Text" export) and the cp1252 bytes Windows Excel
+    writes for its plain "CSV (Comma delimited)" export.
+
+    Raises:
+        HTTPException: If the bytes cannot be decoded by any candidate encoding.
+    """
+    for bom, encoding in BOM_ENCODINGS:
+        if csv_content.startswith(bom):
+            return csv_content.decode(encoding)
+
+    for encoding in FALLBACK_ENCODINGS:
+        try:
+            return csv_content.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+
+    # Unreachable while latin-1 (which decodes any byte sequence) closes the ladder;
+    # kept so the function stays total if FALLBACK_ENCODINGS loses its catch-all codec.
+    logger.warning("[decode_csv_bytes] Failed to decode CSV with any known encoding")
+    raise HTTPException(
+        status_code=422,
+        detail="Unable to read the CSV file. Please re-save it as UTF-8 CSV.",
+    )
+
+
 def parse_csv_items(csv_content: bytes) -> list[dict[str, str]]:
     """
     Parse CSV and extract question/answer/category triples.
@@ -129,7 +172,7 @@ def parse_csv_items(csv_content: bytes) -> list[dict[str, str]]:
         HTTPException: If CSV is invalid or empty
     """
     try:
-        csv_text = csv_content.decode("utf-8")
+        csv_text = decode_csv_bytes(csv_content)
         csv_reader = csv.DictReader(io.StringIO(csv_text))
 
         if not csv_reader.fieldnames:

backend/app/tests/services/evaluations/test_validators.py

@@ -1,5 +1,7 @@
 """Tests for CSV parsing in app.services.evaluations.validators."""
 
+import codecs
+
 import pytest
 from fastapi import HTTPException
 
@@ -55,3 +57,45 @@ def test_empty_csv_raises_422(self) -> None:
         with pytest.raises(HTTPException) as excinfo:
             parse_csv_items(csv)
         assert excinfo.value.status_code == 422
+
+
+class TestParseCsvItemsEncoding:
+    """Tests that exercise the encodings real spreadsheet apps emit."""
+
+    HEADER = "question,answer\n"
+    ROW = 'q1,"The fee is 10 – 20 and he said “hi”"\n'
+    EXPECTED = "The fee is 10 – 20 and he said “hi”"
+
+    def test_utf8_smart_punctuation(self) -> None:
+        csv = (self.HEADER + self.ROW).encode("utf-8")
+        assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED
+
+    def test_utf8_with_bom_resolves_headers(self) -> None:
+        """Excel's "CSV UTF-8" export prefixes a BOM, which used to leave the
+        `question` header as `question` and fail the required-column check."""
+        csv = codecs.BOM_UTF8 + (self.HEADER + self.ROW).encode("utf-8")
+        items = parse_csv_items(csv)
+        assert items[0]["question"] == "q1"
+        assert items[0]["answer"] == self.EXPECTED
+
+    def test_cp1252_recovers_smart_punctuation(self) -> None:
+        """Windows Excel's plain "CSV" export writes cp1252, where the en dash is
+        the single byte 0x96 that UTF-8 rejects as an invalid start byte."""
+        csv = (self.HEADER + self.ROW).encode("cp1252")
+        assert b"\x96" in csv
+        assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED
+
+    def test_utf16_with_bom(self) -> None:
+        csv = (self.HEADER + self.ROW).encode("utf-16")
+        assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED
+
+    def test_utf32_bom_not_misread_as_utf16(self) -> None:
+        """The UTF-32-LE BOM starts with the UTF-16-LE BOM, so probe order matters."""
+        csv = (self.HEADER + self.ROW).encode("utf-32")
+        assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED
+
+    def test_byte_undefined_in_cp1252_falls_through_to_latin1(self) -> None:
+        csv = (self.HEADER + "q1,weird \x81 byte\n").encode("latin-1")
+        with pytest.raises(UnicodeDecodeError):
+            csv.decode("cp1252")
+        assert parse_csv_items(csv)[0]["answer"] == "weird \x81 byte"

docs/wiki/modules/evaluations.md

@@ -30,7 +30,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud
 `EvaluationIterationRun` is a thin bookkeeping row only (`status`, `stop_reason`, `dataset_id`, `config_id`, `initial_config_version`, `callback_url`, `error_message`) — round-by-round state (`round_number`, `current_eval_run_id`, `current_improvement_job_id`, `history`, `best_*`, `consecutive_low_delta_rounds`) lives entirely in the LangGraph checkpoint keyed by `thread_id = str(id)`, not on this table. No FK to `EvaluationRun`/`Job`; those are referenced only inside the checkpoint state.
 
 ## Services / CRUD
-- `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer)
+- `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py` (`decode_csv_bytes` — every uploaded dataset CSV is decoded through a BOM probe then a `utf-8-sig` → `cp1252` → `latin-1` ladder, so both Excel exports work: plain "CSV" writes cp1252, "CSV UTF-8" writes a BOM), `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer)
 - `services/stt_evaluations/`, `services/tts_evaluations/`
 - `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop)
 - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py`