diff --git a/backend/app/services/evaluations/validators.py b/backend/app/services/evaluations/validators.py index 74e59b61c..7ea1e4a35 100644 --- a/backend/app/services/evaluations/validators.py +++ b/backend/app/services/evaluations/validators.py @@ -1,5 +1,6 @@ """Validation utilities for evaluation datasets.""" +import codecs import csv import io import logging @@ -20,6 +21,19 @@ "text/plain", } +# Checked longest-first: the UTF-32-LE BOM starts with the UTF-16-LE BOM, +# so probing UTF-16 first would misread a UTF-32 file. +BOM_ENCODINGS = ( + (codecs.BOM_UTF32_LE, "utf-32"), + (codecs.BOM_UTF32_BE, "utf-32"), + (codecs.BOM_UTF16_LE, "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16"), +) + +# cp1252 before latin-1: it maps 0x80-0x9F to the smart quotes, apostrophes and +# dashes Windows Excel writes there, which latin-1 would turn into C1 controls. +FALLBACK_ENCODINGS = ("utf-8-sig", "cp1252", "latin-1") + def sanitize_dataset_name(name: str) -> str: """ @@ -114,6 +128,35 @@ async def validate_csv_file(file: UploadFile) -> bytes: return await file.read() +def decode_csv_bytes(csv_content: bytes) -> str: + """Decode uploaded CSV bytes, tolerating the encodings spreadsheet apps emit. + + Handles UTF-8, UTF-8 with a BOM (Excel's "CSV UTF-8" export), UTF-16/UTF-32 + with a BOM (Excel's "Unicode Text" export) and the cp1252 bytes Windows Excel + writes for its plain "CSV (Comma delimited)" export. + + Raises: + HTTPException: If the bytes cannot be decoded by any candidate encoding. + """ + for bom, encoding in BOM_ENCODINGS: + if csv_content.startswith(bom): + return csv_content.decode(encoding) + + for encoding in FALLBACK_ENCODINGS: + try: + return csv_content.decode(encoding) + except UnicodeDecodeError: + continue + + # Unreachable while latin-1 (which decodes any byte sequence) closes the ladder; + # kept so the function stays total if FALLBACK_ENCODINGS loses its catch-all codec. + logger.warning("[decode_csv_bytes] Failed to decode CSV with any known encoding") + raise HTTPException( + status_code=422, + detail="Unable to read the CSV file. Please re-save it as UTF-8 CSV.", + ) + + def parse_csv_items(csv_content: bytes) -> list[dict[str, str]]: """ Parse CSV and extract question/answer/category triples. @@ -129,7 +172,7 @@ def parse_csv_items(csv_content: bytes) -> list[dict[str, str]]: HTTPException: If CSV is invalid or empty """ try: - csv_text = csv_content.decode("utf-8") + csv_text = decode_csv_bytes(csv_content) csv_reader = csv.DictReader(io.StringIO(csv_text)) if not csv_reader.fieldnames: diff --git a/backend/app/tests/services/evaluations/test_validators.py b/backend/app/tests/services/evaluations/test_validators.py index c4c6e8488..64db8a288 100644 --- a/backend/app/tests/services/evaluations/test_validators.py +++ b/backend/app/tests/services/evaluations/test_validators.py @@ -1,5 +1,7 @@ """Tests for CSV parsing in app.services.evaluations.validators.""" +import codecs + import pytest from fastapi import HTTPException @@ -55,3 +57,45 @@ def test_empty_csv_raises_422(self) -> None: with pytest.raises(HTTPException) as excinfo: parse_csv_items(csv) assert excinfo.value.status_code == 422 + + +class TestParseCsvItemsEncoding: + """Tests that exercise the encodings real spreadsheet apps emit.""" + + HEADER = "question,answer\n" + ROW = 'q1,"The fee is 10 – 20 and he said “hi”"\n' + EXPECTED = "The fee is 10 – 20 and he said “hi”" + + def test_utf8_smart_punctuation(self) -> None: + csv = (self.HEADER + self.ROW).encode("utf-8") + assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED + + def test_utf8_with_bom_resolves_headers(self) -> None: + """Excel's "CSV UTF-8" export prefixes a BOM, which used to leave the + `question` header as `question` and fail the required-column check.""" + csv = codecs.BOM_UTF8 + (self.HEADER + self.ROW).encode("utf-8") + items = parse_csv_items(csv) + assert items[0]["question"] == "q1" + assert items[0]["answer"] == self.EXPECTED + + def test_cp1252_recovers_smart_punctuation(self) -> None: + """Windows Excel's plain "CSV" export writes cp1252, where the en dash is + the single byte 0x96 that UTF-8 rejects as an invalid start byte.""" + csv = (self.HEADER + self.ROW).encode("cp1252") + assert b"\x96" in csv + assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED + + def test_utf16_with_bom(self) -> None: + csv = (self.HEADER + self.ROW).encode("utf-16") + assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED + + def test_utf32_bom_not_misread_as_utf16(self) -> None: + """The UTF-32-LE BOM starts with the UTF-16-LE BOM, so probe order matters.""" + csv = (self.HEADER + self.ROW).encode("utf-32") + assert parse_csv_items(csv)[0]["answer"] == self.EXPECTED + + def test_byte_undefined_in_cp1252_falls_through_to_latin1(self) -> None: + csv = (self.HEADER + "q1,weird \x81 byte\n").encode("latin-1") + with pytest.raises(UnicodeDecodeError): + csv.decode("cp1252") + assert parse_csv_items(csv)[0]["answer"] == "weird \x81 byte" diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 1e9593fa5..41e342b95 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -30,7 +30,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud `EvaluationIterationRun` is a thin bookkeeping row only (`status`, `stop_reason`, `dataset_id`, `config_id`, `initial_config_version`, `callback_url`, `error_message`) — round-by-round state (`round_number`, `current_eval_run_id`, `current_improvement_job_id`, `history`, `best_*`, `consecutive_low_delta_rounds`) lives entirely in the LangGraph checkpoint keyed by `thread_id = str(id)`, not on this table. No FK to `EvaluationRun`/`Job`; those are referenced only inside the checkpoint state. ## Services / CRUD -- `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer) +- `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py` (`decode_csv_bytes` — every uploaded dataset CSV is decoded through a BOM probe then a `utf-8-sig` → `cp1252` → `latin-1` ladder, so both Excel exports work: plain "CSV" writes cp1252, "CSV UTF-8" writes a BOM), `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer) - `services/stt_evaluations/`, `services/tts_evaluations/` - `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py`