diff --git a/backend/app/core/config.py b/backend/app/core/config.py index e6d8673e7..cc105fbec 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -223,7 +223,7 @@ def AWS_S3_BUCKET(self) -> str: # Consecutive low-delta rounds required before the loop stops as ceiling_reached. EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS: int = 3 - EVAL_JUDGE_MODEL: str = "gpt-5-mini" + EVAL_JUDGE_MODEL: str = "gpt-5.6-luna" # One of: none | minimal | low | medium | high | xhigh. EVAL_JUDGE_REASONING_EFFORT: str = "medium" diff --git a/backend/app/crud/evaluations/summary.py b/backend/app/crud/evaluations/summary.py index 2f87bb652..0fd377dc6 100644 --- a/backend/app/crud/evaluations/summary.py +++ b/backend/app/crud/evaluations/summary.py @@ -31,8 +31,8 @@ LLM-as-judge rationale behind each; plus the AI config that was evaluated, the golden Q&A (expected answers), and the generated answers. When the duplication factor is 5, each question was scored five times and you receive all five sets. -Reference each question by the item id in its Trace ID (the `item_N` prefix, e.g. -item_3). Never invent an id. +Reference each question by its `question_id` as "Question N" (e.g. Question 3). +Never invent an id. SCALE AND BANDS (use exactly these): - low = score 0 or 1 @@ -69,12 +69,12 @@ 1. An overall read in 2–3 lines: the general health of the run in plain warm language (you may note that KB grounding is strong, ground-truth is weak, the judge is unstable, etc.), no score dumps. -2. "Top 3 to check:" — the three highest-severity items or item-clusters only, - each one line: the item id(s), the one-line reason, and if relevant whether the - fix likely lives in the model, the config, or the golden dataset. Group items - into one line only when they share the same root cause; otherwise list them - separately. If fewer than three real problems exist, list only the real ones. - Never pad to three. +2. "Top 3 to check:" — the three highest-severity questions or question-clusters + only, each one line: the question id(s) as "Question N", the one-line reason, + and if relevant whether the fix likely lives in the model, the config, or the + golden dataset. Group questions into one line only when they share the same + root cause; otherwise list them separately. If fewer than three real problems + exist, list only the real ones. Never pad to three. 3. One closing line reminding the reviewer these are go-verify pointers: open the items, confirm the answer actually holds for the stated condition, and decide based on what the use case needs. @@ -99,14 +99,14 @@ def _format_traces_for_prompt( ) -> str: """Per-question judge traces as the summary model's brief. - Traces are handed over ungrouped: `trace_id` is the dataset item id - (`item_{row}_{dup}`), which is what the system prompt tells the model to key on. + Traces are handed over ungrouped: `question_id` is the 1-based dataset row + number, which is what the system prompt tells the model to key on. """ # ponytail: every trace sent whole; ~250 traces (50 questions x dup 5) overflows - # context and degrades to None. Sample or group per item_id if runs get bigger. + # context and degrades to None. Sample or group per question_id if runs get bigger. payload = [ { - "trace_id": trace["trace_id"], + "question_id": trace["question_id"], "question": trace["question"], "ground_truth_answer": trace["ground_truth_answer"], "llm_answer": trace["llm_answer"], diff --git a/backend/app/tests/crud/evaluations/test_run_ai_summary.py b/backend/app/tests/crud/evaluations/test_run_ai_summary.py index 5c86e1d66..44d62bba9 100644 --- a/backend/app/tests/crud/evaluations/test_run_ai_summary.py +++ b/backend/app/tests/crud/evaluations/test_run_ai_summary.py @@ -164,7 +164,7 @@ def test_blank_config_prompt_falls_back_to_the_placeholder(self) -> None: def test_each_trace_carries_its_qa_and_scored_rationales(self) -> None: payload = _payload_from(_brief_for()) - assert [t["trace_id"] for t in payload] == ["item_1_1", "item_2_1"] + assert [t["question_id"] for t in payload] == [1, 2] first = payload[0] assert first["question"] == "How much urea per acre?" assert first["ground_truth_answer"] == "Roughly 45-55 kg per acre." @@ -193,7 +193,7 @@ def test_trace_bookkeeping_keys_never_reach_the_model(self) -> None: payload = _payload_from(brief) assert set(payload[0]) == { - "trace_id", + "question_id", "question", "ground_truth_answer", "llm_answer", diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 3e90cc935..1e9593fa5 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -32,7 +32,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer) - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop) - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async