enhancement(evaluation): Reference summary findings by question number

backend/app/core/config.py

@@ -223,7 +223,7 @@ def AWS_S3_BUCKET(self) -> str:
     # Consecutive low-delta rounds required before the loop stops as ceiling_reached.
     EVAL_ITERATION_CEILING_CONSECUTIVE_ROUNDS: int = 3
 
-    EVAL_JUDGE_MODEL: str = "gpt-5-mini"
+    EVAL_JUDGE_MODEL: str = "gpt-5.6-luna"
 
     # One of: none | minimal | low | medium | high | xhigh.
     EVAL_JUDGE_REASONING_EFFORT: str = "medium"

backend/app/crud/evaluations/summary.py

@@ -31,8 +31,8 @@
 LLM-as-judge rationale behind each; plus the AI config that was evaluated, the
 golden Q&A (expected answers), and the generated answers. When the duplication
 factor is 5, each question was scored five times and you receive all five sets.
-Reference each question by the item id in its Trace ID (the `item_N` prefix, e.g.
-item_3). Never invent an id.
+Reference each question by its `question_id` as "Question N" (e.g. Question 3).
+Never invent an id.
 
 SCALE AND BANDS (use exactly these):
 - low = score 0 or 1
@@ -69,12 +69,12 @@
 1. An overall read in 2–3 lines: the general health of the run in plain warm
    language (you may note that KB grounding is strong, ground-truth is weak, the
    judge is unstable, etc.), no score dumps.
-2. "Top 3 to check:" — the three highest-severity items or item-clusters only,
-   each one line: the item id(s), the one-line reason, and if relevant whether the
-   fix likely lives in the model, the config, or the golden dataset. Group items
-   into one line only when they share the same root cause; otherwise list them
-   separately. If fewer than three real problems exist, list only the real ones.
-   Never pad to three.
+2. "Top 3 to check:" — the three highest-severity questions or question-clusters
+   only, each one line: the question id(s) as "Question N", the one-line reason,
+   and if relevant whether the fix likely lives in the model, the config, or the
+   golden dataset. Group questions into one line only when they share the same
+   root cause; otherwise list them separately. If fewer than three real problems
+   exist, list only the real ones. Never pad to three.
 3. One closing line reminding the reviewer these are go-verify pointers: open the
    items, confirm the answer actually holds for the stated condition, and decide
    based on what the use case needs.
@@ -99,14 +99,14 @@ def _format_traces_for_prompt(
 ) -> str:
     """Per-question judge traces as the summary model's brief.
 
-    Traces are handed over ungrouped: `trace_id` is the dataset item id
-    (`item_{row}_{dup}`), which is what the system prompt tells the model to key on.
+    Traces are handed over ungrouped: `question_id` is the 1-based dataset row
+    number, which is what the system prompt tells the model to key on.
     """
     # ponytail: every trace sent whole; ~250 traces (50 questions x dup 5) overflows
-    # context and degrades to None. Sample or group per item_id if runs get bigger.
+    # context and degrades to None. Sample or group per question_id if runs get bigger.
     payload = [
         {
-            "trace_id": trace["trace_id"],
+            "question_id": trace["question_id"],
             "question": trace["question"],
             "ground_truth_answer": trace["ground_truth_answer"],
             "llm_answer": trace["llm_answer"],

backend/app/tests/crud/evaluations/test_run_ai_summary.py

@@ -164,7 +164,7 @@ def test_blank_config_prompt_falls_back_to_the_placeholder(self) -> None:
     def test_each_trace_carries_its_qa_and_scored_rationales(self) -> None:
         payload = _payload_from(_brief_for())
 
-        assert [t["trace_id"] for t in payload] == ["item_1_1", "item_2_1"]
+        assert [t["question_id"] for t in payload] == [1, 2]
         first = payload[0]
         assert first["question"] == "How much urea per acre?"
         assert first["ground_truth_answer"] == "Roughly 45-55 kg per acre."
@@ -193,7 +193,7 @@ def test_trace_bookkeeping_keys_never_reach_the_model(self) -> None:
         payload = _payload_from(brief)
 
         assert set(payload[0]) == {
-            "trace_id",
+            "question_id",
             "question",
             "ground_truth_answer",
             "llm_answer",

docs/wiki/modules/evaluations.md

@@ -32,7 +32,7 @@ v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native jud
 ## Services / CRUD
 - `services/evaluations/` — `evaluation.py`, `dataset.py` (`upload_dataset`; `use_langfuse=False` is the v2 Langfuse-free upload), `fast.py` (`validate_fast_evaluation_inputs` extracted for reuse by both the direct eval-start path and the iteration loop), `batch_job.py`, `validators.py`, `prompt_improvement.py`, `iteration.py` (`validate_and_start_evaluation_iteration`, `compute_round_scores`), `iteration_graph.py` (the LangGraph `StateGraph`: nodes, checkpointer)
 - `services/stt_evaluations/`, `services/tts_evaluations/`
-- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop)
+- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth`, `prompt`, and `knowledge_base` metrics, applied per-row by which required inputs the row carries, each spec carrying a `weight` for the overall rollup), `score.py` (`VerdictEnum`/`verdict_from_score`, `OverallSummary`/`compute_overall_summary`), `summary.py` (`generate_run_ai_summary` — one-shot Anthropic `messages.create` call via `ClaudeProvider`, structured JSON output; prompt carries every trace's raw per-question scores + judge rationale, golden/generated answers, and the evaluated config, each trace keyed on `question_id` (the 1-based dataset row number from `merge.py`, cited back as "Question N") rather than the Langfuse `trace_id`, and returns a severity-ranked diagnostic note, not just a qualitative band summary), `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py`, `iteration.py` (thin-row CRUD for the iteration loop)
 - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py`
 
 ## Async