diff --git a/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md new file mode 100644 index 000000000..1b46bf3b7 --- /dev/null +++ b/dokimi/changelog.d/20261028_120000_napetrov_stack-p-step2.md @@ -0,0 +1,61 @@ +### Added + +- **`benchmark run --judge-temperature`** — a new flag: it sets the judge's sampling + temperature and is recorded as `judge_temperature` in the eval output. Before this change the + flag did not exist, and `benchmark run` rejected it as an unknown argument. + +### Changed + +- **Stack P scores move (#228 step 2)** — the context arm now asks the doc source for the + `top_k` docs the command was given: `benchmark run` asks for 3 (`DEFAULT_RETRIEVAL_TOP_K`; it + has no `--top-k`), `dokimi evaluate` and `answers generate` for their `--top-k` (default 3). + Before, they left `max_results` at the client default of 5. The doc library id resolves + through the registry's `context7_id` before the client's own resolver, in the `--registry` + that was loaded. A question with no document (no hit, or the reranker raised) is no longer judged on + its `[FALLBACK: ...]` placeholder by default, so it drops out of the paired statistics; the + record and its `retrieval_hit` stay, its diagnosis stays `empty_retrieval`, and the run + summary says how many questions it left out. Pass `--judge-placeholder` to judge the + placeholder again (this restores only that part of the old method). The dashboard's arm + means and delta leave such a question out too, as `report eval` does. Its doc score counts a + retrieval miss as 0 only when the baseline has a score and no error touched the pair (neither + arm has a judge error); a miss with an error leaves the doc score alone. `report eval` and `report generate` show `—` for the arm means and delta when + no question is scored in both arms (the `report generate --format json` `stats` block carries + `null` for `avg`, `min` and `max`), and the run summary prints no means then. `benchmark run`, + `evaluate` and `eval score` take the same flag; `benchmark batch` has none and never judges + the placeholder. Scores from before this change are not comparable with new ones. +- **Trust gate** — `retrieval_hit_rate` counts every evaluation's recorded retrieval outcome, + not only the ones scored in both arms, so unjudged misses still count against the rate, and a + judge-error row keeps the outcome of its answer. The rate is shown even when no question is + scored in both arms. A judge-error row of an older eval file, which has no `retrieval_hit`, + is left out of that count as before. The thresholds are unchanged. +- **Zero-score check** — a judge-error row on a retrieval miss now counts as unusable in + `zero_score_fraction` (a judge error on a retrieval hit is still left out), so a run can fail + that fail-severity check where intel/main passed it. +- **Stamped artifacts** — the answers and eval artifacts carry `method_version` (2). Artifacts + from before this change have no stamp and represent the legacy method (version 1); readers do + not yet normalize or enforce the version, so it does not yet stop a comparison of mixed methods. + `eval score` stamps its eval with the version of the answers file it judges, so legacy answers + re-judged today stay version 1. An `eval score` artifact also carries `judge_placeholder`, so two + files of one version that differ in it are told apart. `eval panel-score` (the JudgePanel path) + writes neither stamp yet, so its artifact reads as version 1 whatever answers it judged. An absent + or null stamp means version 1; any other value must be a positive integer, or `eval score` exits + 2 before any judge call. + +### Fixed + +- **Stack P no longer ignores `top_k`** — `Answerer` passes `max_results=top_k` to the doc + source; the context arm's retrieval request is now the one the arms stack makes. + `answers generate --top-k` takes a positive integer, as `dokimi evaluate` does, and a config + `retrieval.top_k` that is not one exits 2. +- **Library id from the registry** — `EvaluationPipeline` resolves the library id through + `resolve_library_id_for_product` (explicit id, then an `mcp:` source's `;id=`, then `context7_id`, + then the client), in the + registry that `benchmark run` and `benchmark batch` loaded from `--registry`. A registered + product without `context7_id` now fails for a client that needs a declared id. + The question step's topics cache (`.cache/topics`) is keyed on the doc source as well, so + two doc sets of one product (two `local:` dirs, say) no longer share their topics. + +### Removed + +- Dead code in the answering path: the unused `Answerer.llm` client and the unreachable + "no documents" branch of `_generate_with_docs`. diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md index 952f3ca56..a8e6259b0 100644 --- a/dokimi/docs/methodology.md +++ b/dokimi/docs/methodology.md @@ -199,11 +199,11 @@ RAGAS meta-evaluation can be run after answer generation to add faithfulness and Every serious run should answer "Can we trust this run?" before interpreting deltas. The trust gate checks: -- minimum evaluated question count, -- zero-score fraction, -- minimum `with_docs` average, -- average delta, -- score coefficient of variation, +- minimum evaluated question count (both arms scored; the single `Judge` leaves no-document fallback answers unscored by default, but includes them when `--judge-placeholder` is enabled and both arms receive scores; `JudgePanel` also judges nonempty fallback answers), +- zero-score fraction (computed when at least one question is scored in both arms; over those paired questions plus retrieval misses with an unscored context arm and either a scored baseline or a judge error, which count as unusable), +- minimum `with_docs` average (both-arm-scored questions only), +- average delta (both-arm-scored questions only), +- score coefficient of variation (both-arm-scored questions only), - inter-rater agreement when panel data exists, - retrieval hit rate — the share of questions where at least one document reached the `with_docs` prompt, from the per-evaluation `retrieval_hit` the judge and panel record; eval files judged before that field existed show it as not measured, and so do files where fewer than 80% of evaluations record it (`min_retrieval_coverage`), since the rate would then describe a subset, - multi-run variance when `--multi-run N` is used. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..70d10c11f 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -344,7 +344,7 @@ "nargs": null, "position": null, "required": false, - "type": "int" + "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" } ], "path": "dokimi answers generate", @@ -2299,6 +2299,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": [ @@ -2323,6 +2339,22 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--judge-temperature", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": "float" + }, { "action": "_StoreAction", "choices": null, @@ -5805,6 +5837,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": [ @@ -6015,6 +6063,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--judge-placeholder", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": [ diff --git a/dokimi/dokimi/commands/answers.py b/dokimi/dokimi/commands/answers.py index ad2601620..ecb1401f0 100644 --- a/dokimi/dokimi/commands/answers.py +++ b/dokimi/dokimi/commands/answers.py @@ -85,6 +85,10 @@ def cmd_answers_generate(args: argparse.Namespace) -> None: _retrieval_cfg = config.get("retrieval") or {} if args.top_k is None: args.top_k = _retrieval_cfg.get("top_k", DEFAULT_RETRIEVAL_TOP_K) + if isinstance(args.top_k, bool) or not isinstance(args.top_k, int) or args.top_k < 1: + print(f"Error: retrieval.top_k in the config must be a positive integer, " + f"got {args.top_k!r}", file=sys.stderr) + sys.exit(EXIT_USAGE) questions = _load_questions(args.questions) print(f"Loaded {len(questions)} questions from {args.questions}") @@ -162,7 +166,7 @@ def register(sub, positive_int) -> None: gen_a_p.add_argument("--model", default=DEFAULT_ANSWER_MODEL, help="LLM model for answering") gen_a_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=ANSWER_PROVIDERS) gen_a_p.add_argument("--max-tokens", type=int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, help="Max tokens to retrieve per question") - gen_a_p.add_argument("--top-k", type=int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})") + gen_a_p.add_argument("--top-k", type=positive_int, default=None, help=f"Number of docs to retrieve before reranking (default from config/products.yaml retrieval.top_k or {DEFAULT_RETRIEVAL_TOP_K})") gen_a_p.add_argument("--rerank-threshold", type=float, default=0.3, help="Min relevance score (0-1) to keep docs") gen_a_p.add_argument("--debug-retrieval", action="store_true", help="Include retrieval metadata in output") gen_a_p.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY, help="Parallel API calls (default: 5)") diff --git a/dokimi/dokimi/commands/benchmark.py b/dokimi/dokimi/commands/benchmark.py index 44b1c888b..2bce0174e 100644 --- a/dokimi/dokimi/commands/benchmark.py +++ b/dokimi/dokimi/commands/benchmark.py @@ -27,9 +27,20 @@ ) from dokimi.commands.library import _load_registry from dokimi.exit_codes import EXIT_USAGE +from dokimi.llm_cli import SamplingError, check_sampling -def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None) -> dict: +def _refuse_judge_sampling(args: argparse.Namespace) -> None: + """Exit 2 before any answer is paid for when a CLI judge is given a temperature (the judge would refuse later).""" + try: + check_sampling(getattr(args, "judge_provider", DEFAULT_JUDGE_PROVIDER), + getattr(args, "judge_temperature", None), None) + except SamplingError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) + + +def _run_single_library(entry, output_dir: str, model: str, provider: str, judge_model: str, judge_provider: str = DEFAULT_JUDGE_PROVIDER, doc_source_override=None, max_tokens_per_question: int = DEFAULT_MAX_TOKENS_PER_QUESTION, force_regen: bool = False, concurrency: int = DEFAULT_CONCURRENCY, questions_from=None, questions_root=None, judge_temperature=None, judge_placeholder=False, registry=None) -> dict: """Run full evaluation pipeline for one ProductEntry. Returns result dict.""" from dokimi.orchestrator import EvaluationPipeline from pathlib import Path as _Path @@ -60,12 +71,15 @@ def _run_single_library(entry, output_dir: str, model: str, provider: str, judge provider=provider, judge_model=judge_model, judge_provider=judge_provider, + judge_temperature=judge_temperature, + judge_placeholder=judge_placeholder, context7_id=entry.context7_id, doc_source=doc_source, max_tokens_per_question=max_tokens_per_question, force_regen=force_regen, questions_from=questions_from, product_key=entry.key, + registry=registry, questions_root=_Path(questions_root) if questions_root else None, ) result = pipeline.run(concurrency=concurrency) @@ -301,6 +315,7 @@ def refuse(message: str) -> None: def cmd_benchmark_run(args: argparse.Namespace) -> None: """Run full evaluation pipeline for a single registered library.""" import statistics as _stats + _refuse_judge_sampling(args) registry = _load_registry(args) try: entry = registry.get(args.library) @@ -334,6 +349,9 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: concurrency=concurrency, questions_from=questions_from, questions_root=questions_root, + judge_temperature=getattr(args, "judge_temperature", None), + judge_placeholder=getattr(args, "judge_placeholder", False), + registry=registry, ) else: print(f"\n🔁 Multi-run mode: {n_runs} evaluation passes") @@ -356,6 +374,9 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: concurrency=concurrency, questions_from=qfrom, questions_root=questions_root, + judge_temperature=getattr(args, "judge_temperature", None), + judge_placeholder=getattr(args, "judge_placeholder", False), + registry=registry, ) try: eval_summary = r["result"]["steps"]["evaluation"]["summary"] @@ -375,6 +396,7 @@ def cmd_benchmark_run(args: argparse.Namespace) -> None: def cmd_benchmark_batch(args: argparse.Namespace) -> None: """Run evaluation pipeline for multiple registered libraries.""" + _refuse_judge_sampling(args) registry = _load_registry(args) if args.all_libraries or not args.libraries: @@ -410,6 +432,7 @@ def cmd_benchmark_batch(args: argparse.Namespace) -> None: force_regen=getattr(args, "force_regen", False), concurrency=getattr(args, "concurrency", DEFAULT_CONCURRENCY), questions_root=getattr(args, "questions_root", None), + registry=registry, ) results.append(r) except Exception as exc: @@ -456,6 +479,10 @@ def register(sub, positive_int) -> None: help=JUDGE_MODEL_HELP) bench_run_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, dest="judge_provider", choices=JUDGE_PROVIDERS) + bench_run_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", + help="Judge sampling temperature, recorded as judge_temperature in the eval output (default: provider default)") + bench_run_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder", + help="Judge the no-retrieval placeholder answers too (default: they are not scored)") bench_run_p.add_argument("--registry", default=None, help="Path to custom products.yaml") bench_run_p.add_argument("--max-tokens", type=positive_int, default=DEFAULT_MAX_TOKENS_PER_QUESTION, dest="max_tokens", help="Max tokens to retrieve per question from doc source (default: 4000)") diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 18f1da851..dba67e5fc 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -18,6 +18,7 @@ DEFAULT_JUDGE_PROVIDER, JUDGE_MODEL_FOR_PROVIDER, JUDGE_MODEL_HELP, + answers_method_version, ) from dokimi.eval.judge import JUDGE_PROVIDERS from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs @@ -198,6 +199,11 @@ def cmd_eval_score(args: argparse.Namespace) -> None: context="eval-score", ) + try: + method_version = answers_method_version(run.meta) + except ValueError as exc: # refused before any judge call is paid for + print(f"Error: {args.answers}: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) print(f"Evaluating with judge: {args.judge_provider}/{args.judge_model}") # Evaluate @@ -205,6 +211,9 @@ def cmd_eval_score(args: argparse.Namespace) -> None: model=args.judge_model, provider=args.judge_provider, temperature=args.judge_temperature, + judge_placeholder=getattr(args, "judge_placeholder", False), + # The eval says which answering method made the answers it judges; an unstamped file is legacy. + answers_method_version=method_version, run_metadata={ "question_model": run.meta.get("question_model", "unknown"), "question_provider": run.meta.get("question_provider", "unknown"), @@ -480,6 +489,8 @@ def register(sub, positive_int) -> None: score_p.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY, help="Parallel judge calls (default: 5)") score_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", help="Judge sampling temperature, recorded as judge_temperature in the output") + score_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder", + help="Judge the no-retrieval placeholder answers too (default: they are not scored)") score_p.set_defaults(func=cmd_eval_score) # eval ragas diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index 32eed2c0d..f550333dc 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -68,6 +68,7 @@ def cmd_evaluate(args: argparse.Namespace) -> None: judge_model=args.judge_model, judge_provider=args.judge_provider, judge_temperature=args.judge_temperature, + judge_placeholder=getattr(args, "judge_placeholder", False), personas_count=args.personas_count, questions_per_topic=args.questions_per_topic, # Only pass top_k when explicitly set; otherwise let pipeline use config default @@ -130,6 +131,8 @@ def register(sub, positive_int) -> None: eval_p.add_argument("--judge-provider", default=DEFAULT_JUDGE_PROVIDER, choices=JUDGE_PROVIDERS) eval_p.add_argument("--judge-temperature", type=float, default=None, dest="judge_temperature", help="Judge sampling temperature, recorded as judge_temperature in the eval output") + eval_p.add_argument("--judge-placeholder", action="store_true", dest="judge_placeholder", + help="Judge the no-retrieval placeholder answers too (default: they are not scored)") eval_p.add_argument("--personas-count", type=positive_int, default=5, help="Target number of personas") eval_p.add_argument("--questions-per-topic", type=positive_int, default=2, help="Questions per topic per persona") eval_p.add_argument("--top-k", type=positive_int, default=None, help="Docs to retrieve before reranking (default from config/products.yaml retrieval.top_k)") diff --git a/dokimi/dokimi/dashboard/aggregator.py b/dokimi/dokimi/dashboard/aggregator.py index 9a1187293..fbc7bac7a 100644 --- a/dokimi/dokimi/dashboard/aggregator.py +++ b/dokimi/dokimi/dashboard/aggregator.py @@ -29,6 +29,9 @@ class QuestionResult: # Per-dimension scores with_docs_dimensions: Dict[str, float] = field(default_factory=dict) without_docs_dimensions: Dict[str, float] = field(default_factory=dict) + # True when the context score is the 0.0 of a retrieval miss the judge never saw (no judged score). + # It counts in the doc health score, not in the paired means. + unjudged_miss: bool = False @dataclass @@ -46,10 +49,14 @@ class ProductSnapshot: source_file: Optional[str] = None # Breakdown by question source by_source: Dict[str, Dict] = field(default_factory=dict) # {"persona": {...}, "chunk": {...}} + # Mean context score over every question, a retrieval miss counted as 0 (the doc health number). + doc_health: Optional[float] = None @property def doc_score(self) -> Optional[float]: - """Primary score: avg with_docs (or without_docs if no docs run).""" + """Primary score: the doc health (or avg with_docs, or without_docs if no docs run).""" + if self.doc_health is not None: + return self.doc_health return self.avg_with_docs if self.avg_with_docs is not None else self.avg_without_docs @property @@ -158,8 +165,9 @@ def _as_float(value: Any) -> Optional[float]: with_eval: Mapping[str, Any] = with_s.raw if with_s is not None else {} without_eval: Mapping[str, Any] = without_s.raw if without_s is not None else {} - with_score = _as_float(ev.score(CONTEXT)) without_score = _as_float(ev.score(BASELINE)) + judged_with = _as_float(ev.score(CONTEXT)) + with_score = _context_score(ev, with_s, judged_with, without_score) delta = _as_float(ev.delta) q_source = ev.raw.get("question_source", "persona") @@ -177,6 +185,7 @@ def _as_float(value: Any) -> Optional[float]: if isinstance(v, (int, float)) and k != "aggregate"}, without_docs_dimensions={k: v for k, v in without_eval.items() if isinstance(v, (int, float)) and k != "aggregate"}, + unjudged_miss=judged_with is None and with_score is not None, )) by_source = _compute_by_source(questions) @@ -194,9 +203,24 @@ def _as_float(value: Any) -> Optional[float]: questions=sorted(questions, key=lambda q: (q.with_docs_score or 0)), # worst first source_file=str(path), by_source=by_source, + doc_health=_doc_health(questions), ) +def _context_score(ev: Any, with_s: Any, with_score: Optional[float], without_score: Optional[float]) -> Optional[float]: + """The context arm's score; 0.0 for a no-error retrieval miss it never answered, with a baseline score.""" + if (with_score is None and ev.retrieval_hit is False and without_score is not None + and ev.error is None and (with_s is None or with_s.error is None)): + return 0.0 # the docs did not help + return with_score + + +def _doc_health(questions: "List[QuestionResult]") -> Optional[float]: + """Mean context score over every question with one; a retrieval miss is its 0.0.""" + vals = [q.with_docs_score for q in questions if q.with_docs_score is not None] + return round(sum(vals) / len(vals), 1) if vals else None + + def _compute_by_source(questions: "List[QuestionResult]") -> "Dict[str, Dict]": """Compute per-source breakdown stats (persona vs chunk).""" from typing import List as _L, Dict as _D @@ -220,13 +244,15 @@ def _arm_means(questions: "List[QuestionResult]") -> "Dict[str, Optional[float]] The questions scored in both arms, the ones :func:`dokimi.eval.stats.paired_summary` averages for ``report generate``, so the delta is the difference of the two means. - With no pair there is no delta: an arm keeps its mean over its own scores when the - other arm scored nothing (a baseline-only run), and otherwise neither mean is given, - so two means over different questions never stand side by side. + An unjudged retrieval miss is no measurement (its 0.0 is only the doc health), so it + is left out of both arms. With no pair there is no delta: an arm keeps its mean over + its own scores when the other arm scored nothing (a baseline-only run), and otherwise + neither mean is given, so two means over different questions never stand side by side. """ - with_s = [q.with_docs_score for q in questions if q.with_docs_score is not None] - without_s = [q.without_docs_score for q in questions if q.without_docs_score is not None] - pairs = [(q.with_docs_score, q.without_docs_score) for q in questions + measured = [q for q in questions if not q.unjudged_miss] + with_s = [q.with_docs_score for q in measured if q.with_docs_score is not None] + without_s = [q.without_docs_score for q in measured if q.without_docs_score is not None] + pairs = [(q.with_docs_score, q.without_docs_score) for q in measured if q.with_docs_score is not None and q.without_docs_score is not None] if pairs: with_s, without_s = [w for w, _ in pairs], [wo for _, wo in pairs] diff --git a/dokimi/dokimi/defaults.py b/dokimi/dokimi/defaults.py index 10edc9e20..b0b0d26c2 100644 --- a/dokimi/dokimi/defaults.py +++ b/dokimi/dokimi/defaults.py @@ -20,6 +20,7 @@ """ import os +from typing import Any, Mapping DEFAULT_ANSWER_PROVIDER = "openai" DEFAULT_ANSWER_MODEL = "gpt-6.1-sol" @@ -91,6 +92,24 @@ class _JudgeModelForProvider(str): # Decision (2026-03-04): one fixed semantic retrieval config. DEFAULT_RETRIEVAL_TOP_K = 3 + +# Answering/judging method version, stamped into answers and eval artifacts; bump it when a change moves scores. +# A later compare is meant to refuse mixed versions; an unstamped artifact predates it (#228 step 2) and is v1. +ANSWERING_METHOD_VERSION = 2 +#: The version of an artifact that carries no stamp (written before ANSWERING_METHOD_VERSION existed). +LEGACY_ANSWERING_METHOD_VERSION = 1 + + +def answers_method_version(meta: Mapping[str, Any]) -> int: + """The method_version an answers artifact carries; absent or null is legacy, else a positive int.""" + value = meta.get("method_version") + if value is None: + return LEGACY_ANSWERING_METHOD_VERSION + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"method_version must be a positive integer, got {value!r}") + return value + + DEFAULT_MAX_TOKENS_PER_QUESTION = 4000 DEFAULT_CONCURRENCY = 5 DEFAULT_RESULTS_SUFFIX = "_final" diff --git a/dokimi/dokimi/eval/answerer.py b/dokimi/dokimi/eval/answerer.py index 5e2dca750..a80f4f03e 100644 --- a/dokimi/dokimi/eval/answerer.py +++ b/dokimi/dokimi/eval/answerer.py @@ -15,10 +15,14 @@ from dokimi.utils import question_id from dokimi.artifacts import save_artifact -from dokimi.llm import ChatModel +from dokimi.defaults import ( + ANSWERING_METHOD_VERSION, + DEFAULT_ANSWER_MODEL, + DEFAULT_ANSWER_PROVIDER, + DEFAULT_RETRIEVAL_TOP_K, +) +from dokimi.llm import llm_call_with_usage, LANGCHAIN_AVAILABLE from dokimi.llm_cli import BACKENDS as CLI_BACKENDS -from dokimi.llm import llm_call_with_usage, ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE -from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER from dokimi.metrics.usage import UsageRecord, to_record from .reranker import SimpleReranker, SentenceTransformerReranker, SENTENCE_TRANSFORMERS_AVAILABLE @@ -71,7 +75,7 @@ def __init__( model: str = DEFAULT_ANSWER_MODEL, provider: str = DEFAULT_ANSWER_PROVIDER, api_key: Optional[str] = None, - top_k: int = 5, + top_k: Optional[int] = None, rerank_threshold: float = 0.3, debug_retrieval: bool = False ): @@ -81,7 +85,7 @@ def __init__( model: LLM model name provider: "openai" or "anthropic" api_key: Optional API key - top_k: Number of docs to retrieve before reranking + top_k: Number of docs to retrieve before reranking (default DEFAULT_RETRIEVAL_TOP_K) rerank_threshold: Minimum relevance score (0-1) to keep docs debug_retrieval: If True, include detailed retrieval metadata in output """ @@ -96,20 +100,15 @@ def __init__( self.provider = provider self.api_key = api_key - self.llm: Optional[ChatModel] - if provider == "openai": - self.llm = ChatOpenAI(model=model, api_key=api_key) - elif provider == "anthropic": - self.llm = ChatAnthropic(model=model, api_key=api_key) - elif provider in ("google", "gemini", "google-vertex", "openrouter", - "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): - self.llm = None # llm_call_with_usage handles these natively - else: + # The answer call goes through llm_call_with_usage, which picks the provider's client itself. + if provider not in ("openai", "anthropic", "google", "gemini", "google-vertex", + "openrouter", "amazon-bedrock", "vertex_ai", *CLI_BACKENDS): raise ValueError( f"Unsupported provider: '{provider}'. " "Use 'openai', 'anthropic', 'google', 'openrouter', 'amazon-bedrock' or 'claude-cli'." ) - self.top_k = top_k + # None means the default: the doc source must never get max_results=None (it would return every chunk). + self.top_k = DEFAULT_RETRIEVAL_TOP_K if top_k is None else top_k self.debug_retrieval = debug_retrieval # Initialize reranker (prefer semantic, fallback to lexical) @@ -316,12 +315,11 @@ def _retrieve_docs( } try: - # Retrieve top-k docs - # Note: Context7 client returns single doc, so we call it once for now - # TODO: Update Context7 client to support top_k natively + # Retrieve top-k docs, the same request the arms stack sends (max_results=top_k) raw_docs = self.mcp_client.get_library_docs( library_id=library_id, query=question, + max_results=self.top_k, max_tokens=max_tokens ) metadata["raw_count"] = len(raw_docs) @@ -359,24 +357,7 @@ def _generate_with_docs( retrieval_metadata: Optional[Dict[str, Any]] = None, verbatim: bool = False, ) -> Dict[str, Any]: - """Generate context-arm answer.""" - if not docs: - # No LLM call is made — emit a zero-cost/zero-token metrics block so - # this row carries the same telemetry shape as answered rows. - empty = UsageRecord(model=self.model, provider=self.provider, - cost_usd=0.0, n_calls=0) - token_usage = empty.as_token_usage_dict() - token_usage["context_chars"] = 0 - return { - "answer": "[No documentation retrieved]", - "retrieved_docs": [], - "model": self.model, - "doc_source": "none", - "retrieval_metadata": retrieval_metadata if self.debug_retrieval else None, - "token_usage": token_usage, - "metrics": empty.as_metrics_dict(answer_chars=0), - } - + """Generate context-arm answer. Called only with a non-empty *docs* (see ``_generate_answer_pair``).""" # ~4k tokens at ~4 chars/token; keeps prompt within most models' context window MAX_DOC_CHARS = 15_000 @@ -459,6 +440,7 @@ def _build_output(self, answers: List[Dict[str, Any]]) -> Dict[str, Any]: return { "generated_at": self._get_timestamp(), + "method_version": ANSWERING_METHOD_VERSION, "model": self.model, "provider": self.provider, "total_questions": len(answers), diff --git a/dokimi/dokimi/eval/diagnoser.py b/dokimi/dokimi/eval/diagnoser.py index 15e194e6d..1283554f0 100644 --- a/dokimi/dokimi/eval/diagnoser.py +++ b/dokimi/dokimi/eval/diagnoser.py @@ -30,12 +30,13 @@ added noise rather than signal. insufficient_data - Delta could not be computed (one or both answers missing). + Delta could not be computed (one or both answers missing or errored). A + retrieval miss that was left unjudged is ``empty_retrieval``, not this. """ from typing import Any, Dict, Iterable, List, Mapping, Optional -from dokimi.records import Answer, RecordLike, records +from dokimi.records import NO_RETRIEVAL_SOURCES, Answer, RecordLike, records # Keyword-overlap score below which retrieved docs are considered off-topic LOW_RELEVANCE_CUTOFF = 0.3 @@ -73,6 +74,7 @@ def diagnose( context_answer: Optional[Answer], delta: Optional[float], low_relevance_cutoff: float = LOW_RELEVANCE_CUTOFF, + context_scored: bool = False, ) -> Dict[str, Any]: """ Classify the retrieval outcome for a single question. @@ -86,6 +88,9 @@ def diagnose( Context-arm score minus baseline score, ``None`` when either is missing. low_relevance_cutoff: Relevance score below which retrieved docs are treated as off-topic. + context_scored: + True when the judge scored the context answer (``judge_placeholder``), so a + ``None`` delta comes from a missing baseline, not from an unjudged miss. Returns ------- @@ -95,6 +100,11 @@ def diagnose( - ``evidence``: dict of signals used to reach the diagnosis """ if delta is None: + # A retrieval miss the judge did not score has no delta; it is still an + # empty retrieval, not a missing or errored answer. + unjudged = not context_scored + if unjudged and context_answer is not None and not context_answer.error and context_answer.doc_source in NO_RETRIEVAL_SOURCES: + return _result(EMPTY_RETRIEVAL, {"doc_source": context_answer.doc_source, "retrieved_count": 0}) return _result(INSUFFICIENT_DATA, {}) if delta >= 0: diff --git a/dokimi/dokimi/eval/judge.py b/dokimi/dokimi/eval/judge.py index afd8b1317..faf1a398e 100644 --- a/dokimi/dokimi/eval/judge.py +++ b/dokimi/dokimi/eval/judge.py @@ -22,8 +22,8 @@ from dokimi.llm import ChatOpenAI, ChatAnthropic, LANGCHAIN_AVAILABLE from dokimi.eval.diagnoser import diagnose from dokimi.eval.question_text import require_question_texts -from dokimi.records import BASELINE, CONTEXT, Record, RecordLike, records -from dokimi.defaults import DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER +from dokimi.records import BASELINE, CONTEXT, NO_RETRIEVAL_SOURCES, Record, RecordLike, records +from dokimi.defaults import ANSWERING_METHOD_VERSION, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER # One calibration scale for both the single judge and the panel, so their # numbers mean the same thing and can be compared. from dokimi.eval.rubric import ( @@ -197,7 +197,12 @@ class Judge: - Code Quality - Actionability """ - + + #: Class default, so a Judge built without __init__ (tests build one with __new__) keeps the default. + judge_placeholder: bool = False + #: Same for the stamp: the eval artifact carries the method version of the answers it judged. + answers_method_version: int = ANSWERING_METHOD_VERSION + def __init__( self, model: str = DEFAULT_JUDGE_MODEL, @@ -207,6 +212,8 @@ def __init__( temperature: Optional[float] = None, seed: Optional[int] = None, prompt_template: Optional[str] = None, + judge_placeholder: bool = False, + answers_method_version: int = ANSWERING_METHOD_VERSION, ): """ Args: @@ -220,6 +227,12 @@ def __init__( prompt_template: Replaces :data:`JUDGE_PROMPT` on the ungrounded path. Exists so the judge can be measured under a rewording of its own prompt; production scoring leaves it unset. + judge_placeholder: Judge a context answer that is the no-retrieval + placeholder ("[FALLBACK: ...]") too. Off by default: a question + with no document reaching the prompt is not scored as an answer. + answers_method_version: The method version of the answers being judged, + stamped into the eval artifact. Answers read from an artifact with no + stamp are version 1; answers generated in this process are the current version. """ if not LANGCHAIN_AVAILABLE: raise ImportError( @@ -235,6 +248,8 @@ def __init__( self.prompt_sha256 = JUDGE_PROMPT_SHA256 self.run_metadata = dict(run_metadata) if run_metadata else {} self.prompt_template = prompt_template + self.judge_placeholder = judge_placeholder + self.answers_method_version = answers_method_version # Accumulate judge-side token/cost telemetry across all judge calls # (Phase-C.0 covered answering; the judge path was untracked). self._usage_lock = threading.Lock() @@ -325,7 +340,10 @@ def _process(idx: int, answer: Record) -> None: "error": str(e), "with_docs": None, "without_docs": None, - "delta": None + "delta": None, + # What retrieval returned does not depend on the judge: the trust + # gate's coverage floor must not count this row as unrecorded. + "retrieval_hit": answer.retrieval_hit, } elapsed = time.time() - t0 with lock: @@ -376,9 +394,11 @@ def _evaluate_answer_pair( context_answer = answer.answer(CONTEXT) baseline_answer = answer.answer(BASELINE) - # Evaluate context arm + # Evaluate context arm. The no-retrieval placeholder is not an answer: it is + # left unjudged unless the caller opted in with judge_placeholder. with_docs_eval = None - if context_answer is not None and context_answer.text: + placeholder = context_answer is not None and context_answer.doc_source in NO_RETRIEVAL_SOURCES + if context_answer is not None and context_answer.text and (self.judge_placeholder or not placeholder): with_docs_eval = self._judge_answer( library_name=library_name, question=question_text, @@ -415,7 +435,7 @@ def _evaluate_answer_pair( } # Classify retrieval outcome (why docs helped or didn't) - result["diagnosis"] = diagnose(context_answer, delta) + result["diagnosis"] = diagnose(context_answer, delta, context_scored=with_docs_eval is not None) # Carried onto the eval record because the trust gate reads eval.v1, not # the answers it was judged from. result["retrieval_hit"] = answer.retrieval_hit @@ -612,6 +632,8 @@ def _build_evaluation_output(self, evaluations: List[Dict[str, Any]]) -> Dict[st """Build the serialisable output structure for a set of evaluations.""" output: Dict[str, Any] = { "evaluated_at": self._get_timestamp(), + "method_version": self.answers_method_version, + "judge_placeholder": self.judge_placeholder, "judge_model": self.model, "judge_provider": self.provider, "judge_temperature": self.temperature, diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py index fda12ff5f..72b8ad912 100644 --- a/dokimi/dokimi/gate/trust_gate.py +++ b/dokimi/dokimi/gate/trust_gate.py @@ -15,7 +15,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, Iterable, List, Optional -from dokimi.records import BASELINE, CONTEXT, RecordLike, records +from dokimi.records import BASELINE, CONTEXT, Record, RecordLike, records # ── Thresholds (can be overridden via spec/config) ──────────────────────────── @@ -94,6 +94,61 @@ def status(self) -> str: return "✅ PASS" +def _retrieval_check(every: List[Record], cfg: Dict[str, Any]) -> TrustCheck: + """The retrieval hit rate over every evaluation, ``measured=False`` when it cannot be read. + + A question "hit" retrieval when at least one document reached the + context-arm prompt. The judge stamps that per evaluation as + ``retrieval_hit`` (from the answer's retrieved_docs / doc_source); eval + artifacts judged before that carry no retrieval outcome at all, and the + check reports "not measured" for them rather than a pass. So does a mix of + the two below ``min_retrieval_coverage``: the rate would describe a subset. + Over every evaluation, not only the both-arms-scored ones: a question with no + document is not judged by default, and it still records that it missed. + """ + measured = [h for r in every if (h := r.retrieval_hit) is not None] + # A legacy judge-error row has no retrieval_hit: left out of the count, as before. + n_all = sum(1 for r in every if not (r.error is not None and r.retrieval_hit is None)) + if measured and len(measured) / n_all < cfg["min_retrieval_coverage"]: + return TrustCheck( + name="retrieval_hit_rate", + passed=True, + value=None, + threshold=cfg["min_retrieval_hit_rate"], + message=( + f"Not measured: only {len(measured)} of {n_all} evaluations record a retrieval " + f"outcome (retrieval_hit), under the {cfg['min_retrieval_coverage']:.0%} coverage " + f"floor; {sum(measured)} of those {len(measured)} hit" + ), + severity="warn", + measured=False, + ) + if measured: + retrieval_hits = sum(measured) + hit_rate = retrieval_hits / len(measured) + coverage = "" if len(measured) == n_all else f"; {n_all - len(measured)} unrecorded" + return TrustCheck( + name="retrieval_hit_rate", + passed=hit_rate >= cfg["min_retrieval_hit_rate"], + value=round(hit_rate, 3), + threshold=cfg["min_retrieval_hit_rate"], + message=( + f"Retrieval hit rate = {hit_rate:.1%} ({retrieval_hits}/{len(measured)}{coverage}) " + f"(min {cfg['min_retrieval_hit_rate']:.0%})" + ), + severity="warn", + ) + return TrustCheck( + name="retrieval_hit_rate", + passed=True, + value=None, + threshold=cfg["min_retrieval_hit_rate"], + message="Not measured: the evaluations record no retrieval outcome (retrieval_hit)", + severity="warn", + measured=False, + ) + + def evaluate_trust( evaluations: Iterable[RecordLike], thresholds: Optional[Dict[str, Any]] = None, @@ -114,9 +169,11 @@ def evaluate_trust( cfg = {**DEFAULTS, **(thresholds or {})} checks: List[TrustCheck] = [] - # Every check is over the questions scored in both arms. + # Parsed once: ``evaluations`` may be a one-shot iterable. + every = records(evaluations, kind="evaluations") + # The score checks below are over the questions scored in both arms. valid = [ - r for r in records(evaluations, kind="evaluations") + r for r in every if r.score(CONTEXT) is not None and r.score(BASELINE) is not None ] with_scores: List[float] = [r.score(CONTEXT) or 0 for r in valid] @@ -133,17 +190,31 @@ def evaluate_trust( severity="fail", )) if n == 0: + # Nothing is scored in both arms (every question an unjudged miss, say): the + # retrieval rate is still known, and it is the cause, so it goes beside the count. + if any(r.retrieval_hit is not None for r in every): + checks.append(_retrieval_check(every, cfg)) return TrustVerdict(checks=checks) # can't compute anything else # ── 2. Zero-score fraction ───────────────────────────────────────────── - zero_with = sum(1 for v in with_scores if v == 0) - zero_frac = zero_with / n + # Misses count as unusable, judge-error misses too (by their error field): else many misses would pass. + unjudged_misses = [ + r for r in every + if r.score(CONTEXT) is None and r.retrieval_hit is False + and (r.score(BASELINE) is not None or r.error is not None) + ] + zero_with = sum(1 for v in with_scores if v == 0) + len(unjudged_misses) + unusable_n = n + len(unjudged_misses) + zero_frac = zero_with / unusable_n checks.append(TrustCheck( name="zero_score_fraction", passed=zero_frac <= cfg["max_zero_fraction"], value=round(zero_frac, 3), threshold=cfg["max_zero_fraction"], - message=f"{zero_with}/{n} questions scored 0 in the context arm ({zero_frac:.1%})", + message=( + f"{zero_with}/{unusable_n} questions scored 0 or had no document in the context " + f"arm ({zero_frac:.1%})" + ), severity="fail", )) @@ -195,52 +266,7 @@ def evaluate_trust( )) # ── 7. Retrieval hit rate ───────────────────────────────────────────── - # A question "hit" retrieval when at least one document reached the - # context-arm prompt. The judge stamps that per evaluation as - # ``retrieval_hit`` (from the answer's retrieved_docs / doc_source); eval - # artifacts judged before that carry no retrieval outcome at all, and the - # check reports "not measured" for them rather than a pass. So does a mix of - # the two below ``min_retrieval_coverage``: the rate would describe a subset. - measured = [h for r in valid if (h := r.retrieval_hit) is not None] - if measured and len(measured) / n < cfg["min_retrieval_coverage"]: - checks.append(TrustCheck( - name="retrieval_hit_rate", - passed=True, - value=None, - threshold=cfg["min_retrieval_hit_rate"], - message=( - f"Not measured: only {len(measured)} of {n} evaluations record a retrieval " - f"outcome (retrieval_hit), under the {cfg['min_retrieval_coverage']:.0%} coverage " - f"floor; {sum(measured)} of those {len(measured)} hit" - ), - severity="warn", - measured=False, - )) - elif measured: - retrieval_hits = sum(measured) - hit_rate = retrieval_hits / len(measured) - coverage = "" if len(measured) == n else f"; {n - len(measured)} unrecorded" - checks.append(TrustCheck( - name="retrieval_hit_rate", - passed=hit_rate >= cfg["min_retrieval_hit_rate"], - value=round(hit_rate, 3), - threshold=cfg["min_retrieval_hit_rate"], - message=( - f"Retrieval hit rate = {hit_rate:.1%} ({retrieval_hits}/{len(measured)}{coverage}) " - f"(min {cfg['min_retrieval_hit_rate']:.0%})" - ), - severity="warn", - )) - else: - checks.append(TrustCheck( - name="retrieval_hit_rate", - passed=True, - value=None, - threshold=cfg["min_retrieval_hit_rate"], - message="Not measured: the evaluations record no retrieval outcome (retrieval_hit)", - severity="warn", - measured=False, - )) + checks.append(_retrieval_check(every, cfg)) # ── 8. Multi-run variance (optional) ───────────────────────────────── multirun_std = None diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index 690afc766..60bd22c1a 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -9,9 +9,10 @@ from typing import List, Dict, Any, Optional from pathlib import Path -from dokimi.defaults import DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K +from dokimi.defaults import (ANSWERING_METHOD_VERSION, DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, DEFAULT_JUDGE_MODEL, DEFAULT_JUDGE_PROVIDER, DEFAULT_RETRIEVAL_TOP_K, + answers_method_version as answers_method_version_of) from dokimi.eval.stats import paired_summary -from dokimi.mcp.factory import create_doc_source_client +from dokimi.mcp.factory import create_doc_source_client, resolve_library_id_for_product from dokimi.utils import same_evaluator logger = logging.getLogger(__name__) @@ -96,6 +97,8 @@ def __init__( questions_from: Optional[Path] = None, product_key: Optional[str] = None, questions_root: Optional[Path] = None, + registry: Optional[Any] = None, + judge_placeholder: bool = False, ): """ Initialize pipeline. @@ -136,6 +139,10 @@ def __init__( skipped; when it is absent a warning is logged and questions are generated. Without this argument (and without ``custom_questions_path``) the run has no golden set. + registry: ProductRegistry used to find the declared context7_id + (default: the bundled products.yaml) + judge_placeholder: Also judge a with-docs answer that is the + no-retrieval placeholder (default: recorded, not scored) """ if not repo and not description: raise ValueError( @@ -192,6 +199,8 @@ def __init__( self.judge_model = judge_model self.judge_provider = judge_provider self.judge_temperature = judge_temperature + # Judge the no-retrieval placeholder answers too (off: they are not scored). + self.judge_placeholder = judge_placeholder self.personas_count = personas_count self.questions_per_topic = questions_per_topic @@ -200,6 +209,8 @@ def __init__( self.debug_retrieval = debug_retrieval self.doc_source = doc_source self.context7_id = context7_id + # Product registry for the library id (defaults to the bundled products.yaml). + self.registry = registry self.max_tokens_per_question = max_tokens_per_question self.force_regen = force_regen @@ -383,6 +394,10 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]: # Summary stats over the complete pairs, the numbers the reports print for the same file. summary = paired_summary(evaluations) + # Questions scored in one arm only (a retrieval miss is left unjudged, a judge may fail) + # are in none of the averages: say how many, so the line is not read as covering all. + results["steps"]["evaluation"]["paired"] = summary.paired + results["steps"]["evaluation"]["unpaired"] = summary.unpaired if summary.paired: results["steps"]["evaluation"]["summary"] = { @@ -393,7 +408,9 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]: print(f"✓ Evaluated {len(evaluations)} answers: " f"context={results['steps']['evaluation']['summary']['with_avg']}, " f"baseline={results['steps']['evaluation']['summary']['without_avg']}, " - f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}") + f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}" + + (f" (over the {summary.paired} scored in both arms; {summary.unpaired} left out)" + if summary.unpaired else "")) else: print(f"✓ Evaluated {len(evaluations)} answers (summary unavailable)") @@ -442,15 +459,30 @@ def _discover_personas(self) -> Dict[str, Any]: return personas + def _library_id(self, mcp_client) -> str: + """The doc-source library id: explicit id, then the ``;id=`` of an ``mcp:`` source, then the + registry's ``context7_id``, then the client. + + The caller's own choices (the explicit id, the source's descriptor) go before the registry, as + they did before the pipeline used the registry lookup; ``eval/arms_run.py`` resolves registry-first. + """ + descriptor_id = getattr(mcp_client, "default_library_id", None) + return resolve_library_id_for_product( + self.product_key, mcp_client, explicit_id=self.context7_id or descriptor_id, registry=self.registry, + ) + def _generate_questions(self, personas: Dict[str, Any]) -> List[Dict[str, Any]]: """Generate questions from personas.""" from dokimi.questions import RagasSeedExtractor, QuestionGenerator # Extract topics mcp_client = create_doc_source_client(self.doc_source) - library_id = self.context7_id or mcp_client.resolve_library_id(self.product) - - extractor = RagasSeedExtractor(mcp_client=mcp_client, cache_dir=Path(".cache/topics")) + library_id = self._library_id(mcp_client) + + # The id is the registry's for every source, so the topics cache is also keyed on the source. + source_key = hashlib.sha256(self.doc_source.encode("utf-8")).hexdigest()[:16] + extractor = RagasSeedExtractor(mcp_client=mcp_client, + cache_dir=Path(".cache/topics") / source_key) topics = extractor.extract_topics( library_id=library_id, library_name=self.product, @@ -617,8 +649,8 @@ def _generate_answers(self, questions: List[Dict[str, Any]], concurrency: int = # Setup documentation source client mcp_client = create_doc_source_client(self.doc_source) - library_id = self.context7_id or mcp_client.resolve_library_id(self.product) - + library_id = self._library_id(mcp_client) + # Generate answers answerer = Answerer( mcp_client=mcp_client, @@ -653,15 +685,23 @@ def _evaluate_answers(self, answers: List[Dict[str, Any]] | Dict[str, Any], # Use in-memory answers from the previous step (avoid stale disk read); # fall back to the answers file when none were passed. if answers: - answers_list = list(read(answers, source="answers", section="answers").records) + run = read(answers, source="answers", section="answers") + else: + run = load(self.answers_path, section="answers") + answers_list = list(run.records) + # Fresh in-memory answers are this process's method; an artifact says its own, unstamped is legacy. + if answers and isinstance(answers, list): + answers_method_version = ANSWERING_METHOD_VERSION else: - answers_list = list(load(self.answers_path, section="answers").records) + answers_method_version = answers_method_version_of(run.meta) # Evaluate judge = Judge( model=self.judge_model, provider=self.judge_provider, temperature=self.judge_temperature, + judge_placeholder=self.judge_placeholder, + answers_method_version=answers_method_version, run_metadata={ "question_model": self.model, "question_provider": self.provider, diff --git a/dokimi/dokimi/report/eval_report.py b/dokimi/dokimi/report/eval_report.py index 29d91ad3f..fc7e93d55 100755 --- a/dokimi/dokimi/report/eval_report.py +++ b/dokimi/dokimi/report/eval_report.py @@ -115,6 +115,8 @@ def _unpaired_note(s: dict) -> list[str]: """A line saying what the Summary means leave out; nothing when every question is paired.""" if not s["unpaired"]: return [] + if not s["valid"]: + return [f'_No complete pairs: {s["unpaired"]} unpaired questions, so no arm mean or delta is given._', ""] return [ f'_Arm means and delta are over the {s["valid"]} complete pairs; ' f'{s["unpaired"]} unpaired (scored in one arm only) left out._', @@ -122,6 +124,36 @@ def _unpaired_note(s: dict) -> list[str]: ] +def _summary_cells(st: dict) -> tuple[str, str, str]: + """The context, baseline and delta cells of a Summary row: an em dash when no question is paired.""" + if not st["valid"]: + return "—", "—", "—" + return str(st["with_avg"]), str(st["without_avg"]), f'**{st["delta_avg"]:+}**' + + +def _overall_line(st: dict) -> str: + """The Overall delta bullet, or a note that no question is paired.""" + if not st["valid"]: + return "- **Overall delta:** none — no complete pairs (a question scored in both arms), so there is no delta." + verdict = "context arm is helping." if st["delta_avg"] > 2 else "context arm is neutral or slightly harmful." + return f'- **Overall delta: {st["delta_avg"]:+}** — {verdict}' + + +def _agreement_line(dyn: dict, static: dict) -> str: + """The Dynamic vs Static bullet; no agreement is claimed when either set has no complete pair.""" + if not (dyn["valid"] and static["valid"]): + return "- **Dynamic vs Static agreement:** none — a set has no complete pairs, so there is no delta to compare." + gap = abs(dyn["delta_avg"] - static["delta_avg"]) + verdict = "good consistency, benchmark is stable." if gap < 3 else "notable gap — investigate question quality." + return f'- **Dynamic vs Static agreement:** {dyn["delta_avg"]:+} vs {static["delta_avg"]:+} — {verdict}' + + +def _delta_summary(st: dict) -> str: + if not st["valid"]: + return " Delta: none (no complete pairs)" + return f" Delta: {st['delta_avg']:+} (with={st['with_avg']}, without={st['without_avg']})" + + def fmt_delta(d): if d is None: return "N/A" @@ -282,9 +314,9 @@ def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = Non "## Summary", "| Set | Count | Context arm | Baseline | Delta |", "|---|---:|---:|---:|---:|", - f'| **All** | {_count_cell(all_stats)} | {all_stats["with_avg"]} | {all_stats["without_avg"]} | **{all_stats["delta_avg"]:+}** |', - f'| Generated (dynamic) | {_count_cell(dyn_stats)} | {dyn_stats["with_avg"]} | {dyn_stats["without_avg"]} | **{dyn_stats["delta_avg"]:+}** |', - f'| Golden (static) | {_count_cell(static_stats)} | {static_stats["with_avg"]} | {static_stats["without_avg"]} | **{static_stats["delta_avg"]:+}** |', + f'| **All** | {_count_cell(all_stats)} | {" | ".join(_summary_cells(all_stats))} |', + f'| Generated (dynamic) | {_count_cell(dyn_stats)} | {" | ".join(_summary_cells(dyn_stats))} |', + f'| Golden (static) | {_count_cell(static_stats)} | {" | ".join(_summary_cells(static_stats))} |', "", ] lines += _unpaired_note(all_stats) @@ -338,8 +370,8 @@ def _val(key): ("**Total**", all_stats), ]: lines.append( - f'| {label} | {_count_cell(s)} | {s["with_avg"]} | {s["without_avg"]} ' - f'| **{s["delta_avg"]:+}** | {s["improvements"]} | {s["degradations"]} | {s["neutral"]} |' + f'| {label} | {_count_cell(s)} | {" | ".join(_summary_cells(s))} ' + f'| {s["improvements"]} | {s["degradations"]} | {s["neutral"]} |' ) lines.append("") @@ -353,8 +385,8 @@ def _val(key): if d in diff_groups: s = stats(diff_groups[d]) lines.append( - f'| {d} | {_count_cell(s)} | {s["with_avg"]} | {s["without_avg"]} ' - f'| **{s["delta_avg"]:+}** | {s["improvements"]} | {s["degradations"]} |' + f'| {d} | {_count_cell(s)} | {" | ".join(_summary_cells(s))} ' + f'| {s["improvements"]} | {s["degradations"]} |' ) lines.append("") @@ -464,12 +496,8 @@ def _val(key): lines += [ "## Key Observations & Recommendations", "", - f'- **Overall delta: {all_stats["delta_avg"]:+}** — ' - + ("context arm is helping." if all_stats["delta_avg"] > 2 else - "context arm is neutral or slightly harmful."), - f'- **Dynamic vs Static agreement:** {dyn_stats["delta_avg"]:+} vs {static_stats["delta_avg"]:+} — ' - + ("good consistency, benchmark is stable." if abs(dyn_stats["delta_avg"] - static_stats["delta_avg"]) < 3 - else "notable gap — investigate question quality."), + _overall_line(all_stats), + _agreement_line(dyn_stats, static_stats), f'- **Win rate:** {round(all_stats["improvements"]/all_stats["valid"]*100) if all_stats["valid"] else 0}% questions improved with context ' f'({all_stats["improvements"]}/{all_stats["valid"]}).', f'- **Degradations:** {all_stats["degradations"]} questions ' @@ -522,7 +550,7 @@ def _answer_text(a: Record | None, condition: str): print(f"✅ Report written to: {out_path}") print(f"✅ Q&A JSON written to: {qa_json_out}") print(f" Questions: {all_stats['count']} ({dyn_stats['count']} dynamic + {static_stats['count']} static)") - print(f" Delta: {all_stats['delta_avg']:+} (with={all_stats['with_avg']}, without={all_stats['without_avg']})") + print(_delta_summary(all_stats)) return out_path diff --git a/dokimi/dokimi/report/generator.py b/dokimi/dokimi/report/generator.py index 116542726..734c1c3a9 100644 --- a/dokimi/dokimi/report/generator.py +++ b/dokimi/dokimi/report/generator.py @@ -158,11 +158,9 @@ def _compute_stats(self, evaluations: Sequence[Record]) -> Dict[str, Any]: "total_questions": s.total, "paired_questions": s.paired, "unpaired_questions": s.unpaired, - "with_docs": {"avg": round(s.with_mean, 1), "min": round(s.with_min, 1), "max": round(s.with_max, 1)}, - "without_docs": { - "avg": round(s.without_mean, 1), "min": round(s.without_min, 1), "max": round(s.without_max, 1), - }, - "delta": {"avg": round(s.delta_mean, 1), "min": round(s.delta_min, 1), "max": round(s.delta_max, 1)}, + "with_docs": _stat_block(s, s.with_mean, s.with_min, s.with_max), + "without_docs": _stat_block(s, s.without_mean, s.without_min, s.without_max), + "delta": _stat_block(s, s.delta_mean, s.delta_min, s.delta_max), "doc_improvements": s.improvements, "doc_degradations": s.degradations, } @@ -253,7 +251,7 @@ def _cluster_by_topic( clusters.append({"topic": topic, **_arm_means(evals)}) # Sort by delta (worst first) - clusters.sort(key=lambda x: x["delta_avg"]) + clusters.sort(key=lambda x: (x["delta_avg"] is None, x["delta_avg"] or 0.0)) return clusters @@ -309,9 +307,12 @@ def _format_markdown( "", "| Metric | Context Arm | Baseline | Delta |", "|--------|-----------|--------------|-------|", - f"| Average | {stats['with_docs']['avg']} | {stats['without_docs']['avg']} | **{stats['delta']['avg']:+.1f}** |", - f"| Min | {stats['with_docs']['min']} | {stats['without_docs']['min']} | {stats['delta']['min']:+.1f} |", - f"| Max | {stats['with_docs']['max']} | {stats['without_docs']['max']} | {stats['delta']['max']:+.1f} |", + f"| Average | {_shown(stats['with_docs']['avg'])} | {_shown(stats['without_docs']['avg'])} " + f"| **{_shown(stats['delta']['avg'], signed=True)}** |", + f"| Min | {_shown(stats['with_docs']['min'])} | {_shown(stats['without_docs']['min'])} " + f"| {_shown(stats['delta']['min'], signed=True)} |", + f"| Max | {_shown(stats['with_docs']['max'])} | {_shown(stats['without_docs']['max'])} " + f"| {_shown(stats['delta']['max'], signed=True)} |", "", ] lines += _unpaired_note(stats) @@ -409,8 +410,8 @@ def _format_markdown( for cluster in clusters: lines.append( f"| {cluster['topic']} | {_count_cell(cluster)} | " - f"{cluster['with_avg']:.1f} | {cluster['without_avg']:.1f} | " - f"**{cluster['delta_avg']:+.1f}** |" + f"{_shown(cluster['with_avg'])} | {_shown(cluster['without_avg'])} | " + f"**{_shown(cluster['delta_avg'], signed=True)}** |" ) # ── Source Breakdown ───────────────────────────────────────────── @@ -427,8 +428,8 @@ def _format_markdown( for src in sources_summary: lines.append( f"| {src['source_type']} | {_count_cell(src)} | " - f"{src['with_avg']:.1f} | {src['without_avg']:.1f} | " - f"**{src['delta_avg']:+.1f}** |" + f"{_shown(src['with_avg'])} | {_shown(src['without_avg'])} | " + f"**{_shown(src['delta_avg'], signed=True)}** |" ) # ── Failure Analysis section ───────────────────────────────────── @@ -485,24 +486,12 @@ def _format_markdown( ]) # Add recommendations based on data - if stats["delta"]["avg"] < 0: - lines.append("⚠️ **Overall:** Documentation is hurting answer quality. Consider:") - lines.append("- Improving retrieval relevance") - lines.append("- Adding more specific examples to docs") - lines.append("- Removing outdated/misleading content") - elif stats["delta"]["avg"] > 2: - lines.append("✅ **Overall:** Documentation is significantly helping. Continue:") - lines.append("- Maintaining doc quality") - lines.append("- Expanding coverage for low-performing topics") - else: - lines.append("⚪ **Overall:** Documentation has minimal impact. Consider:") - lines.append("- Improving doc discoverability") - lines.append("- Adding more practical examples") + lines.extend(_overall_advice(stats)) lines.append("") # Identify worst clusters - worst_clusters = [c for c in clusters if c["delta_avg"] < -1] + worst_clusters = [c for c in clusters if c["delta_avg"] is not None and c["delta_avg"] < -1] if worst_clusters: lines.append("### Topics Needing Doc Improvements:") lines.append("") @@ -512,15 +501,47 @@ def _format_markdown( return "\n".join(lines) +def _stat_block(s: Any, mean: float, low: float, high: float) -> Dict[str, Optional[float]]: + """avg, min and max of one statistic, rounded; all None when no question is a complete pair.""" + if not s.paired: + return {"avg": None, "min": None, "max": None} # no complete pair: no value, not a zero + return {"avg": round(mean, 1), "min": round(low, 1), "max": round(high, 1)} + + +def _shown(value: Optional[float], signed: bool = False) -> str: + """A statistic for the page: an em dash where no complete pair gives it a value.""" + if value is None: + return "—" + return f"{value:+.1f}" if signed else f"{value:.1f}" + + +def _overall_advice(stats: Dict[str, Any]) -> List[str]: + """The Overall recommendation lines; no conclusion is drawn without a complete pair.""" + if not stats["paired_questions"]: + return ["⚪ **Overall:** no complete pairs, so no conclusion can be drawn."] + if stats["delta"]["avg"] < 0: + return ["⚠️ **Overall:** Documentation is hurting answer quality. Consider:", + "- Improving retrieval relevance", + "- Adding more specific examples to docs", + "- Removing outdated/misleading content"] + if stats["delta"]["avg"] > 2: + return ["✅ **Overall:** Documentation is significantly helping. Continue:", + "- Maintaining doc quality", + "- Expanding coverage for low-performing topics"] + return ["⚪ **Overall:** Documentation has minimal impact. Consider:", + "- Improving doc discoverability", + "- Adding more practical examples"] + + def _arm_means(evaluations: Sequence[Record]) -> Dict[str, Any]: - """The arms, delta and unpaired count of one group, rounded for display.""" + """The arms, delta and unpaired count of one group, rounded for display; None without a complete pair.""" s = paired_summary(evaluations) return { "count": s.total, "unpaired": s.unpaired, - "with_avg": round(s.with_mean, 1), - "without_avg": round(s.without_mean, 1), - "delta_avg": round(s.delta_mean, 1), + "with_avg": None if not s.paired else round(s.with_mean, 1), + "without_avg": None if not s.paired else round(s.without_mean, 1), + "delta_avg": None if not s.paired else round(s.delta_mean, 1), } @@ -538,6 +559,9 @@ def _unpaired_note(stats: Dict[str, Any]) -> List[str]: """A line saying what the overall means leave out; nothing when every question is paired.""" if not stats["unpaired_questions"]: return [] + if not stats["paired_questions"]: + return [f"_No complete pairs: {stats['unpaired_questions']} unpaired questions, " + "so no arm mean or delta is given._", ""] return [ f"_Averages, min/max, delta and the top/bottom tables are over the {stats['paired_questions']} complete pairs; " f"{stats['unpaired_questions']} unpaired (scored in one arm only) left out._", diff --git a/dokimi/dokimi/schemas/answers.v1.json b/dokimi/dokimi/schemas/answers.v1.json index 48a6f34d8..fd9abc965 100644 --- a/dokimi/dokimi/schemas/answers.v1.json +++ b/dokimi/dokimi/schemas/answers.v1.json @@ -25,6 +25,10 @@ "null" ] }, + "method_version": { + "type": "integer", + "description": "Answering method version (defaults.ANSWERING_METHOD_VERSION); absent on artifacts from before it was stamped." + }, "total_questions": { "type": "integer" }, diff --git a/dokimi/dokimi/schemas/eval.v1.json b/dokimi/dokimi/schemas/eval.v1.json index 61ae91b8c..b3b609d79 100644 --- a/dokimi/dokimi/schemas/eval.v1.json +++ b/dokimi/dokimi/schemas/eval.v1.json @@ -85,6 +85,14 @@ } } }, + "method_version": { + "type": "integer", + "description": "Answering method version (defaults.ANSWERING_METHOD_VERSION); absent on artifacts from before it was stamped." + }, + "judge_placeholder": { + "type": "boolean", + "description": "True when the judge also scored a context answer that is the no-retrieval placeholder (--judge-placeholder), which the default leaves unjudged; absent on artifacts from before it was stamped. Two eval files with the same method_version but a different value here are not comparable." + }, "judge_temperature": { "description": "Sampling temperature the judge ran at; null when left to the provider default." }, diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index d751c4643..224f16802 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -65,7 +65,7 @@ "dokimi/records.py::record": {"complexity": 14, "rationale": "pre-existing when the gate was introduced"}, "dokimi/report/arms_report.py::render_arms_report": {"complexity": 25}, "dokimi/report/eval_report.py::generate_report": {"complexity": 18}, - "dokimi/report/generator.py::ReportGenerator._format_markdown": {"complexity": 18}, + "dokimi/report/generator.py::ReportGenerator._format_markdown": {"complexity": 16}, "dokimi/report/model_compare.py::check_run_consistency": {"complexity": 13}, "dokimi/report/model_compare.py::extract_scores": {"complexity": 16}, "dokimi/report/subject_report.py::render_subject_scorecard": {"complexity": 12}, diff --git a/dokimi/tests/answering_stack_parity.py b/dokimi/tests/answering_stack_parity.py index 148596523..da7af8793 100644 --- a/dokimi/tests/answering_stack_parity.py +++ b/dokimi/tests/answering_stack_parity.py @@ -259,6 +259,8 @@ def run_stack_p( max_tokens: int = 4000, context7_id: Optional[str] = LIBRARY_ID, questions: Optional[List[Dict[str, Any]]] = None, + registry: Any = None, + judge_placeholder: bool = False, ) -> Observed: """Stack P end to end on the fixture: answer, then judge, through the real pipeline code.""" from dokimi.orchestrator import pipeline as pipeline_mod @@ -281,6 +283,8 @@ def run_stack_p( max_tokens_per_question=max_tokens, context7_id=context7_id, product_key=PRODUCT_KEY, + registry=registry, + judge_placeholder=judge_placeholder, ) qs = QUESTIONS if questions is None else questions mark = len(provider_calls.calls) diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..4d7446630 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -87,6 +87,7 @@ --judge-configs --judge-model --judge-panel +--judge-placeholder # placeholder judge (#274) adds it: no registry concept yet; a F1b migration names it --judge-provider --judge-temperature --judges diff --git a/dokimi/tests/test_answering_stack_p_step2.py b/dokimi/tests/test_answering_stack_p_step2.py new file mode 100644 index 000000000..85b60a8d9 --- /dev/null +++ b/dokimi/tests/test_answering_stack_p_step2.py @@ -0,0 +1,105 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Wiring checks for the Stack P fixes of #228 step 2. + +The parity table in ``test_answering_stack_parity.py`` pins what each fix changes +in numbers. These tests cover the parts that table cannot see: the value a flag +parses to, and where it lands. +""" + +import pytest + +from tests.test_answering_stack_parity import _subparser + + +class _FakePipeline: + """Stands in for EvaluationPipeline: keeps the kwargs ``benchmark run`` builds it with.""" + + built: list = [] + + def __init__(self, **kwargs): + _FakePipeline.built.append(kwargs) + + def run(self, concurrency): + return {"steps": {}} + + +@pytest.mark.parametrize("extra, runs", [([], 1), (["--multi-run", "2"], 2)], ids=["single-run", "multi-run"]) +def test_benchmark_run_hands_the_judge_flags_to_the_pipeline(monkeypatch, tmp_path, extra, runs): + """The real handler runs; only the pipeline is faked. Both flags reach it in every mode.""" + from dokimi.commands import benchmark + from dokimi import orchestrator + + _FakePipeline.built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", _FakePipeline) + args = _subparser("benchmark", "run").parse_args( + ["--library", "onetbb", "--output-dir", str(tmp_path), "--judge-temperature", "0.3", + "--judge-placeholder", *extra] + ) + + benchmark.cmd_benchmark_run(args) + + assert len(_FakePipeline.built) == runs + for kwargs in _FakePipeline.built: + assert kwargs["judge_temperature"] == 0.3 + assert kwargs["judge_placeholder"] is True + + +def test_benchmark_run_leaves_the_judge_flags_off_when_not_given(monkeypatch, tmp_path): + from dokimi.commands import benchmark + from dokimi import orchestrator + + _FakePipeline.built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", _FakePipeline) + args = _subparser("benchmark", "run").parse_args(["--library", "onetbb", "--output-dir", str(tmp_path)]) + + benchmark.cmd_benchmark_run(args) + + (kwargs,) = _FakePipeline.built + assert kwargs["judge_temperature"] is None + assert kwargs["judge_placeholder"] is False + + +def test_benchmark_run_judge_temperature_defaults_to_provider_default(): + args = _subparser("benchmark", "run").parse_args(["--library", "onetbb"]) + assert args.judge_temperature is None + + +def test_judge_placeholder_flag_is_off_by_default_and_parses_when_given(): + run = _subparser("benchmark", "run") + assert run.parse_args(["--library", "onetbb"]).judge_placeholder is False + assert run.parse_args(["--library", "onetbb", "--judge-placeholder"]).judge_placeholder is True + + +def test_judge_placeholder_flag_judges_the_placeholder_answers(monkeypatch, tmp_path, provider_calls): + from tests.answering_stack_parity import run_stack_p + + # q-empty and q-rerank carry the placeholder as their with-docs answer: with the flag they + # are judged again, so P's judge calls go back to the 12 that Stack A makes. + p = run_stack_p(monkeypatch, tmp_path, provider_calls, judge_placeholder=True) + assert len(p.judge_inputs) == 12 + assert len([e for e in p.evaluations if e["delta"] is not None]) == 6 + + +def test_eval_output_is_stamped_with_the_method_version(monkeypatch): + from dokimi.defaults import ANSWERING_METHOD_VERSION + from dokimi.eval.judge import Judge + + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + out = Judge(model="gpt-x", provider="openai")._build_evaluation_output([]) + assert out["method_version"] == ANSWERING_METHOD_VERSION == 2 + + +def test_eval_output_says_whether_the_placeholder_was_judged(monkeypatch): + """The two modes give different numbers on the same answers under one method_version.""" + from tests.artifact_fixtures import artifact_fixture + from dokimi.eval.judge import Judge + + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + default = Judge(model="gpt-x", provider="openai")._build_evaluation_output([]) + opted_in = Judge(model="gpt-x", provider="openai", judge_placeholder=True)._build_evaluation_output([]) + + assert default["judge_placeholder"] is False + assert opted_in["judge_placeholder"] is True + artifact_fixture("eval", opted_in) # eval.v1 declares the key diff --git a/dokimi/tests/test_answering_stack_parity.py b/dokimi/tests/test_answering_stack_parity.py index 7ad72a58e..df6934de5 100644 --- a/dokimi/tests/test_answering_stack_parity.py +++ b/dokimi/tests/test_answering_stack_parity.py @@ -19,7 +19,7 @@ import argparse from pathlib import Path -from typing import Any, Dict, Tuple +from typing import Any, Dict, List, Optional, Tuple import pytest @@ -55,15 +55,14 @@ PINNED: Tuple[Tuple[str, str, Any, Any], ...] = ( # shared: the two stacks agree today ("shared", "retrieval.count", 6, 6), - ("shared", "judge.calls", 12, 12), ("shared", "reranker.calls", 5, 5), ("shared", "answer.roles", ("user",), ("user",)), ("shared", "answer.system", None, None), ("shared", "answer.cache_blocks", False, False), # D1 retrieval request - ("D1", "retrieval.passed", ("max_tokens",), ("max_results",)), - # what each side asked for, not what came back: both get 4 docs; P keeps 3, A keeps 3 - ("D1", "retrieval.max_results", 5, 3), + ("D1", "retrieval.passed", ("max_results", "max_tokens"), ("max_results",)), + # what each side asked for, not what came back: both get 4 docs; both ask for top_k=3 + ("D1", "retrieval.max_results", 3, 3), ("D1", "retrieval.max_tokens", 4000, 8000), # D2 no docs found ("D2", "empty.answer_kinds", ("plain",), ("plain", "plain")), @@ -102,31 +101,35 @@ "(No documentation provided)", "(No documentation retrieved)", ), - # D6 judge temperature (CLI: P = benchmark run, A = arms run; benchmark run has no such flag) - ("D6", "cli.judge_temperature_flag", False, True), + # D6 judge temperature (CLI: P = benchmark run, A = arms run) + ("D6", "cli.judge_temperature_flag", True, True), # D7 --max-tokens (CLI defaults; where the configured value lands) ("D7", "cli.max_tokens_default", 4000, None), ("D7", "max_tokens.lands_in", "retrieval", "answer"), # D8 library id: EvaluationPipeline with no context7_id (the `orchestrate` path without # --context7-id). `benchmark run` passes the registry's context7_id, so it is not this row. - ("D8", "library_id", "client/resolved", "registry/lib"), - # D9 statistics. The delta also moves with D3: P's q-rerank is judged with no docs - # (delta 0), A's falls back to raw docs (delta 20), so a D3 fix flips this row too. - ("D9", "stats.delta_mean", 13.33, 16.67), - ("D9", "significance.n", 6, 6), + ("D8", "library_id", "registry/lib", "registry/lib"), + # D2 judge calls: P no longer judges the placeholder with-docs answers of q-empty and + # q-rerank (D2, D3), so 2 of the 12 calls are gone. A still judges all 12. + ("D2", "judge.calls", 10, 12), + # D9 statistics. P's paired deltas are q-docs, q-docs-2, q-docs-3, q-gt: 4 pairs, below + # MIN_PAIRS_FOR_TEST (6), so P has no significance test. A has 6 pairs. + ("D9", "stats.delta_mean", 20.0, 16.67), + ("D9", "stats.pairs", 4, 6), + ("D9", "significance.n", None, 6), ("D9", "significance.correction", None, "holm"), - ("D9", "significance.significant", False, False), + ("D9", "significance.significant", None, False), # what the judge sees per case: no docs (D2), reranker raises (D3), ground-truth chunk (D5) ( "D2", "judge.contexts.q-empty", - ("(No documentation provided)", "(No documentation retrieved)"), + ("(No documentation provided)",), ("(No documentation retrieved)",), ), ( "D3", "judge.contexts.q-rerank", - ("(No documentation provided)", "(No documentation retrieved)"), + ("(No documentation provided)",), ("(No documentation retrieved)", ("docs", "Passage", 3, True)), ), ( @@ -138,8 +141,8 @@ # what the judge is handed as ground truth: the q-gt chunk on both stacks, None on the rest ("shared", "judge.ground_truth.q-gt", (GROUND_TRUTH_CHUNK,), (GROUND_TRUTH_CHUNK,)), ("shared", "judge.ground_truth.others", (None,), (None,)), - # the per-question deltas that feed D9's statistics (P: stored "delta"; A: "docs" vs "baseline"). - # q-rerank is the one that moves with D3: P judges it with no docs (0), A with the raw docs (20). + # Per-question deltas feeding D9's statistics. q-empty and q-rerank have no P delta (no with-docs judgement, D2/D3); + # A's q-empty is 0 and its q-rerank falls back to the raw docs (20). ( "D9", "delta.per_question", @@ -147,9 +150,9 @@ ("q-docs", 20.0), ("q-docs-2", 20.0), ("q-docs-3", 20.0), - ("q-empty", 0.0), + ("q-empty", None), ("q-gt", 20.0), - ("q-rerank", 0.0), + ("q-rerank", None), ), ( ("q-docs", 20.0), @@ -164,6 +167,8 @@ ("only", "record.shape", "with_docs/without_docs", "arms"), ("only", "answer.token_usage.context_chars", True, False), ("only", "artifact.cost_summary", False, True), + # #228 step 2 stamp: the answers artifact carries method_version; the arms artifact does not + ("only", "artifact.method_version", 2, None), ) @@ -253,15 +258,6 @@ def _where_4000_lands(obs: Observed) -> str: def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]: - with pytest.MonkeyPatch.context() as mp: - p = run_stack_p(mp, tmp_path / "p", provider_calls) - with pytest.MonkeyPatch.context() as mp: - a = run_stack_a(mp, tmp_path / "a", provider_calls) - with pytest.MonkeyPatch.context() as mp: - p_noid = run_stack_p( - mp, tmp_path / "p-noid", provider_calls, context7_id=None, questions=QUESTIONS[:1] - ) - from dokimi.eval.arms_run import resolve_doc_library_id from dokimi.registry import ProductRegistry from dokimi.treatments.arms import DocTreatment @@ -272,6 +268,22 @@ def _observe(tmp_path: Path, provider_calls) -> Dict[str, Tuple[Any, Any]]: f" description: fixture\n context7_id: {REGISTRY_ID}\n", encoding="utf-8", ) + registry = ProductRegistry(registry_path) + + with pytest.MonkeyPatch.context() as mp: + p = run_stack_p(mp, tmp_path / "p", provider_calls) + with pytest.MonkeyPatch.context() as mp: + a = run_stack_a(mp, tmp_path / "a", provider_calls) + with pytest.MonkeyPatch.context() as mp: + p_noid = run_stack_p( + mp, + tmp_path / "p-noid", + provider_calls, + context7_id=None, + questions=QUESTIONS[:1], + registry=registry, + ) + a_lib = resolve_doc_library_id( [DocTreatment(a.client, name="docs", top_k=3, reranker=a.reranker)], PRODUCT, @@ -303,6 +315,7 @@ def sampling(pairs, field): sig_p = _significance_p(p) sig_a = a.artifact["summary"]["per_arm"]["docs"]["significance"] + paired_p = _paired_deltas(p) return { "retrieval.count": (len(p.client.calls), len(a.client.calls)), @@ -311,7 +324,8 @@ def sampling(pairs, field): "answer.roles": (sampling(p_all, "roles"), sampling(a_all, "roles")), "answer.system": (sampling(p_all, "system"), sampling(a_all, "system")), "answer.cache_blocks": (sampling(p_all, "cache_blocks"), sampling(a_all, "cache_blocks")), - "stats.delta_mean": (sig_p["delta_mean"], sig_a["delta_mean"]), + "stats.delta_mean": (round(sum(paired_p) / len(paired_p), 2), sig_a["delta_mean"]), + "stats.pairs": (len(paired_p), sig_a["n"]), "retrieval.passed": ( distinct([c["passed"] for c in p.client.calls]), distinct([c["passed"] for c in a.client.calls]), @@ -365,15 +379,23 @@ def sampling(pairs, field): ), "max_tokens.lands_in": (_where_4000_lands(p), _where_4000_lands(a)), "library_id": (p_noid.client.calls[0]["library_id"], a_lib), - "significance.n": (sig_p["n"], sig_a["n"]), - "significance.correction": (sig_p.get("correction"), sig_a.get("correction")), - "significance.significant": (sig_p["significant"], sig_a["significant"]), + # None: the paired test does not run below MIN_PAIRS_FOR_TEST pairs (P has 4 here) + "significance.n": (sig_p["n"] if sig_p else None, sig_a["n"]), + "significance.correction": ( + sig_p.get("correction") if sig_p else None, + sig_a.get("correction"), + ), + "significance.significant": ( + sig_p["significant"] if sig_p else None, + sig_a["significant"], + ), "record.shape": (_record_shape(p_rec_empty), _record_shape(a_rec_empty)), "answer.token_usage.context_chars": ( "context_chars" in docs_block(p, p_rec_docs)["token_usage"], "context_chars" in docs_block(a, a_rec_docs)["token_usage"], ), "artifact.cost_summary": ("cost_summary" in p.artifact, "cost_summary" in a.artifact), + "artifact.method_version": (p.artifact.get("method_version"), a.artifact.get("method_version")), "judge.contexts.q-empty": (_context_kinds(p, EMPTY_QUERY), _context_kinds(a, EMPTY_QUERY)), "judge.contexts.q-rerank": ( _context_kinds(p, RERANK_QUERY), @@ -403,12 +425,16 @@ def sampling(pairs, field): } -def _significance_p(obs: Observed) -> Dict[str, Any]: +def _significance_p(obs: Observed) -> Optional[Dict[str, Any]]: + """Stack P's paired test; None when it does not run (fewer pairs than MIN_PAIRS_FOR_TEST).""" from dokimi.report.eval_report import significance_test - result = significance_test(obs.evaluations) - assert result is not None, "P paired test did not run on the fixture" - return result + return significance_test(obs.evaluations) + + +def _paired_deltas(obs: Observed) -> List[float]: + """The deltas of the questions both arms were judged on.""" + return [e["delta"] for e in obs.evaluations if e.get("delta") is not None] def _record_shape(rec: Dict[str, Any]) -> str: diff --git a/dokimi/tests/test_answers_command.py b/dokimi/tests/test_answers_command.py index 1eb720361..e806e470f 100644 --- a/dokimi/tests/test_answers_command.py +++ b/dokimi/tests/test_answers_command.py @@ -308,3 +308,28 @@ def test_generate_unreadable_questions_exits_2_naming_the_path(tmp_path, monkeyp answers.cmd_answers_generate(_args(questions=str(path))) assert exc.value.code == 2 assert f"Error: cannot read {path}" in capsys.readouterr().err + + +@pytest.mark.parametrize("value", ["0", "-1", '"3"', "true"]) +def test_a_config_top_k_that_is_not_a_positive_int_exits_2(tmp_path, monkeypatch, capsys, value): + # top_k reaches the doc source as max_results: 0 would make every answer a placeholder. + path = tmp_path / "products.yaml" + path.write_text(f"retrieval:\n top_k: {value}\n", encoding="utf-8") + _point_config(monkeypatch, path) + monkeypatch.setattr(dokimi.questions.artifact, "load_questions", + lambda path: pytest.fail("questions loaded after a bad top_k")) + with pytest.raises(SystemExit) as exc: + answers.cmd_answers_generate(_args()) + assert exc.value.code == 2 + assert "retrieval.top_k in the config must be a positive integer" in capsys.readouterr().err + + +@pytest.mark.parametrize("value", ["0", "-1"]) +def test_top_k_flag_below_one_is_refused(capsys, value): + from dokimi.cli import build_parser + + with pytest.raises(SystemExit) as exc: + build_parser().parse_args(["answers", "generate", "--product", "L", + "--questions", "q.json", "--top-k", value]) + assert exc.value.code == 2 + assert "--top-k" in capsys.readouterr().err diff --git a/dokimi/tests/test_benchmark_registry_wiring.py b/dokimi/tests/test_benchmark_registry_wiring.py new file mode 100644 index 000000000..e7fef9f52 --- /dev/null +++ b/dokimi/tests/test_benchmark_registry_wiring.py @@ -0,0 +1,84 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``benchmark run --registry`` resolves the library id through that registry.""" + +import pytest + +from dokimi import orchestrator +from dokimi.commands import benchmark +from dokimi.orchestrator.pipeline import EvaluationPipeline +from tests.test_answering_stack_parity import _subparser + +CUSTOM = """\ +products: + my_lib: + name: My Lib + description: A library only this registry knows + doc_sources: [context7] + onetbb: + name: Custom TBB + description: A key that the bundled registry also has, with no id here + doc_sources: [context7] +""" + + +class _DeclaredIdClient: + """A doc source like Context7: a name that is not a declared id is not resolved.""" + + requires_declared_library_id = True + + def resolve_library_id(self, name): + return name + + +class _Spy: + """Stands in for EvaluationPipeline: keeps what ``benchmark run`` hands it.""" + + built: list = [] + + def __init__(self, **kwargs): + self.kwargs = kwargs + _Spy.built.append(kwargs) + + def run(self, concurrency): + return {"steps": {}} + + +@pytest.fixture +def spy(monkeypatch): + _Spy.built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", _Spy) + return _Spy + + +def _run(tmp_path, library, *extra): + registry = tmp_path / "my.yaml" + registry.write_text(CUSTOM, encoding="utf-8") + args = _subparser("benchmark", "run").parse_args( + ["--library", library, "--registry", str(registry), "--output-dir", str(tmp_path / "out"), *extra] + ) + benchmark.cmd_benchmark_run(args) + + +@pytest.mark.parametrize("library", ["my_lib", "onetbb"]) +def test_a_custom_entry_without_an_id_is_not_looked_up_in_the_bundled_registry(spy, tmp_path, library): + _run(tmp_path, library) + + (kwargs,) = spy.built + real = EvaluationPipeline(**kwargs) + with pytest.raises(ValueError, match="declares no 'context7_id'"): + real._library_id(_DeclaredIdClient()) + + +def test_multi_run_and_batch_hand_the_registry_over_too(spy, tmp_path): + _run(tmp_path, "my_lib", "--multi-run", "2") + registry = tmp_path / "my.yaml" + args = _subparser("benchmark", "batch").parse_args( + ["--libraries", "my_lib", "--registry", str(registry), "--output-dir", str(tmp_path / "batch")] + ) + benchmark.cmd_benchmark_batch(args) + + assert len(spy.built) == 3 + for kwargs in spy.built: + assert "my_lib" in kwargs["registry"] diff --git a/dokimi/tests/test_dashboard.py b/dokimi/tests/test_dashboard.py index 1aa3982f8..29cb235aa 100644 --- a/dokimi/tests/test_dashboard.py +++ b/dokimi/tests/test_dashboard.py @@ -5,6 +5,7 @@ import json from pathlib import Path +import pytest from dokimi.dashboard.aggregator import ( DashboardData, @@ -241,3 +242,40 @@ def test_aggregator_ignores_the_judge_cap_marker(tmp_path): _make_eval_file(tmp_path, "oneTBB", [ev]) q = ResultsAggregator(tmp_path).aggregate().products[0].questions[0] assert q.with_docs_dimensions == {"correctness": 40, "completeness": 40} + + +def _miss(q_id, without_score): + """A question the context arm never answered from the docs: no retrieval hit, no context score.""" + return {"question_id": q_id, "question": "Q?", "retrieval_hit": False, "with_docs": None, + "without_docs": {"aggregate": without_score, "correctness": without_score}, "delta": None} + + +def test_a_retrieval_miss_with_a_baseline_score_counts_as_context_zero_in_the_doc_score_only(tmp_path): + _make_eval_file(tmp_path, "oneTBB", [_ev("q1", "Q?", 80, 60), _miss("q2", 60)]) + p = ResultsAggregator(tmp_path).aggregate().products[0] + # The miss is no judged measurement: the arms and the delta are over the one complete pair. + assert (p.avg_with_docs, p.avg_without_docs, p.avg_delta) == (80.0, 60.0, 20.0) + assert p.by_source["persona"]["avg_delta"] == 20.0 + # The doc health number, which the dashboard shows, still counts the miss as 0. + assert (p.doc_health, p.doc_score) == (40.0, 40.0) + assert p.status == "poor" + + +def test_an_all_miss_run_scores_its_misses_as_zero_with_no_delta(tmp_path): + _make_eval_file(tmp_path, "oneTBB", [_miss("q1", 60), _miss("q2", 60)]) + p = ResultsAggregator(tmp_path).aggregate().products[0] + assert (p.avg_with_docs, p.avg_without_docs, p.avg_delta) == (None, None, None) + assert (p.doc_health, p.doc_score, p.status) == (0.0, 0.0, "poor") + + +@pytest.mark.parametrize("miss", [ + {"question_id": "q2", "question": "Q?", "retrieval_hit": False, "with_docs": None, "without_docs": None, + "delta": None}, + {**_miss("q2", 60), "error": "judge failed"}, + {**_miss("q2", 60), "with_docs": {"error": "judge failed"}}, +]) +def test_a_miss_without_a_clean_baseline_score_leaves_the_doc_score_alone(tmp_path, miss): + # Only a retrieval miss that the baseline answered and no error touched counts as a context zero. + _make_eval_file(tmp_path, "oneTBB", [_ev("q1", "Q?", 80, 60), miss]) + p = ResultsAggregator(tmp_path).aggregate().products[0] + assert (p.doc_health, p.doc_score) == (80.0, 80.0) diff --git a/dokimi/tests/test_diagnoser.py b/dokimi/tests/test_diagnoser.py index 5fabb7db2..34704fa65 100644 --- a/dokimi/tests/test_diagnoser.py +++ b/dokimi/tests/test_diagnoser.py @@ -3,6 +3,8 @@ """Tests for failure diagnoser.""" +from unittest.mock import MagicMock + import pytest from dokimi import records from dokimi.records import CONTEXT @@ -16,15 +18,18 @@ KNOWLEDGE_SUFFICIENT, INSUFFICIENT_DATA, ) +from dokimi.eval.judge import Judge -def _make_answer(doc_source="context7", retrieved_docs=None): +def _make_answer(doc_source="context7", retrieved_docs=None, error=None): """The context-arm answer of an answers.v1 record, as the judge hands it to diagnose().""" + with_docs = {"answer": "an answer", "doc_source": doc_source, "retrieved_docs": retrieved_docs or []} + if error: + with_docs["error"] = error data = artifact_fixture("answers", {"answers": [{ "question_id": "q1", "question_text": "How?", - "with_docs": {"answer": "an answer", "doc_source": doc_source, - "retrieved_docs": retrieved_docs or []}, + "with_docs": with_docs, "without_docs": {"answer": "some answer"}, }]}) return records.read(data).records[0].answer(CONTEXT) @@ -53,6 +58,57 @@ def test_empty_retrieval_no_docs(): assert result["label"] == EMPTY_RETRIEVAL +def test_unjudged_retrieval_miss_is_an_empty_retrieval_not_insufficient_data(): + """The judge leaves the no-document placeholder unjudged, so its delta is None.""" + answer = _make_answer(doc_source="fallback_none", retrieved_docs=[]) + result = diagnose(answer, None) + assert result["label"] == EMPTY_RETRIEVAL + assert result["evidence"]["doc_source"] == "fallback_none" + + +def test_unjudged_none_source_is_an_empty_retrieval(): + answer = _make_answer(doc_source="none", retrieved_docs=[]) + result = diagnose(answer, None) + assert result["label"] == EMPTY_RETRIEVAL + assert result["evidence"]["doc_source"] == "none" + + +def test_judged_placeholder_with_no_baseline_is_insufficient_data(): + """With judge_placeholder the context is scored; a missing baseline is the reason there is no delta.""" + answer = _make_answer(doc_source="fallback_none", retrieved_docs=[]) + result = diagnose(answer, None, context_scored=True) + assert result["label"] == INSUFFICIENT_DATA + + +def test_judge_diagnoses_a_judged_placeholder_without_a_baseline_as_insufficient_data(): + """Through Judge: the context arm was scored, so the missing baseline is why there is no delta.""" + judge = Judge.__new__(Judge) + judge._accumulate_usage = lambda usage: None + judge.temperature = judge.seed = None + judge.judge_placeholder = True + judge.llm = MagicMock() + judge.llm.invoke.return_value = MagicMock(content=( + '{"correctness": 60, "completeness": 60, "specificity": 60, ' + '"code_quality": 60, "actionability": 60, "aggregate": 0}' + )) + row = judge._evaluate_answer_pair("L", { + "question_id": "q1", + "question_text": "How?", + "with_docs": {"answer": "[FALLBACK: No relevant documentation found]", + "doc_source": "fallback_none", "retrieved_docs": []}, + }) + assert row["with_docs"] is not None + assert row["without_docs"] is None + assert row["diagnosis"]["label"] == INSUFFICIENT_DATA + + +def test_errored_placeholder_answer_stays_insufficient_data(): + """An errored answer is missing, not a retrieval miss, whatever its doc source.""" + answer = _make_answer(doc_source="fallback_none", retrieved_docs=[], error="timeout") + result = diagnose(answer, None) + assert result["label"] == INSUFFICIENT_DATA + + def test_empty_retrieval_no_mcp_client(): answer = _make_answer(doc_source="none", retrieved_docs=[]) result = diagnose(answer, -2.0) diff --git a/dokimi/tests/test_eval_answerer.py b/dokimi/tests/test_eval_answerer.py index 7a9b62568..5c9e2824a 100644 --- a/dokimi/tests/test_eval_answerer.py +++ b/dokimi/tests/test_eval_answerer.py @@ -18,6 +18,7 @@ sys.modules[mod_name] = mock_mod from dokimi.eval.answerer import Answerer, ANSWER_PROMPT_WITH_DOCS, ANSWER_PROMPT_WITHOUT_DOCS +from dokimi.defaults import DEFAULT_RETRIEVAL_TOP_K # Canonical return value for llm_call_with_usage mock _LLM_MOCK_RETURN = ("This is a test answer from LLM.", { @@ -74,29 +75,17 @@ def mock_llm_call(): class TestAnswererInit: - def test_init_openai(self): - mock_cls = Mock() - with patch('dokimi.eval.answerer.ChatOpenAI', mock_cls, create=True), \ - patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): - answerer = Answerer(model="gpt-4o", provider="openai") - assert answerer.model == "gpt-4o" - assert answerer.provider == "openai" - mock_cls.assert_called_once() - - def test_init_anthropic(self): - mock_cls = Mock() - with patch('dokimi.eval.answerer.ChatAnthropic', mock_cls, create=True), \ - patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): - answerer = Answerer(model="claude-opus", provider="anthropic", api_key="test") - # The anthropic client is the one built, with the caller's model and key. - mock_cls.assert_called_once_with(model="claude-opus", api_key="test") - assert answerer.llm is mock_cls.return_value - assert answerer.provider == "anthropic" + @pytest.mark.parametrize( + "provider", + ["openai", "anthropic", "google", "gemini", "google-vertex", "openrouter", + "amazon-bedrock", "vertex_ai"], + ) + def test_init_accepts_supported_provider(self, provider): # test-quality: ok must-not-raise smoke test of the provider allow-list + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + Answerer(model="m", provider=provider) def test_init_unsupported_provider(self): - mock_cls = Mock() - with patch('dokimi.eval.answerer.ChatOpenAI', mock_cls, create=True), \ - patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): with pytest.raises(ValueError) as exc: Answerer(provider="unsupported") assert "Unsupported provider" in str(exc.value) @@ -191,18 +180,40 @@ def test_token_usage_summary_in_output(self, sample_questions, mock_mcp_client, class TestRetrieveDocs: + def test_asks_the_doc_source_for_top_k_docs(self, mock_mcp_client): + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + answerer = Answerer(mcp_client=mock_mcp_client, top_k=7) + answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000) + + mock_mcp_client.get_library_docs.assert_called_once_with( + library_id="uxlfoundation/oneTBB", query="parallel_for", + max_results=7, max_tokens=4000, + ) + def test_calls_mcp_client(self, mock_mcp_client): with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): answerer = Answerer(mcp_client=mock_mcp_client) docs = answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000) mock_mcp_client.get_library_docs.assert_called_once_with( - library_id="uxlfoundation/oneTBB", query="parallel_for", max_tokens=4000 + library_id="uxlfoundation/oneTBB", query="parallel_for", + max_results=DEFAULT_RETRIEVAL_TOP_K, max_tokens=4000, ) assert [d["content"] for d in docs] == [ mock_mcp_client.get_library_docs.return_value[0]["content"] ] + def test_no_top_k_asks_the_doc_source_for_the_default(self, mock_mcp_client): + """`answers generate` without --top-k reaches here as top_k=None: the doc source gets the default, not None.""" + with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): + answerer = Answerer(mcp_client=mock_mcp_client, top_k=None) + answerer._retrieve_docs("uxlfoundation/oneTBB", "parallel_for", 4000) + + mock_mcp_client.get_library_docs.assert_called_once_with( + library_id="uxlfoundation/oneTBB", query="parallel_for", + max_results=3, max_tokens=4000, + ) + def test_error_handling(self, mock_mcp_client): mock_mcp_client.get_library_docs.side_effect = Exception("Network error") @@ -228,15 +239,6 @@ def test_with_docs(self, mock_llm_call): assert result["doc_source"] == "context7" assert result["answer"] == "This is a test answer from LLM." - def test_empty_docs(self): - with patch('dokimi.eval.answerer.LANGCHAIN_AVAILABLE', True): - answerer = Answerer() - result = answerer._generate_with_docs("Test question?", []) - - assert result["answer"] == "[No documentation retrieved]" - assert result["retrieved_docs"] == [] - assert "token_usage" in result - class TestGenerateWithoutDocs: def test_generates_answer(self, mock_llm_call): diff --git a/dokimi/tests/test_eval_score_method_version.py b/dokimi/tests/test_eval_score_method_version.py new file mode 100644 index 000000000..9c8c30a7c --- /dev/null +++ b/dokimi/tests/test_eval_score_method_version.py @@ -0,0 +1,123 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``eval score`` stamps the eval artifact with the method version of the answers it judged.""" + +import json +from types import SimpleNamespace + +import pytest + +from dokimi.cli import build_parser + +GOOD = ('{"reasoning":"r","correctness":80,"completeness":80,"specificity":80,' + '"code_quality":80,"actionability":80,"aggregate":80}') + +ANSWERS = [{"question_id": "Q1", "question_text": "q", "question": "q", + "with_docs": {"answer": "an answer", "doc_source": "x", "retrieved_docs": ["d"]}, + "without_docs": {"answer": "without docs"}}] + +JUDGE = ["--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x"] + + +@pytest.fixture +def judge_calls(monkeypatch): + import litellm + + sent = [] + + def completion(**kw): + sent.append(kw) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=GOOD))], + usage=None) + + monkeypatch.setattr(litellm, "completion", completion) + return sent + + +def _eval_score(tmp_path, answers_doc): + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(answers_doc), encoding="utf-8") + out = tmp_path / "eval.json" + args = build_parser().parse_args(["eval", "score", "--product", "L", "--answers", str(answers), + "--output", str(out), "--concurrency", "1", *JUDGE]) + args.func(args) + return json.loads(out.read_text(encoding="utf-8")) + + +def test_unstamped_answers_are_legacy_so_the_eval_says_version_1(tmp_path, judge_calls): + """Answers written before the stamp are the legacy method; re-judging them today does not make them version 2.""" + out = _eval_score(tmp_path, ANSWERS) + + assert len(judge_calls) == 2 # both arms judged: the stamp does not change what is judged + assert out["method_version"] == 1 + + +@pytest.mark.parametrize("version", [1, 2]) +def test_stamped_answers_keep_their_method_version(tmp_path, judge_calls, version): + out = _eval_score(tmp_path, {"method_version": version, "answers": ANSWERS}) + + assert out["method_version"] == version + + +def _pipeline_judge_version(monkeypatch, tmp_path, answers, answers_file=None): + """The method version the pipeline's judge is built with, for these answers.""" + from dokimi.eval.judge import Judge + from dokimi.orchestrator.pipeline import EvaluationPipeline + + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + seen = [] + + def evaluate(self, product, records, concurrency=5): + seen.append(self.answers_method_version) + return [] + + monkeypatch.setattr(Judge, "evaluate_answers", evaluate) + monkeypatch.setattr(Judge, "save_evaluations", lambda self, evaluations, path: None) + p = EvaluationPipeline.__new__(EvaluationPipeline) # the attributes _evaluate_answers reads + p.product, p.model, p.provider = "L", "gpt-x", "openai" + p.judge_model, p.judge_provider, p.judge_temperature, p.judge_placeholder = "gpt-x", "openai", None, False + p.answers_path, p.eval_path = answers_file, tmp_path / "eval.json" + p._evaluate_answers(answers, concurrency=1) + return seen[0] + + +def test_pipeline_judges_a_wrapped_version_1_artifact_as_version_1(monkeypatch, tmp_path): + assert _pipeline_judge_version(monkeypatch, tmp_path, {"method_version": 1, "answers": ANSWERS}) == 1 + + +def test_pipeline_judges_an_unstamped_answers_file_as_legacy_version_1(monkeypatch, tmp_path): + answers_file = tmp_path / "answers.json" + answers_file.write_text(json.dumps({"answers": ANSWERS}), encoding="utf-8") + + assert _pipeline_judge_version(monkeypatch, tmp_path, [], answers_file) == 1 + + +def test_pipeline_judges_fresh_in_memory_answers_as_the_current_version(monkeypatch, tmp_path): + from dokimi.defaults import ANSWERING_METHOD_VERSION + + assert _pipeline_judge_version(monkeypatch, tmp_path, ANSWERS) == ANSWERING_METHOD_VERSION + + +@pytest.mark.parametrize("bad", ["2", True, 3.5, [1], 0, -7]) +def test_a_malformed_stamp_exits_2_before_any_judge_call(tmp_path, judge_calls, capsys, bad): + """main ignored the field; writing it unchecked failed eval validation after paid calls.""" + with pytest.raises(SystemExit) as exc: + _eval_score(tmp_path, {"method_version": bad, "answers": ANSWERS}) + + assert exc.value.code == 2 + assert judge_calls == [] + assert not (tmp_path / "eval.json").exists() + assert "method_version must be a positive integer" in capsys.readouterr().err + + +def test_a_null_stamp_is_legacy(tmp_path, judge_calls): + assert _eval_score(tmp_path, {"method_version": None, "answers": ANSWERS})["method_version"] == 1 + + +def test_pipeline_refuses_an_answers_file_with_a_malformed_stamp(monkeypatch, tmp_path): + answers_file = tmp_path / "answers.json" + answers_file.write_text(json.dumps({"method_version": "2", "answers": ANSWERS}), encoding="utf-8") + + with pytest.raises(ValueError, match="method_version must be a positive integer"): + _pipeline_judge_version(monkeypatch, tmp_path, [], answers_file) diff --git a/dokimi/tests/test_judge_placeholder_entry_points.py b/dokimi/tests/test_judge_placeholder_entry_points.py new file mode 100644 index 000000000..31f69d931 --- /dev/null +++ b/dokimi/tests/test_judge_placeholder_entry_points.py @@ -0,0 +1,96 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""--judge-placeholder reaches the judge of every entry point that takes it.""" + +import json +from types import SimpleNamespace + +import pytest + +from dokimi.cli import build_parser + +GOOD = ('{"reasoning":"r","correctness":80,"completeness":80,"specificity":80,' + '"code_quality":80,"actionability":80,"aggregate":80}') + +#: A question whose retrieval missed: the with-docs answer is the no-retrieval placeholder. +ANSWERS = [{"question_id": "Q1", "question_text": "q", "question": "q", + "with_docs": {"answer": "[FALLBACK: No relevant documentation found]", + "doc_source": "fallback_none", "retrieved_docs": []}, + "without_docs": {"answer": "without docs"}}] + +JUDGE = ["--judge-provider", "amazon-bedrock", "--judge-model", "us.anthropic.x"] + + +@pytest.fixture +def requests(monkeypatch): + import litellm + + sent = [] + + def completion(**kw): + sent.append(kw) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=GOOD))], + usage=None) + + monkeypatch.setattr(litellm, "completion", completion) + return sent + + +def _run(argv): + args = build_parser().parse_args(argv) + args.func(args) + + +def _eval_score(tmp_path, *flag): + answers = tmp_path / "answers.json" + answers.write_text(json.dumps(ANSWERS), encoding="utf-8") + out = tmp_path / "eval.json" + _run(["eval", "score", "--product", "L", "--answers", str(answers), "--output", str(out), + "--concurrency", "1", *JUDGE, *flag]) + return json.loads(out.read_text(encoding="utf-8")) + + +def _evaluate(tmp_path, monkeypatch, *flag): + from dokimi.orchestrator import EvaluationPipeline + + def run(self, **kw): + self._evaluate_answers(ANSWERS) + raise RuntimeError("stop after the judge step") + + monkeypatch.setattr(EvaluationPipeline, "run", run) + with pytest.raises(SystemExit): + _run(["evaluate", "--product", "L", "--description", "d", "--output-dir", str(tmp_path), *JUDGE, *flag]) + return json.loads((tmp_path / "eval" / "L.json").read_text(encoding="utf-8")) + + +def test_eval_score_leaves_the_placeholder_unjudged_by_default(tmp_path, requests): + out = _eval_score(tmp_path) + + assert len(requests) == 1 # the baseline answer only + assert out["evaluations"][0]["with_docs"] is None + assert out["judge_placeholder"] is False + + +def test_eval_score_judge_placeholder_judges_it_and_says_so(tmp_path, requests): + out = _eval_score(tmp_path, "--judge-placeholder") + + assert len(requests) == 2 + assert out["evaluations"][0]["with_docs"]["aggregate"] == 80 + assert out["judge_placeholder"] is True + + +def test_evaluate_leaves_the_placeholder_unjudged_by_default(tmp_path, requests, monkeypatch): + out = _evaluate(tmp_path, monkeypatch) + + assert len(requests) == 1 + assert out["evaluations"][0]["with_docs"] is None + assert out["judge_placeholder"] is False + + +def test_evaluate_judge_placeholder_judges_it_and_says_so(tmp_path, requests, monkeypatch): + out = _evaluate(tmp_path, monkeypatch, "--judge-placeholder") + + assert len(requests) == 2 + assert out["evaluations"][0]["with_docs"]["aggregate"] == 80 + assert out["judge_placeholder"] is True diff --git a/dokimi/tests/test_llm_cli.py b/dokimi/tests/test_llm_cli.py index c70b40ad6..8986ff57f 100644 --- a/dokimi/tests/test_llm_cli.py +++ b/dokimi/tests/test_llm_cli.py @@ -831,6 +831,21 @@ def test_evaluate_with_a_cli_judge_given_a_temperature_exits_2_like_the_other_ju assert "Traceback" not in err +def test_benchmark_run_with_a_cli_judge_given_a_temperature_refuses_before_any_answer(monkeypatch, capsys): + """The refusal comes before the pipeline is built, so no answer is paid for.""" + from dokimi import cli, orchestrator + + built = [] + monkeypatch.setattr(orchestrator, "EvaluationPipeline", lambda **kwargs: built.append(kwargs)) + monkeypatch.setattr("sys.argv", ["dokimi", "benchmark", "run", "--library", "onetbb", + "--judge-provider", "claude-cli", "--judge-temperature", "0"]) + with pytest.raises(SystemExit) as exc: + cli.main() + err = capsys.readouterr().err + assert exc.value.code == 2 and err.startswith("Error: claude-cli takes no temperature") + assert built == [] + + def test_the_main_model_falls_back_to_most_output_when_the_requested_family_is_unknown(): usage = {"helper-a": {"outputTokens": 5}, "helper-b": {"outputTokens": 500}} assert llm_cli._main_model(usage, "claude-fable-5-1") == "helper-b" diff --git a/dokimi/tests/test_pipeline_library_id.py b/dokimi/tests/test_pipeline_library_id.py new file mode 100644 index 000000000..64d07be0e --- /dev/null +++ b/dokimi/tests/test_pipeline_library_id.py @@ -0,0 +1,151 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``EvaluationPipeline._library_id``: which doc-source library a run queries.""" + +import pytest + +from dokimi.orchestrator.pipeline import EvaluationPipeline +from dokimi.registry import ProductRegistry + +REGISTRY = """\ +products: + my_lib: + name: My Lib + description: The registry key differs from the display name + context7_id: org/declared + doc_sources: [context7] +""" + + +class _RecordingClient: + """Records what the pipeline asks it to resolve; a bare name resolves to a marker.""" + + requires_declared_library_id = True + + def __init__(self): + self.asked = [] + + def resolve_library_id(self, name): + self.asked.append(name) + return f"client:{name}" + + +@pytest.fixture +def registry(tmp_path): + path = tmp_path / "products.yaml" + path.write_text(REGISTRY, encoding="utf-8") + return ProductRegistry(path=path) + + +def _pipeline(tmp_path, registry, **kwargs): + return EvaluationPipeline( + product="My Lib", description="d", product_key="my_lib", output_dir=tmp_path / "run", registry=registry, **kwargs + ) + + +def test_an_explicit_id_wins_over_the_registry_entry(tmp_path, registry): + client = _RecordingClient() + + library_id = _pipeline(tmp_path, registry, context7_id="org/explicit")._library_id(client) + + assert library_id == "org/explicit" + assert client.asked == [] + + +def test_an_mcp_descriptor_id_wins_over_the_registry_entry(tmp_path, registry): + """``mcp:...;id=X`` names the library for that source; the registry's id does not replace it.""" + from dokimi.mcp.factory import create_doc_source_client + + client = create_doc_source_client("mcp:http=https://example.com/mcp;id=org/custom") + + library_id = _pipeline(tmp_path, registry)._library_id(client) + + assert library_id == "org/custom" + + +def test_an_explicit_id_wins_over_the_mcp_descriptor_id(tmp_path, registry): + from dokimi.mcp.factory import create_doc_source_client + + client = create_doc_source_client("mcp:http=https://example.com/mcp;id=org/custom") + + library_id = _pipeline(tmp_path, registry, context7_id="org/explicit")._library_id(client) + + assert library_id == "org/explicit" + + +def test_the_registry_key_not_the_display_name_finds_the_declared_id(tmp_path, registry): + client = _RecordingClient() + + library_id = _pipeline(tmp_path, registry)._library_id(client) + + assert library_id == "org/declared" + assert client.asked == [] + + +def test_an_unregistered_product_asks_the_client_for_its_key(tmp_path, registry): + """A key that is not in the registry goes to the client, by its key and not its display name.""" + client = _RecordingClient() + pipeline = EvaluationPipeline( + product="Other Lib", description="d", product_key="other_lib", output_dir=tmp_path / "run", registry=registry + ) + + library_id = pipeline._library_id(client) + + assert library_id == "client:other_lib" + assert client.asked == ["other_lib"] + + +def test_question_generation_extracts_topics_for_the_declared_library(tmp_path, registry, monkeypatch): + """The question step asks the extractor for the registry's id, not for whatever the + client resolves the display name to: the arms stack and this step must agree.""" + import dokimi.questions as questions + + client = _RecordingClient() + seen = {} + + class _StopHere(Exception): + pass + + class _Extractor: + def __init__(self, **kwargs): + pass + + def extract_topics(self, library_id, **kwargs): + seen["library_id"] = library_id + raise _StopHere + + monkeypatch.setattr("dokimi.orchestrator.pipeline.create_doc_source_client", lambda source: client) + monkeypatch.setattr(questions, "RagasSeedExtractor", _Extractor) + + with pytest.raises(_StopHere): + _pipeline(tmp_path, registry)._generate_questions({}) + + assert seen["library_id"] == "org/declared" + + +def test_two_local_doc_sets_of_one_product_get_their_own_topics(tmp_path, registry, monkeypatch): + """Both runs resolve the registry's id, so a topics cache keyed on the id alone gave the + second doc set the first one's topics.""" + import dokimi.questions as questions + + seen = [] + + class _Generator: + def __init__(self, **kwargs): + pass + + def generate_questions(self, topics, **kwargs): + seen.append(sorted(topics)) + return [] + + monkeypatch.setattr(questions, "QuestionGenerator", _Generator) + monkeypatch.chdir(tmp_path) + docs = {"docsA": "`parallel_for` `task_arena`", "docsB": "`flow_graph` `concurrent_queue`"} + for name, words in docs.items(): + (tmp_path / name).mkdir() + (tmp_path / name / "api.md").write_text(f"My Lib API overview {words}\n", encoding="utf-8") + pipeline = _pipeline(tmp_path, registry, doc_source=f"local:{tmp_path / name}") + pipeline._generate_questions({"personas": []}) + + assert seen == [["parallel_for", "task_arena"], ["concurrent_queue", "flow_graph"]] diff --git a/dokimi/tests/test_pipeline_summary.py b/dokimi/tests/test_pipeline_summary.py index 35e802037..a2c38ef07 100644 --- a/dokimi/tests/test_pipeline_summary.py +++ b/dokimi/tests/test_pipeline_summary.py @@ -70,3 +70,28 @@ def test_summary_is_left_out_when_no_question_was_scored(tmp_path, monkeypatch): result = _run(tmp_path, monkeypatch, [_evaluation("q1", None, None)]) assert "summary" not in result["steps"]["evaluation"] + + +def test_summary_says_how_many_questions_it_left_out(tmp_path, monkeypatch, capsys): + """3 of 10 questions are unjudged retrieval misses: the means are over 7, and the + run result and the printed line say so instead of reading as a result over all 10.""" + evaluations = [_evaluation(f"q{i}", 80, 60) for i in range(7)] + evaluations += [_evaluation(f"m{i}", None, 60) for i in range(3)] + + result = _run(tmp_path, monkeypatch, evaluations) + + step = result["steps"]["evaluation"] + assert (step["paired"], step["unpaired"]) == (7, 3) + printed = capsys.readouterr().out + assert "Evaluated 10 answers: context=80.0, baseline=60.0, delta=+20.0" in printed + assert "over the 7 scored in both arms; 3 left out" in printed + + +def test_summary_does_not_say_left_out_when_every_question_is_paired(tmp_path, monkeypatch, capsys): + result = _run(tmp_path, monkeypatch, [_evaluation("q1", 80, 60), _evaluation("q2", 70, 50)]) + + step = result["steps"]["evaluation"] + assert (step["paired"], step["unpaired"]) == (2, 0) + printed = capsys.readouterr().out + assert "Evaluated 2 answers: context=75.0, baseline=55.0, delta=+20.0" in printed + assert "left out" not in printed diff --git a/dokimi/tests/test_provider_call_budgets.py b/dokimi/tests/test_provider_call_budgets.py index cbcd7627a..47229e1e5 100644 --- a/dokimi/tests/test_provider_call_budgets.py +++ b/dokimi/tests/test_provider_call_budgets.py @@ -328,7 +328,7 @@ def test_eval_panel_score_of_1_record_with_3_judges_makes_6_provider_calls( class _DocSource: """A doc-source client with one relevant chunk per query.""" - def get_library_docs(self, library_id, query, max_tokens): + def get_library_docs(self, library_id, query, max_tokens, max_results=5): return [{"content": f"{query} Use parallel_for with a blocked_range.", "source": "s"}] diff --git a/dokimi/tests/test_report_arm_agreement.py b/dokimi/tests/test_report_arm_agreement.py index 96381209f..9d30af4dc 100644 --- a/dokimi/tests/test_report_arm_agreement.py +++ b/dokimi/tests/test_report_arm_agreement.py @@ -238,3 +238,88 @@ def test_eval_report_win_and_degradation_rates_divide_by_the_complete_pairs(tmp_ assert "- **Win rate:** 50% questions improved with context (2/4)." in eval_md assert "- **Degradations:** 1 questions (25%)." in eval_md + + +def test_report_eval_with_no_complete_pair_says_so_instead_of_a_zero_delta(tmp_path): + # One question scored in each arm only: no pair, so there is no delta to report. + evaluations = [ + {"question_id": "q1", "question_text": "How do I pin threads?", + "with_docs": {"aggregate": 80.0}, "without_docs": None, "delta": None}, + {"question_id": "q2", "question_text": "How do I size a pool?", + "with_docs": None, "without_docs": {"aggregate": 60.0}, "delta": None}, + ] + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"judge_model": "m", "evaluations": evaluations}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + text = md.read_text(encoding="utf-8") + + all_row = next(line for line in text.splitlines() if line.startswith("| **All** |")) + assert all_row.split("|")[3:6] == [" — ", " — ", " — "] + assert "Overall delta: +0.0" not in text + assert "over the 0 complete pairs" not in text + assert "no complete pairs" in text + + +def test_breakdown_tables_and_agreement_say_no_pair_instead_of_a_zero_delta(tmp_path): + # The static set (onetbb-Q*) has no complete pair, so its rows and the agreement bullet print no + # zero delta and no agreement with the dynamic set, which has one. + evaluations = [ + {"question_id": "q1", "question_text": "How do I pin threads?", "difficulty": "easy", + "with_docs": {"aggregate": 80.0}, "without_docs": {"aggregate": 60.0}, "delta": 20.0}, + {"question_id": "onetbb-Q1", "question_text": "How do I size a pool?", "difficulty": "hard", + "with_docs": {"aggregate": 80.0}, "without_docs": None, "delta": None}, + {"question_id": "onetbb-Q2", "question_text": "How do I flush a queue?", "difficulty": "hard", + "with_docs": None, "without_docs": {"aggregate": 60.0}, "delta": None}, + ] + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"judge_model": "m", "evaluations": evaluations}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + text = md.read_text(encoding="utf-8") + + static_row = next(line for line in text.splitlines() if line.startswith("| Golden (static) |")) + assert static_row.split("|")[3:6] == [" — ", " — ", " — "] + hard_row = next(line for line in text.splitlines() if line.startswith("| hard |")) + assert hard_row.split("|")[3:6] == [" — ", " — ", " — "] + easy_row = next(line for line in text.splitlines() if line.startswith("| easy |")) + assert "**+20.0**" in easy_row + assert "Dynamic vs Static agreement:** none" in text + assert "Dynamic vs Static agreement:** +" not in text + + +def test_an_all_miss_run_prints_no_zero_mean_delta_or_conclusion(tmp_path): + # The context arm was never judged (a no-retrieval placeholder), so no question is a complete pair. + data = {"judge_model": "m", "evaluations": [ + {"question_id": "q1", "question_text": "How do I pin threads?", "with_docs": None, + "without_docs": {"aggregate": 60.0}, "delta": None}, + {"question_id": "q2", "question_text": "How do I size a pool?", "with_docs": None, + "without_docs": {"aggregate": 40.0}, "delta": None}, + ]} + md = ReportGenerator().generate_report(data, {"questions": []}, "markdown") + stats = json.loads(ReportGenerator().generate_report(data, {"questions": []}, "json"))["stats"] + + assert stats["paired_questions"] == 0 + assert stats["delta"] == {"avg": None, "min": None, "max": None} + assert "minimal impact" not in md + assert "no complete pairs" in md + average = next(line for line in md.splitlines() if line.startswith("| Average |")) + assert average == "| Average | — | — | **—** |" + + +def test_a_group_with_no_complete_pair_has_no_means_in_the_grouped_tables(): + # Persona B and source "manual" hold only q3, which has a context score and no baseline score. + evaluations = [EVALUATIONS[0], EVALUATIONS[2]] + questions = [{"id": "q1", "persona_id": "A", "source_type": "generated"}, + {"id": "q3", "persona_id": "B", "source_type": "manual"}] + report = json.loads(ReportGenerator().generate_report({"evaluations": evaluations}, {"questions": questions}, "json")) + clusters = {c["topic"]: c for c in report["clusters"]} + assert clusters["B"] == {"topic": "B", "count": 1, "unpaired": 1, + "with_avg": None, "without_avg": None, "delta_avg": None} + sources = {s["source_type"]: s for s in report["sources_summary"]} + assert sources["manual"]["with_avg"] is None and sources["manual"]["delta_avg"] is None + md = ReportGenerator().generate_report({"evaluations": evaluations}, {"questions": questions}, "markdown") + assert "| B | 1 (1 unpaired) | — | — | **—** |" in md + assert "| manual | 1 (1 unpaired) | — | — | **—** |" in md diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py index ecb3932c3..81382eae9 100644 --- a/dokimi/tests/test_trust_gate.py +++ b/dokimi/tests/test_trust_gate.py @@ -182,11 +182,24 @@ def _judged(n_hit, n_miss): ] def test_fails_when_retrieval_never_returned_a_doc(self): - evals = self._judged(n_hit=0, n_miss=12) - assert all(e["with_docs"]["aggregate"] > 0 for e in evals) + evals = self._judged(n_hit=0, n_miss=14) + # Every question is an unjudged miss: nothing is scored in both arms, the run is not + # trusted, and the message names the cause (0 of 14 hit) beside the question count. + verdict = evaluate_trust(evals) + assert not verdict.is_trusted + assert [c.name for c in verdict.failures] == ["min_questions"] + check = next(c for c in verdict.checks if c.name == "retrieval_hit_rate") + assert not check.passed and check.measured and check.value == 0.0 + assert "(0/14)" in check.message + + def test_fails_when_retrieval_rarely_returned_a_doc(self): + evals = self._judged(n_hit=2, n_miss=12) + # The placeholder is not judged by default: only the two hits carry a with-docs score, + # and the twelve misses still count against the rate. + assert sum(e["with_docs"] is None for e in evals) == 12 check = next(c for c in evaluate_trust(evals).checks if c.name == "retrieval_hit_rate") assert not check.passed, check.message - assert check.value == 0.0 + assert check.value == round(2 / 14, 3) assert check.measured def test_rate_counts_recorded_hits(self): @@ -353,6 +366,15 @@ def test_coverage_below_the_default_floor_is_not_measured(self): assert not _check(v, "retrieval_hit_rate").measured assert v.not_measured == ["retrieval_hit_rate"] + def test_legacy_judge_error_rows_leave_a_stored_rate_measured(self): + # A judge-error row as eval score wrote it before error rows carried retrieval_hit. + legacy_error = {"question_id": "q", "error": "judge failed", "with_docs": None, + "without_docs": None, "delta": None} + evals = ([_row(70, 50, hit=True)] * 5 + [_row(70, 50, hit=False)] * 10 + + [legacy_error] * 5) + check = _check(evaluate_trust(evals), "retrieval_hit_rate") + assert (check.measured, check.value, check.passed) == (True, 0.333, False) + def test_a_zero_baseline_counts_as_zero(self): check = _check(evaluate_trust(_evals(10, with_score=40, without_score=0)), "min_delta") assert check.value == 40.0 @@ -432,7 +454,7 @@ def test_failures_and_warnings_render_apart(self): "| Check | Status | Value | Threshold | Notes |", "|-------|--------|-------|-----------|-------|", "| min_questions | ❌ | 3 | 10 | 3 questions evaluated (min 10) |", - "| zero_score_fraction | ❌ | 0.333 | 0.2 | 1/3 questions scored 0 in the context arm (33.3%) |", + "| zero_score_fraction | ❌ | 0.333 | 0.2 | 1/3 questions scored 0 or had no document in the context arm (33.3%) |", "| min_with_docs_avg | ✅ | 33.3 | 30.0 | Context-arm avg = 33.3 (min 30.0) |", "| min_delta | ✅ | 13.3 | -5.0 | Avg delta (context arm minus baseline) = +13.3 (min -5.0) |", "| score_cv | ⚠️ | 0.917 | 0.6 | Score CV = 0.92 std=30.6 (max CV 0.6) |", @@ -442,7 +464,7 @@ def test_failures_and_warnings_render_apart(self): "", "**Blocking issues:**", "- ❌ **min_questions**: 3 questions evaluated (min 10)", - "- ❌ **zero_score_fraction**: 1/3 questions scored 0 in the context arm (33.3%)", + "- ❌ **zero_score_fraction**: 1/3 questions scored 0 or had no document in the context arm (33.3%)", "", "**Warnings:**", "- ⚠️ **score_cv**: Score CV = 0.92 std=30.6 (max CV 0.6)", diff --git a/dokimi/tests/test_trust_gate_unjudged_misses.py b/dokimi/tests/test_trust_gate_unjudged_misses.py new file mode 100644 index 000000000..0b2053282 --- /dev/null +++ b/dokimi/tests/test_trust_gate_unjudged_misses.py @@ -0,0 +1,196 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The zero-score check counts unjudged retrieval misses as unusable. + +A question whose retrieval missed gets no with-docs judgement, so it has no +context score. The zero-score check must still count it. Otherwise a run where +most questions miss passes a fail-severity check it fails on intel/main. +""" + +from unittest.mock import MagicMock + +import pytest + +from dokimi.eval.diagnoser import EMPTY_RETRIEVAL, INSUFFICIENT_DATA, summarise_diagnoses +from dokimi.eval.judge import Judge +from dokimi.gate.trust_gate import evaluate_trust + +GOOD = MagicMock(content=( + '{"correctness": 60, "completeness": 60, "specificity": 60, ' + '"code_quality": 60, "actionability": 60, "aggregate": 0}' +)) +HIT = { + "answer": "use parallel_for", + "doc_source": "context7", + "retrieved_docs": [{"source": "context7", "snippet": "parallel_for ...", + "relevance_score": 0.8}], +} +MISS = { + "answer": "[FALLBACK: No relevant documentation found]", + "doc_source": "fallback_none", + "retrieved_docs": [], +} + + +def _rows(n_hit, n_miss): + """Judge n_hit questions with a document and n_miss without, as the judge does.""" + judge = Judge.__new__(Judge) + judge._accumulate_usage = lambda usage: None + judge.temperature = judge.seed = None + judge.llm = MagicMock() + judge.llm.invoke.return_value = GOOD + rows = [] + for i in range(n_hit + n_miss): + rows.append(judge._evaluate_answer_pair("L", { + "question_id": f"q{i}", + "question_text": "q?", + "with_docs": dict(HIT if i < n_hit else MISS), + "without_docs": {"answer": "baseline"}, + })) + return rows + + +def _zero_check(verdict): + (check,) = [c for c in verdict.checks if c.name == "zero_score_fraction"] + return check + + +def test_four_misses_of_fourteen_fail_the_zero_check(): + check = _zero_check(evaluate_trust(_rows(10, 4))) + assert check.severity == "fail" + assert check.value == 0.286 + assert check.passed is False + + +def test_all_hits_pass_the_zero_check(): + check = _zero_check(evaluate_trust(_rows(14, 0))) + assert check.value == 0.0 + assert check.passed is True + + +def _retrieval_check(verdict): + (check,) = [c for c in verdict.checks if c.name == "retrieval_hit_rate"] + return check + + +def test_judge_error_rows_keep_the_retrieval_outcome_of_their_answer(monkeypatch): + """5 of 20 answers fail in the judge: the retrieval rate is still measured over all 20.""" + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + judge = Judge(model="gpt-x", provider="openai") + judge.llm = MagicMock() + judge.llm.invoke.side_effect = lambda msgs, *a, **k: GOOD + real = judge._evaluate_answer_pair + + def flaky(library_name, item): + if int(item.question_id[1:]) >= 15: + raise RuntimeError("judge down") + return real(library_name, item) + + judge._evaluate_answer_pair = flaky + answers = [ + {"question_id": f"q{i}", "question_text": "q?", + "with_docs": dict(HIT), "without_docs": {"answer": "baseline"}} + for i in range(20) + ] + rows = judge.evaluate_answers("L", answers) + assert sum("error" in r for r in rows) == 5 + check = _retrieval_check(evaluate_trust(rows)) + assert check.value == 1.0 + assert check.measured is not False + assert "(20/20)" in check.message + + +def test_unjudged_misses_stay_empty_retrievals_in_the_failure_analysis(): + summary = summarise_diagnoses(_rows(10, 4)) + assert summary["counts"][EMPTY_RETRIEVAL] == 4 + assert summary["counts"][INSUFFICIENT_DATA] == 0 + + +def _row(question_id, context, baseline, retrieval_hit): + """An eval record as the judge writes it: a null condition is one that was not scored.""" + return { + "question_id": question_id, + "question_text": "q?", + "with_docs": None if context is None else {"aggregate": context}, + "without_docs": None if baseline is None else {"aggregate": baseline}, + "delta": None if context is None or baseline is None else context - baseline, + "retrieval_hit": retrieval_hit, + } + + +@pytest.mark.parametrize("extra", [ + # no context score, and no retrieval outcome recorded: an older artifact, or a judge error + # on an answer whose outcome was never stamped + _row("legacy", None, 60, None), + # a document reached the prompt, but the context answer was not scored (judge error) + _row("judge-error", None, 60, True), + # a miss with no baseline score: it is not a question both arms were meant to score + _row("no-baseline", None, None, False), +], ids=["legacy-row", "hit-not-scored", "miss-without-baseline"]) +def test_only_a_recorded_miss_with_a_baseline_counts_as_unusable(extra): + rows = [_row(f"q{i}", 70, 60, True) for i in range(10)] + + check = _zero_check(evaluate_trust(rows + [extra])) + + assert (check.value, check.message.split(" ")[0]) == (0.0, "0/10") + + +def test_a_scored_miss_counts_once_in_the_zero_check(): + # A miss the judge scored (--judge-placeholder) with a 0 context score is a zero already; it + # must not also count as an unjudged miss, or the numerator and the denominator both grow. + rows = [_row(f"q{i}", 70, 60, True) for i in range(10)] + [_row("scored-miss", 0, 60, False)] + + check = _zero_check(evaluate_trust(rows)) + + assert (check.value, check.message.split(" ")[0]) == (round(1 / 11, 3), "1/11") + + +def _judged_with_failures(n_hit, n_miss, failing): + """Judge the answers through ``evaluate_answers``; each question id in ``failing`` raises in the judge.""" + judge = Judge(model="gpt-x", provider="openai") + judge.llm = MagicMock() + judge.llm.invoke.side_effect = lambda msgs, *a, **k: GOOD + real = judge._evaluate_answer_pair + + def flaky(library_name, item): + if item.question_id in failing: + raise RuntimeError("judge down") + return real(library_name, item) + + judge._evaluate_answer_pair = flaky + answers = [ + {"question_id": f"h{i}", "question_text": "q?", + "with_docs": dict(HIT), "without_docs": {"answer": "baseline"}} + for i in range(n_hit) + ] + [ + {"question_id": f"m{i}", "question_text": "q?", + "with_docs": dict(MISS), "without_docs": {"answer": "baseline"}} + for i in range(n_miss) + ] + return judge.evaluate_answers("L", answers) + + +def test_a_judge_error_on_a_retrieval_miss_counts_as_unusable(monkeypatch): + """The judge raises on a miss: its error row has no scores, but it is still an unusable question.""" + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + rows = _judged_with_failures(3, 1, failing={"m0"}) + error_row = next(r for r in rows if r["question_id"] == "m0") + assert "error" in error_row and error_row["retrieval_hit"] is False + + check = _zero_check(evaluate_trust(rows)) + + assert (check.value, check.passed) == (0.25, False) + assert check.message.split(" ")[0] == "1/4" + + +def test_a_judge_error_on_a_retrieval_hit_stays_out_of_the_zero_check(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-key-not-used") + rows = _judged_with_failures(3, 0, failing={"h0"}) + error_row = next(r for r in rows if r["question_id"] == "h0") + assert "error" in error_row and error_row["retrieval_hit"] is True + + check = _zero_check(evaluate_trust(rows)) + + assert (check.value, check.passed) == (0.0, True) + assert check.message.split(" ")[0] == "0/2"