diff --git a/dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md b/dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md new file mode 100644 index 000000000..a8b29f3c1 --- /dev/null +++ b/dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md @@ -0,0 +1,9 @@ + + +### Added + +- **`subjects evidence SCORECARD`: a subject scorecard as a `skill_evidence.v1` document** — the file a publisher puts next to a skill so a client can read what was measured: one result per (matrix cell, non-baseline arm) with model, provider, harness, effort, and the paired test of the arm against the cell's recorded baseline arm on `avg_aggregate` (`n` pairs, delta, confidence interval, corrected p-value) and a verdict (`improved` or `regressed` only on a delta that is significant after correction, `inconclusive` otherwise). It is keyed by `subject.digest`, the scorecard's `distribution_digest`, which is what a `.well-known/agent-skills` index entry carries for the skill. Free: it reads the saved scorecard and makes no model call. Every value comes from the scorecard; one it does not hold, such as `effort` before runs record it, is `null`. Writes `skill-evidence.json` next to the scorecard unless `--out` names a file; exits 2 when the scorecard has nothing to publish or `--out` is the scorecard itself. The schema is experimental until the agentskills evidence proposal settles. +- **Matrix rollup and subject scorecard cells record `baseline_arm`**, so a reader no longer has to guess the baseline from the arm that has no delta (an arm whose every answer failed has none either). diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 8ab11f976..13ddffed5 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -13050,6 +13050,7 @@ "'repo_environment'", "'repo_readiness'", "'result'", + "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", @@ -13170,6 +13171,62 @@ "path": "dokimi subjects", "subcommand_required": true, "subcommands": { + "evidence": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "scorecard", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi subjects evidence", + "subcommand_required": null, + "subcommands": {} + }, "list": { "allow_abbrev": true, "params": [ diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 55fd25cf6..5f81d1b59 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -47,6 +47,7 @@ "matrix_rollup": "matrix_rollup.v1", "plugin_delta": "plugin_delta.v1", "subject_scorecard": "subject_scorecard.v2", + "skill_evidence": "skill_evidence.v1", "novelty": "novelty.v1", "task_runs": "task_runs.v1", "explore_runs": "explore_runs.v1", diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 8d57df1ed..ed6dc3265 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -99,6 +99,7 @@ class Tombstone: Leaf(("subjects", "list"), FREE), Leaf(("subjects", "show"), FREE), Leaf(("subjects", "run"), PAID), + Leaf(("subjects", "evidence"), FREE), Leaf(("tasks", "list"), FREE), Leaf(("tasks", "inspect"), FREE), Leaf(("tasks", "harnesses"), FREE), diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index 731b5fdd7..7f4081104 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -166,6 +166,45 @@ def cmd_subjects_run(args: argparse.Namespace) -> None: print(f"OK Saved subject report: {out_md}") +def cmd_subjects_evidence(args: argparse.Namespace) -> None: + """Render a saved subject scorecard as a skill_evidence.v1 document; no model calls.""" + from dokimi.artifacts import ArtifactValidationError, load_artifact, save_artifact + from dokimi.subjects.evidence import build_skill_evidence + + try: + scorecard = load_artifact("subject_scorecard", Path(args.scorecard)) + except (OSError, ValueError, ArtifactValidationError) as exc: + usage_exit(f"cannot read subject scorecard {args.scorecard}: {exc}") + try: + evidence = build_skill_evidence(scorecard, source=str(args.scorecard)) + except (AttributeError, TypeError) as exc: + usage_exit(f"{args.scorecard} is not a well-formed subject scorecard: {exc}") + if not evidence["results"]: + usage_exit(f"{args.scorecard} holds no awareness arm with a baseline to compare against: " + "nothing to publish") + out = Path(args.out) if args.out else Path(args.scorecard).with_name("skill-evidence.json") + try: + out_real = out.resolve() + # save_artifact keeps and removes ".rejected"; that sibling must not be the scorecard. + stale_real = out.with_name(out.name + ".rejected").resolve() + card_real = Path(args.scorecard).resolve() + # A symlink loop raises RuntimeError before Python 3.13; "." or "/" has no name, so with_name raises ValueError. + except (OSError, RuntimeError, ValueError) as exc: + usage_exit(f"cannot resolve --out {out}: {exc}") + if out_real == card_real: + usage_exit("--out names the scorecard itself; pick another file") + if stale_real == card_real: + usage_exit(f"{out}.rejected is the scorecard; a save to {out} would delete or overwrite it; pick another --out") + try: + with exit_on_unwritable(out): + save_artifact("skill_evidence", evidence, out) + except ArtifactValidationError as exc: + usage_exit(f"{args.scorecard} holds a value the evidence schema rejects: {exc}") + for warning in evidence["warnings"]: + print(f"WARN {warning}", file=sys.stderr) + print(f"OK Saved skill evidence: {out} ({len(evidence['results'])} results)") + + def register(sub, positive_int) -> None: """Add the `subjects` subcommand group.""" subjects_p = sub.add_parser( @@ -264,6 +303,15 @@ def register(sub, positive_int) -> None: ) run_p.set_defaults(func=cmd_subjects_run) + evidence_p = subjects_sub.add_parser( + "evidence", + help="Render a subject scorecard as a skill_evidence.v1 document (no model calls)", + ) + evidence_p.add_argument("scorecard", help="subject_scorecard JSON written by `subjects run`") + evidence_p.add_argument("--out", default=None, + help="Output file (default: skill-evidence.json next to the scorecard)") + evidence_p.set_defaults(func=cmd_subjects_evidence) + def _run_plan(descriptor, args: argparse.Namespace) -> tuple[Path, str, list[str]]: """The output dir, baseline arm name and per-product questions, or exit 2 when the descriptor cannot run. diff --git a/dokimi/dokimi/eval/matrix_rollup.py b/dokimi/dokimi/eval/matrix_rollup.py index 4eb4fef1c..38ff41ce5 100644 --- a/dokimi/dokimi/eval/matrix_rollup.py +++ b/dokimi/dokimi/eval/matrix_rollup.py @@ -30,6 +30,7 @@ def build_matrix_rollup( "harness": data.get("harness"), "plugin_set": data.get("plugin_set", "none"), "plugin_set_id": data.get("plugin_set_id"), + "baseline_arm": data.get("baseline_arm"), "summary": data.get("summary", {}), "cost_summary": data.get("cost_summary", {}), } diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index fde7e7cfa..7652f6580 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -45,6 +45,8 @@ class KindFields: "plugin_delta": KindFields(subject=("library_name",), model=("model",), counts=("arms",)), "subject_scorecard": KindFields(subject=("subject.id",), time=("generated_at",), counts=("cells",)), + "skill_evidence": KindFields(subject=("subject.id",), time=("generated_at",), + counts=("results", "warnings")), "novelty": KindFields(subject=("artifact.ref",), model=("model",), time=("generated_at",), counts=("conflicts", "warnings")), "task_runs": KindFields(model=("model",), time=("generated_at",), diff --git a/dokimi/dokimi/schemas/matrix_rollup.v1.json b/dokimi/dokimi/schemas/matrix_rollup.v1.json index 0c964b14d..1f68767c6 100644 --- a/dokimi/dokimi/schemas/matrix_rollup.v1.json +++ b/dokimi/dokimi/schemas/matrix_rollup.v1.json @@ -74,6 +74,12 @@ }, "cost_summary": { "type": "object" + }, + "baseline_arm": { + "type": [ + "string", + "null" + ] } }, "additionalProperties": true diff --git a/dokimi/dokimi/schemas/skill_evidence.v1.json b/dokimi/dokimi/schemas/skill_evidence.v1.json new file mode 100644 index 000000000..c52232b84 --- /dev/null +++ b/dokimi/dokimi/schemas/skill_evidence.v1.json @@ -0,0 +1,255 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/skill_evidence.v1.json", + "title": "Skill evidence document v1", + "description": "Experimental until the agentskills evidence proposal settles. Measured effect of a skill against a baseline, keyed by the digest a .well-known/agent-skills index carries for it. Rendered from a subject scorecard; no value is filled in that the scorecard does not hold. Every statistic in a result is the paired test's: n is the number of paired questions behind delta, ci and p_value.", + "type": "object", + "required": [ + "schema_version", + "subject", + "results" + ], + "properties": { + "schema_version": { + "const": "skill_evidence.v1" + }, + "generated_at": { + "type": "string" + }, + "producer": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "subject": { + "type": "object", + "required": [ + "digest" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "type": [ + "string", + "null" + ] + }, + "ref": { + "type": [ + "string", + "null" + ] + }, + "digest": { + "type": [ + "string", + "null" + ], + "description": "sha256 of the served SKILL.md bytes, as in the index entry; null when it cannot be derived" + }, + "content_digest": { + "type": [ + "string", + "null" + ] + } + } + }, + "eval_set": { + "type": "object", + "properties": { + "products": { + "type": "array", + "items": { + "type": "string" + } + }, + "questions": { + "type": "array", + "items": { + "type": "string" + } + }, + "digest": { + "type": [ + "string", + "null" + ] + } + } + }, + "baseline": { + "type": [ + "string", + "null" + ] + }, + "measured_at": { + "type": [ + "string", + "null" + ] + }, + "source": { + "type": "object", + "properties": { + "kind": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": [ + "string", + "null" + ] + } + } + }, + "results": { + "type": "array", + "items": { + "type": "object", + "required": [ + "arm", + "metric", + "delta", + "verdict" + ], + "properties": { + "product": { + "type": [ + "string", + "null" + ] + }, + "matrix_cell": { + "type": [ + "string", + "null" + ] + }, + "model": { + "type": [ + "string", + "null" + ], + "description": "model as configured for the cell, not a resolved provider id" + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "harness": { + "type": [ + "string", + "null" + ] + }, + "effort": { + "type": [ + "string", + "null" + ] + }, + "judge_model": { + "type": [ + "string", + "null" + ] + }, + "arm": { + "type": "string" + }, + "metric": { + "type": "string", + "description": "dokimi's mean judge aggregate score per answer (higher is better); delta is arm minus baseline" + }, + "n": { + "type": [ + "integer", + "null" + ] + }, + "delta": { + "type": [ + "number", + "null" + ] + }, + "ci": { + "type": [ + "object", + "null" + ], + "properties": { + "lo": { + "type": [ + "number", + "null" + ] + }, + "hi": { + "type": [ + "number", + "null" + ] + }, + "confidence": { + "type": [ + "number", + "null" + ] + }, + "method": { + "type": [ + "string", + "null" + ] + } + } + }, + "p_value": { + "type": [ + "number", + "null" + ] + }, + "p_correction": { + "type": [ + "string", + "null" + ] + }, + "verdict": { + "enum": [ + "improved", + "regressed", + "inconclusive" + ] + } + } + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py new file mode 100644 index 000000000..8f76cf478 --- /dev/null +++ b/dokimi/dokimi/subjects/evidence.py @@ -0,0 +1,119 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Render a subject scorecard as a ``skill_evidence.v1`` document. + +The scorecard is dokimi's own record of a run: every cell, rollup path and warning. The evidence +document is what a publisher hands a client next to the skill: one row per (cell, arm) with the +measured effect against the baseline, keyed by the digest a ``.well-known/agent-skills`` index +carries for the skill, so a client can match it to the skill it installed. Nothing is measured +here: every number is read from the scorecard, and a value the scorecard does not hold is null, +never filled in. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import PurePath +from typing import Any + +from dokimi import __version__ + +#: The arm metric the awareness summary carries per arm (``summary.per_arm[arm]``). +METRIC = "avg_aggregate" + + +def build_skill_evidence(scorecard: dict[str, Any], *, source: str | None = None) -> dict[str, Any]: + """A ``skill_evidence.v1`` payload from a ``subject_scorecard`` payload. + + ``source`` names the scorecard file it was read from; only its file name is kept, since the + directory is the producer's disk layout and not part of the result. + """ + subject = scorecard.get("subject") or {} + suite = scorecard.get("suite") or {} + warnings = list(scorecard.get("warnings") or []) + results: list[dict[str, Any]] = [] + for cell in scorecard.get("cells") or []: + per_arm = (cell.get("summary") or {}).get("per_arm") or {} + baseline = _baseline_arm(cell, per_arm) + if baseline is None: + warnings.append(f"matrix cell {cell.get('matrix_cell')!r}: no single baseline arm, skipped") + continue + results.extend(_result(cell, arm, row) for arm, row in per_arm.items() if arm != baseline) + if subject.get("distribution_digest") is None: + warnings.append("subject has no distribution_digest (not a SKILL.md, a skill with resources, " + "or a non-local ref): a client cannot match this document by digest") + if subject.get("kind") != "skill": + warnings.append(f"subject kind is {subject.get('kind')!r}, not 'skill'") + if (scorecard.get("work") or {}).get("status") == "run": + warnings.append("task-suite (work) results are not in this document; only the awareness arms are") + return { + "schema_version": "skill_evidence.v1", + "generated_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z", + "producer": {"name": "dokimi", "version": __version__}, + "subject": { + "id": subject.get("id"), + "kind": subject.get("kind"), + "ref": subject.get("ref"), + "digest": subject.get("distribution_digest"), + "content_digest": subject.get("ref_digest"), + }, + "eval_set": { + "products": list(suite.get("products") or []), + "questions": list(suite.get("questions") or []), + "digest": suite.get("question_set_hash"), + }, + "baseline": scorecard.get("baseline"), + "measured_at": scorecard.get("generated_at"), + "source": {"kind": scorecard.get("schema_version"), "path": source and PurePath(source).name}, + "results": results, + "warnings": sorted(set(warnings)), + } + + +def _baseline_arm(cell: dict[str, Any], per_arm: dict[str, Any]) -> str | None: + """The cell's baseline arm: as recorded, else the one arm with no delta at all, else None. + + ``delta_vs_baseline`` is also None for an arm with no paired question (every answer failed), + so the fallback for a cell recorded before ``baseline_arm`` holds only when one arm qualifies. + """ + if cell.get("baseline_arm") in per_arm: + return cell["baseline_arm"] + candidates = [name for name, row in per_arm.items() + if row.get("delta_vs_baseline") is None and not row.get("significance")] + return candidates[0] if len(candidates) == 1 else None + + +def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, Any]: + # Every statistic is the paired test's, so n, delta, CI and p describe the same questions. + # The per-arm means are left out: their difference is not the delta once pairs drop out. + sig = row.get("significance") or {} + delta = sig.get("delta_mean") if sig else None + return { + "product": cell.get("product"), + "matrix_cell": cell.get("matrix_cell"), + "model": cell.get("model"), + "provider": cell.get("provider"), + "harness": cell.get("harness"), + # Absent until a run records the effort it applied; null means "not recorded", not "default". + "effort": cell.get("effort"), + "judge_model": cell.get("judge_model"), + "arm": arm, + "metric": METRIC, + "n": sig.get("n"), + "delta": delta, + "ci": None if sig.get("ci_lo") is None else { + "lo": sig.get("ci_lo"), "hi": sig.get("ci_hi"), + "confidence": sig.get("ci_confidence"), "method": sig.get("ci_method"), + }, + "p_value": sig.get("p_adjusted"), + "p_correction": sig.get("correction"), + "verdict": _verdict(delta, sig), + } + + +def _verdict(delta: float | None, sig: dict[str, Any]) -> str: + """``improved``/``regressed`` only on a corrected significant delta; else ``inconclusive``.""" + if delta is None or sig.get("significant_adjusted") is not True: # a string "false" is not a verdict + return "inconclusive" + return "improved" if delta > 0 else "regressed" if delta < 0 else "inconclusive" diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index b2772bf19..634331ff8 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -58,6 +58,7 @@ def build_subject_scorecard( "model": cell.get("model"), "provider": cell.get("provider"), "harness": cell.get("harness"), + "baseline_arm": cell.get("baseline_arm"), "plugin_set": cell.get("plugin_set"), "summary": cell.get("summary", {}), "cost_summary": cell.get("cost_summary", {}), diff --git a/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json b/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json new file mode 100644 index 000000000..1c82de3ee --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json @@ -0,0 +1,77 @@ +{ + "baseline": "baseline", + "eval_set": { + "digest": "sha256:7b0379b853ffc6170161dede74ec099e8819110188311444256faab6eea2d06e", + "products": [ + "oneTBB" + ], + "questions": [ + "questions.json" + ] + }, + "generated_at": "2026-10-31T10:05:00Z", + "measured_at": "2026-10-04T17:47:19.175356Z", + "producer": { + "name": "dokimi", + "version": "0.0.0" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 13.5, + "lo": 10.5, + "method": "student_t" + }, + "delta": 12.0, + "effort": null, + "judge_model": null, + "harness": "arms-runner", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "model": "m", + "n": 6, + "p_correction": "holm", + "p_value": 0.001, + "product": "oneTBB", + "provider": "p", + "verdict": "improved" + }, + { + "arm": "placebo", + "ci": { + "confidence": 0.95, + "hi": 2.5, + "lo": -0.5, + "method": "student_t" + }, + "delta": 1.0, + "effort": null, + "judge_model": null, + "harness": "arms-runner", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "model": "m", + "n": 6, + "p_correction": "holm", + "p_value": 0.4, + "product": "oneTBB", + "provider": "p", + "verdict": "inconclusive" + } + ], + "schema_version": "skill_evidence.v1", + "source": { + "kind": "subject_scorecard.v2", + "path": "subject-scorecard.json" + }, + "subject": { + "content_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index b5bd54ee2..a5df980c3 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -34,6 +34,7 @@ "matrix_rollup.v1": ({}, "oneTBB", None, "2026-10-04T17:47:16.294238Z"), "plugin_delta.v1": ({}, "oneTBB", "m", None), "subject_scorecard.v2": ({}, "docs-subject", None, "2026-10-04T17:47:19.175356Z"), + "skill_evidence.v1": ({}, "s", None, "2026-10-31T10:05:00Z"), "novelty.v1": ({}, "skill:/tmp/run/accel-tuning/SKILL.md", "answerer-m", "2026-10-04T17:47:16.841483Z"), "task_runs.v1": ({}, None, "haiku", "2026-01-01T00:00:00Z"), diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index de76bb8d5..7d5a7c8ad 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -36,6 +36,7 @@ "repo_readiness.v1": {"files": 2, "claims": 52}, "result.v1": {"cells": 6}, "subject_scorecard.v2": {"cells": 0}, + "skill_evidence.v1": {"results": 2, "warnings": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, "eval_coverage.v1": {"gaps": 1, "evals": 2, "problems": 0}, "need_graph.v1": {"nodes": 3, "edges": 1}, diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py new file mode 100644 index 000000000..15f0c2625 --- /dev/null +++ b/dokimi/tests/test_subjects_evidence.py @@ -0,0 +1,265 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""skill_evidence.v1: a subject scorecard rendered for publishing next to the skill.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +from artifact_fixtures import artifact_fixture +from dokimi.artifacts import load_artifact, save_artifact +from dokimi.subjects.evidence import build_skill_evidence + +DIGEST = "sha256:" + "a" * 64 +FIXTURE = Path(__file__).parent / "fixtures" / "artifacts" / "subject_scorecard.v2.json" + + +def _significance(delta: float, *, significant: bool) -> dict: + return {"ci_confidence": 0.95, "ci_lo": delta - 1.5, "ci_hi": delta + 1.5, "ci_method": "student_t", + "correction": "holm", "delta_mean": delta, "n": 6, "p_adjusted": 0.001 if significant else 0.4, + "significant_adjusted": significant} + + +def _scorecard(**subject) -> dict: + per_arm = { + "baseline": {"avg_aggregate": 50.0, "delta_vs_baseline": None, "n": 6}, + "skill:s": {"avg_aggregate": 62.0, "delta_vs_baseline": 12.0, "n": 6, + "significance": _significance(12.0, significant=True)}, + "placebo": {"avg_aggregate": 51.0, "delta_vs_baseline": 1.0, "n": 6, + "significance": _significance(1.0, significant=False)}, + } + # The committed fixture, so the base is a scorecard a producer wrote; only subject and cells change. + base = json.loads(FIXTURE.read_text(encoding="utf-8")) + base["subject"] = {"id": "s", "kind": "skill", "ref": "skills/s", "ref_digest": "sha256:" + "b" * 64, + "distribution_digest": DIGEST, "members": [], **subject} + base["cells"] = [{"product": "oneTBB", "questions": "q.json", "matrix_cell": "c1", "artifact": "c1.json", + "model": "m", "provider": "p", "harness": "arms-runner", "baseline_arm": "baseline", + "plugin_set": "none", + "summary": {"per_arm": per_arm}, "cost_summary": {}}] + return artifact_fixture("subject_scorecard", base, closed=False) + + +def test_each_non_baseline_arm_is_one_result_read_from_the_scorecard(tmp_path: Path): + evidence = build_skill_evidence(_scorecard(), source="sc.json") + + assert evidence["subject"]["digest"] == DIGEST + assert evidence["eval_set"]["digest"] == _scorecard()["suite"]["question_set_hash"] + by_arm = {row["arm"]: row for row in evidence["results"]} + assert set(by_arm) == {"skill:s", "placebo"} + skill = by_arm["skill:s"] + assert (skill["model"], skill["harness"], skill["n"]) == ("m", "arms-runner", 6) + assert (skill["n"], skill["delta"]) == (6, 12.0) + assert skill["ci"] == {"lo": 10.5, "hi": 13.5, "confidence": 0.95, "method": "student_t"} + assert (skill["p_value"], skill["p_correction"], skill["verdict"]) == (0.001, "holm", "improved") + assert by_arm["placebo"]["verdict"] == "inconclusive" + # Not recorded by the run, so not filled in. + assert skill["effort"] is None and skill["judge_model"] is None + save_artifact("skill_evidence", evidence, tmp_path / "e.json") # validates against the schema + + +@pytest.mark.parametrize("delta, significant, verdict", [ + (-3.0, True, "regressed"), (-3.0, False, "inconclusive"), (0.0, True, "inconclusive"), +]) +def test_only_a_corrected_significant_delta_gets_a_direction(delta, significant, verdict): + scorecard = _scorecard() + row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] + row["delta_vs_baseline"] = delta + row["significance"] = _significance(delta, significant=significant) + + by_arm = {r["arm"]: r for r in build_skill_evidence(scorecard)["results"]} + assert by_arm["skill:s"]["verdict"] == verdict + + +def test_a_subject_with_no_distribution_digest_is_flagged_not_guessed(): + evidence = build_skill_evidence(_scorecard(distribution_digest=None)) + + assert evidence["subject"]["digest"] is None + assert any("distribution_digest" in w for w in evidence["warnings"]) + + +def test_subjects_evidence_writes_the_document_next_to_the_scorecard(tmp_path: Path, monkeypatch): + from dokimi.cli import main + + scorecard = tmp_path / "subject-scorecard.json" + save_artifact("subject_scorecard", _scorecard(), scorecard) + + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard)]) + main() + + out = load_artifact("skill_evidence", tmp_path / "skill-evidence.json") + assert out["source"] == {"kind": "subject_scorecard.v2", "path": "subject-scorecard.json"} + assert len(out["results"]) == 2 + + +def test_subjects_evidence_refuses_a_file_that_is_not_a_scorecard(tmp_path: Path, monkeypatch): + from dokimi.cli import main + + bad = tmp_path / "x.json" + bad.write_text(json.dumps({"schema_version": "subject_scorecard.v2"}), encoding="utf-8") + with pytest.raises(SystemExit) as exc: + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad)]) + main() + assert exc.value.code == 2 + + +@pytest.mark.parametrize("cells", [ + [None], + [{"summary": []}], + [{"summary": {"per_arm": []}}], + [{"summary": {"per_arm": {"skill:s": 5}}}], + [{"summary": {"per_arm": {"skill:s": {"significance": "x"}}}}], +]) +def test_subjects_evidence_refuses_a_scorecard_with_a_wrong_typed_cell(tmp_path: Path, monkeypatch, cells): + from dokimi.cli import main + + bad = tmp_path / "x.json" + bad.write_text(json.dumps({**_scorecard(), "cells": cells}), encoding="utf-8") + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + + +def test_subjects_evidence_refuses_a_nested_statistic_the_evidence_schema_rejects(tmp_path: Path, monkeypatch): + # The scorecard schema accepts a string n and rendering copies it through; only the evidence + # schema rejects it at save time. That must be an input diagnostic (exit 2), not a traceback. + from dokimi.cli import main + + card = _scorecard() + card["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"]["n"] = "6" + bad = tmp_path / "x.json" + bad.write_text(json.dumps(card), encoding="utf-8") + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + assert not (tmp_path / "skill-evidence.json").exists() + + +def test_subjects_evidence_refuses_an_out_whose_rejected_sibling_is_the_scorecard(tmp_path: Path, monkeypatch): + # save_artifact removes ".rejected" on a valid save; here that would delete the scorecard. + from dokimi.cli import main + + card = tmp_path / "sc.json.rejected" + card.write_text(json.dumps(_scorecard()), encoding="utf-8") + before = card.read_bytes() + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(card), "--out", str(tmp_path / "sc.json")]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + assert card.read_bytes() == before + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows") +def test_subjects_evidence_refuses_an_out_that_is_a_symlink_loop(tmp_path: Path, monkeypatch): + # resolve() raises RuntimeError on a symlink loop before Python 3.13; that must be a diagnostic. + from dokimi.cli import main + + bad = tmp_path / "sc.json" + bad.write_text(json.dumps(_scorecard()), encoding="utf-8") + (tmp_path / "loop").symlink_to(tmp_path / "loop") + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad), "--out", str(tmp_path / "loop")]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + + +@pytest.mark.skipif(sys.platform == "win32", reason="the root is not a bare name there") +@pytest.mark.parametrize("out", [".", "/"]) +def test_subjects_evidence_refuses_an_out_with_no_file_name(tmp_path: Path, monkeypatch, out): + # Path("." or "/").with_name(...) raises ValueError; that must be a usage diagnostic, not a traceback. + from dokimi.cli import main + + scorecard = tmp_path / "sc.json" + scorecard.write_text(json.dumps(_scorecard()), encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard), "--out", out]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + assert not (tmp_path / "skill-evidence.json").exists() + + +def test_n_and_delta_are_the_paired_tests_not_the_arms_own_counts(): + scorecard = _scorecard() + row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] + row["n"], row["delta_vs_baseline"] = 10, 20.0 # the arm alone; 3 pairs behind the test + row["significance"] = {**_significance(4.0, significant=True), "n": 3} + + skill = {r["arm"]: r for r in build_skill_evidence(scorecard)["results"]}["skill:s"] + assert (skill["n"], skill["delta"]) == (3, 4.0) + + +def test_the_recorded_baseline_wins_over_an_arm_that_has_no_pairs(): + scorecard = _scorecard() + per_arm = scorecard["cells"][0]["summary"]["per_arm"] + # Every answer of this arm failed: no delta, no significance, like a baseline row; listed first. + scorecard["cells"][0]["summary"]["per_arm"] = {"broken": {"avg_aggregate": None, "delta_vs_baseline": None, + "n": 0}, **per_arm} + + arms = {r["arm"] for r in build_skill_evidence(scorecard)["results"]} + assert arms == {"broken", "skill:s", "placebo"} + + del scorecard["cells"][0]["baseline_arm"] # older cell: two candidates, so the cell is skipped + evidence = build_skill_evidence(scorecard) + assert evidence["results"] == [] + assert any("no single baseline arm" in w for w in evidence["warnings"]) + + +def test_subjects_evidence_refuses_to_overwrite_the_scorecard_or_publish_nothing(tmp_path: Path, monkeypatch): + from dokimi.cli import main + + scorecard = tmp_path / "sc.json" + save_artifact("subject_scorecard", _scorecard(), scorecard) + before = scorecard.read_bytes() + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard), "--out", str(scorecard)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 and scorecard.read_bytes() == before + + empty = _scorecard() + empty["cells"] = [] + save_artifact("subject_scorecard", empty, scorecard) + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 and not (tmp_path / "skill-evidence.json").exists() + + +def test_the_rollup_cells_carry_the_baseline_arm(tmp_path: Path): + from dokimi.eval.matrix_rollup import build_matrix_rollup + + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", {"baseline_arm": "none"})]) + assert rollup["cells"][0]["baseline_arm"] == "none" + + +def test_a_significance_flag_that_is_not_a_boolean_gives_no_direction(): + scorecard = _scorecard() + row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] + row["significance"] = {**_significance(12.0, significant=True), "significant_adjusted": "false"} + + by_arm = {r["arm"]: r for r in build_skill_evidence(scorecard)["results"]} + assert by_arm["skill:s"]["verdict"] == "inconclusive" + + +def test_the_scorecard_cells_carry_the_baseline_arm_of_the_rollup(tmp_path: Path): + from dokimi.subjects import build_subject_scorecard, load_subject + + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["oneTBB"]\nquestions = ["questions.json"]\n', encoding="utf-8") + rollup = {"cells": [{"matrix_cell": "c1", "artifact": "c1.json", "model": "m", "provider": "p", + "harness": "arms-runner", "baseline_arm": "none", "plugin_set": "none", + "summary": {}, "cost_summary": {}}], "plugin_deltas": [], "warnings": []} + scorecard = build_subject_scorecard( + load_subject(tmp_path / "subject.toml"), + awareness_runs=[{"product": "oneTBB", "questions": "questions.json", "matrix_rollup": "m.json", + "rollup": rollup}], + out_dir=tmp_path) + assert scorecard["cells"][0]["baseline_arm"] == "none"