From 099c2b80bdded7180a7a958738a2f9395574f1ad Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:26:54 -0700 Subject: [PATCH 1/9] feat(subjects): distribution_digest, the digest a well-known skill index carries subject_scorecard.v2 gains subject.distribution_digest: sha256 of the SKILL.md bytes for a file ref or a directory holding only SKILL.md, the value a .well-known/agent-skills index.json entry of type skill-md carries (agentskills PR #254). An archive's bytes depend on how it was packed, so a skill with resources gets null, as does a non-local ref. ref_digest is unchanged. Signed-off-by: Nikolay Petrov --- ...031_120000_napetrov_distribution-digest.md | 8 +++ .../dokimi/schemas/subject_scorecard.v2.json | 6 +++ dokimi/dokimi/subjects/scorecard.py | 28 +++++++++++ dokimi/tests/test_subjects_digest.py | 50 +++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 dokimi/changelog.d/20261031_120000_napetrov_distribution-digest.md create mode 100644 dokimi/tests/test_subjects_digest.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_distribution-digest.md b/dokimi/changelog.d/20261031_120000_napetrov_distribution-digest.md new file mode 100644 index 000000000..e2538d4fe --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_distribution-digest.md @@ -0,0 +1,8 @@ + + +### Added + +- **`subject.distribution_digest` in the subject scorecard** — the `sha256:` digest a `.well-known/agent-skills/index.json` entry (agentskills PR #254) would carry for the subject: the bytes of `SKILL.md` for a file ref, or for a directory whose only file is `SKILL.md`. A skill with resources is served as an archive, whose bytes a checkout cannot predict, so it gets `null`, and so does a ref that is not a local path. A client can match a published result to the skill it installed by this field. `ref_digest` is unchanged: a content digest of the tree, which no index carries. diff --git a/dokimi/dokimi/schemas/subject_scorecard.v2.json b/dokimi/dokimi/schemas/subject_scorecard.v2.json index 3c3d85899..af9b200be 100644 --- a/dokimi/dokimi/schemas/subject_scorecard.v2.json +++ b/dokimi/dokimi/schemas/subject_scorecard.v2.json @@ -45,6 +45,12 @@ "null" ] }, + "distribution_digest": { + "type": [ + "string", + "null" + ] + }, "members": { "type": "array" } diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 4e0493541..ebb7abda5 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -76,11 +76,13 @@ def build_subject_scorecard( "kind": descriptor.subject.kind, "ref": descriptor.subject.ref, "ref_digest": _digest_ref(descriptor.subject.ref, base=Path.cwd()), + "distribution_digest": distribution_digest(descriptor.subject.ref, base=Path.cwd()), "members": [ { "kind": member.kind, "ref": member.ref, "ref_digest": _digest_ref(member.ref, base=Path.cwd()), + "distribution_digest": distribution_digest(member.ref, base=Path.cwd()), } for member in descriptor.subject.members ], @@ -138,6 +140,32 @@ def _digest_values(values: tuple[str, ...]) -> str: return "sha256:" + h.hexdigest() +def distribution_digest(ref: str | None, *, base: Path) -> str | None: + """The digest a ``.well-known/agent-skills`` index would publish for *ref*, or None. + + That index (agentskills PR #254) hashes the bytes it serves: one ``SKILL.md`` for a + ``skill-md`` entry, the archive for an ``archive`` entry. Only the first can be derived + from a checkout: a ``SKILL.md`` file ref is hashed as is, and a directory whose only file is + ``SKILL.md`` is hashed as that file. A directory with resources is served as an archive, + whose bytes depend on how it was packed, so it gets None rather than a digest no index + carries. So does a ref that is not a local path. ``ref_digest`` stays a content digest + of the tree. + """ + if not ref: + return None + path = Path(ref.removeprefix("local:")) + if not path.is_absolute(): + path = base / path + if path.is_dir(): + # Any other entry makes it an archive; one stray file is enough, so stop at the second. + if [p.name for _, p in zip(range(2), path.iterdir(), strict=False)] != ["SKILL.md"]: + return None + path = path / "SKILL.md" + if path.name != "SKILL.md" or not path.is_file(): + return None + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + def _digest_ref(ref: str | None, *, base: Path) -> str | None: if not ref: return None diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py new file mode 100644 index 000000000..e29923b9c --- /dev/null +++ b/dokimi/tests/test_subjects_digest.py @@ -0,0 +1,50 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""subject.distribution_digest: the digest a .well-known/agent-skills index carries.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from dokimi.subjects.scorecard import distribution_digest + + +def test_distribution_digest_matches_a_well_known_skill_md_entry(tmp_path: Path): + # agentskills PR #254: a skill-md index entry carries sha256 of the SKILL.md bytes it serves. + skill = tmp_path / "skill" + skill.mkdir() + (skill / "SKILL.md").write_bytes(b"---\nname: s\n---\nbody\n") + expected = "sha256:" + hashlib.sha256(b"---\nname: s\n---\nbody\n").hexdigest() + + assert distribution_digest("skill", base=tmp_path) == expected + assert distribution_digest("local:skill/SKILL.md", base=tmp_path) == expected + + # With resources the skill is served as an archive, whose bytes a checkout cannot predict. + (skill / "ref.md").write_text("more", encoding="utf-8") + assert distribution_digest("skill", base=tmp_path) is None + assert distribution_digest("skill/SKILL.md", base=tmp_path) == expected + # A ref that is not a local path has no served bytes to hash. + assert distribution_digest("context7:/org/lib", base=tmp_path) is None + assert distribution_digest(None, base=tmp_path) is None + # Only SKILL.md is served as a skill-md entry; another file has no index digest. + (tmp_path / "notes.md").write_text("x", encoding="utf-8") + assert distribution_digest("notes.md", base=tmp_path) is None + + +def test_the_scorecard_carries_the_distribution_digest(tmp_path: Path, monkeypatch): + from dokimi.artifacts import validate_artifact + from dokimi.subjects import build_subject_scorecard, load_subject + + (tmp_path / "skill").mkdir() + (tmp_path / "skill" / "SKILL.md").write_bytes(b"body") + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "skill"\nref = "skill"\n\n' + '[suite]\nproducts = ["p"]\nquestions = ["q.json"]\n', encoding="utf-8") + monkeypatch.chdir(tmp_path) + + scorecard = build_subject_scorecard(load_subject(tmp_path / "subject.toml"), awareness_runs=[], out_dir=tmp_path) + + assert scorecard["subject"]["distribution_digest"] == "sha256:" + hashlib.sha256(b"body").hexdigest() + validate_artifact("subject_scorecard", scorecard) From 005c296813e409a2ec150d98896e833ec7e6f8de Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:31:33 -0700 Subject: [PATCH 2/9] feat(subjects): subjects evidence renders a scorecard as skill_evidence.v1 A free leaf that reads a saved subject_scorecard and writes the document a publisher puts next to a skill: one result per (cell, non-baseline arm) with model, harness, effort, n, delta, CI, corrected p-value and a verdict, keyed by subject.distribution_digest so a client can match it to the .well-known/agent-skills index entry it installed from. Values the scorecard does not hold stay null. Signed-off-by: Nikolay Petrov --- ...20261031_130000_napetrov_skill-evidence.md | 9 + dokimi/docs/reference/cli-interface.json | 57 ++++ dokimi/dokimi/artifacts.py | 1 + dokimi/dokimi/cli_routes.py | 1 + dokimi/dokimi/commands/subjects.py | 32 +++ dokimi/dokimi/eval/matrix_rollup.py | 1 + dokimi/dokimi/runs/kinds.py | 2 + dokimi/dokimi/schemas/matrix_rollup.v1.json | 6 + dokimi/dokimi/schemas/skill_evidence.v1.json | 249 ++++++++++++++++++ dokimi/dokimi/subjects/evidence.py | 118 +++++++++ dokimi/dokimi/subjects/scorecard.py | 1 + .../fixtures/artifacts/skill_evidence.v1.json | 75 ++++++ dokimi/tests/test_runs_scan.py | 1 + dokimi/tests/test_runs_show.py | 1 + dokimi/tests/test_subjects_evidence.py | 160 +++++++++++ 15 files changed, 714 insertions(+) create mode 100644 dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md create mode 100644 dokimi/dokimi/schemas/skill_evidence.v1.json create mode 100644 dokimi/dokimi/subjects/evidence.py create mode 100644 dokimi/tests/fixtures/artifacts/skill_evidence.v1.json create mode 100644 dokimi/tests/test_subjects_evidence.py diff --git a/dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md b/dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md new file mode 100644 index 000000000..a8b29f3c1 --- /dev/null +++ b/dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md @@ -0,0 +1,9 @@ + + +### Added + +- **`subjects evidence SCORECARD`: a subject scorecard as a `skill_evidence.v1` document** — the file a publisher puts next to a skill so a client can read what was measured: one result per (matrix cell, non-baseline arm) with model, provider, harness, effort, and the paired test of the arm against the cell's recorded baseline arm on `avg_aggregate` (`n` pairs, delta, confidence interval, corrected p-value) and a verdict (`improved` or `regressed` only on a delta that is significant after correction, `inconclusive` otherwise). It is keyed by `subject.digest`, the scorecard's `distribution_digest`, which is what a `.well-known/agent-skills` index entry carries for the skill. Free: it reads the saved scorecard and makes no model call. Every value comes from the scorecard; one it does not hold, such as `effort` before runs record it, is `null`. Writes `skill-evidence.json` next to the scorecard unless `--out` names a file; exits 2 when the scorecard has nothing to publish or `--out` is the scorecard itself. The schema is experimental until the agentskills evidence proposal settles. +- **Matrix rollup and subject scorecard cells record `baseline_arm`**, so a reader no longer has to guess the baseline from the arm that has no delta (an arm whose every answer failed has none either). diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..c9a74b207 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -12896,6 +12896,7 @@ "'repo_environment'", "'repo_readiness'", "'result'", + "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", @@ -13016,6 +13017,62 @@ "path": "dokimi subjects", "subcommand_required": true, "subcommands": { + "evidence": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "scorecard", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--out", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi subjects evidence", + "subcommand_required": null, + "subcommands": {} + }, "list": { "allow_abbrev": true, "params": [ diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index 0088ff5eb..a233eaa45 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -46,6 +46,7 @@ "matrix_rollup": "matrix_rollup.v1", "plugin_delta": "plugin_delta.v1", "subject_scorecard": "subject_scorecard.v2", + "skill_evidence": "skill_evidence.v1", "novelty": "novelty.v1", "task_runs": "task_runs.v1", "explore_runs": "explore_runs.v1", diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 47008bcc9..8205775ba 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -99,6 +99,7 @@ class Tombstone: Leaf(("subjects", "list"), FREE), Leaf(("subjects", "show"), FREE), Leaf(("subjects", "run"), PAID), + Leaf(("subjects", "evidence"), FREE), Leaf(("tasks", "list"), FREE), Leaf(("tasks", "inspect"), FREE), Leaf(("tasks", "harnesses"), FREE), diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index 731b5fdd7..6c97db706 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -166,6 +166,29 @@ def cmd_subjects_run(args: argparse.Namespace) -> None: print(f"OK Saved subject report: {out_md}") +def cmd_subjects_evidence(args: argparse.Namespace) -> None: + """Render a saved subject scorecard as a skill_evidence.v1 document; no model calls.""" + from dokimi.artifacts import ArtifactValidationError, load_artifact, save_artifact + from dokimi.subjects.evidence import build_skill_evidence + + try: + scorecard = load_artifact("subject_scorecard", Path(args.scorecard)) + except (OSError, ValueError, ArtifactValidationError) as exc: + usage_exit(f"cannot read subject scorecard {args.scorecard}: {exc}") + evidence = build_skill_evidence(scorecard, source=str(args.scorecard)) + if not evidence["results"]: + usage_exit(f"{args.scorecard} holds no awareness arm with a baseline to compare against: " + "nothing to publish") + out = Path(args.out) if args.out else Path(args.scorecard).with_name("skill-evidence.json") + if out.resolve() == Path(args.scorecard).resolve(): + usage_exit("--out names the scorecard itself; pick another file") + with exit_on_unwritable(out): + save_artifact("skill_evidence", evidence, out) + for warning in evidence["warnings"]: + print(f"WARN {warning}", file=sys.stderr) + print(f"OK Saved skill evidence: {out} ({len(evidence['results'])} results)") + + def register(sub, positive_int) -> None: """Add the `subjects` subcommand group.""" subjects_p = sub.add_parser( @@ -264,6 +287,15 @@ def register(sub, positive_int) -> None: ) run_p.set_defaults(func=cmd_subjects_run) + evidence_p = subjects_sub.add_parser( + "evidence", + help="Render a subject scorecard as a skill_evidence.v1 document (no model calls)", + ) + evidence_p.add_argument("scorecard", help="subject_scorecard JSON written by `subjects run`") + evidence_p.add_argument("--out", default=None, + help="Output file (default: skill-evidence.json next to the scorecard)") + evidence_p.set_defaults(func=cmd_subjects_evidence) + def _run_plan(descriptor, args: argparse.Namespace) -> tuple[Path, str, list[str]]: """The output dir, baseline arm name and per-product questions, or exit 2 when the descriptor cannot run. diff --git a/dokimi/dokimi/eval/matrix_rollup.py b/dokimi/dokimi/eval/matrix_rollup.py index 4eb4fef1c..38ff41ce5 100644 --- a/dokimi/dokimi/eval/matrix_rollup.py +++ b/dokimi/dokimi/eval/matrix_rollup.py @@ -30,6 +30,7 @@ def build_matrix_rollup( "harness": data.get("harness"), "plugin_set": data.get("plugin_set", "none"), "plugin_set_id": data.get("plugin_set_id"), + "baseline_arm": data.get("baseline_arm"), "summary": data.get("summary", {}), "cost_summary": data.get("cost_summary", {}), } diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index cb8e49a39..f4422be26 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -45,6 +45,8 @@ class KindFields: "plugin_delta": KindFields(subject=("library_name",), model=("model",), counts=("arms",)), "subject_scorecard": KindFields(subject=("subject.id",), time=("generated_at",), counts=("cells",)), + "skill_evidence": KindFields(subject=("subject.id",), time=("generated_at",), + counts=("results", "warnings")), "novelty": KindFields(subject=("artifact.ref",), model=("model",), time=("generated_at",), counts=("conflicts", "warnings")), "task_runs": KindFields(model=("model",), time=("generated_at",), diff --git a/dokimi/dokimi/schemas/matrix_rollup.v1.json b/dokimi/dokimi/schemas/matrix_rollup.v1.json index 0c964b14d..1f68767c6 100644 --- a/dokimi/dokimi/schemas/matrix_rollup.v1.json +++ b/dokimi/dokimi/schemas/matrix_rollup.v1.json @@ -74,6 +74,12 @@ }, "cost_summary": { "type": "object" + }, + "baseline_arm": { + "type": [ + "string", + "null" + ] } }, "additionalProperties": true diff --git a/dokimi/dokimi/schemas/skill_evidence.v1.json b/dokimi/dokimi/schemas/skill_evidence.v1.json new file mode 100644 index 000000000..bc1554cda --- /dev/null +++ b/dokimi/dokimi/schemas/skill_evidence.v1.json @@ -0,0 +1,249 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/skill_evidence.v1.json", + "title": "Skill evidence document v1", + "description": "Experimental until the agentskills evidence proposal settles. Measured effect of a skill against a baseline, keyed by the digest a .well-known/agent-skills index carries for it. Rendered from a subject scorecard; no value is filled in that the scorecard does not hold. Every statistic in a result is the paired test's: n is the number of paired questions behind delta, ci and p_value.", + "type": "object", + "required": [ + "schema_version", + "subject", + "results" + ], + "properties": { + "schema_version": { + "const": "skill_evidence.v1" + }, + "generated_at": { + "type": "string" + }, + "producer": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "subject": { + "type": "object", + "required": [ + "digest" + ], + "properties": { + "id": { + "type": [ + "string", + "null" + ] + }, + "kind": { + "type": [ + "string", + "null" + ] + }, + "ref": { + "type": [ + "string", + "null" + ] + }, + "digest": { + "type": [ + "string", + "null" + ], + "description": "sha256 of the served SKILL.md bytes, as in the index entry; null when it cannot be derived" + }, + "content_digest": { + "type": [ + "string", + "null" + ] + } + } + }, + "eval_set": { + "type": "object", + "properties": { + "products": { + "type": "array", + "items": { + "type": "string" + } + }, + "questions": { + "type": "array", + "items": { + "type": "string" + } + }, + "digest": { + "type": [ + "string", + "null" + ] + } + } + }, + "baseline": { + "type": [ + "string", + "null" + ] + }, + "measured_at": { + "type": [ + "string", + "null" + ] + }, + "source": { + "type": "object", + "properties": { + "kind": { + "type": [ + "string", + "null" + ] + }, + "path": { + "type": [ + "string", + "null" + ] + } + } + }, + "results": { + "type": "array", + "items": { + "type": "object", + "required": [ + "arm", + "metric", + "delta", + "verdict" + ], + "properties": { + "product": { + "type": [ + "string", + "null" + ] + }, + "matrix_cell": { + "type": [ + "string", + "null" + ] + }, + "model": { + "type": [ + "string", + "null" + ], + "description": "model as configured for the cell, not a resolved provider id" + }, + "provider": { + "type": [ + "string", + "null" + ] + }, + "harness": { + "type": [ + "string", + "null" + ] + }, + "effort": { + "type": [ + "string", + "null" + ] + }, + "arm": { + "type": "string" + }, + "metric": { + "type": "string", + "description": "dokimi's mean judge aggregate score per answer (higher is better); delta is arm minus baseline" + }, + "n": { + "type": [ + "integer", + "null" + ] + }, + "delta": { + "type": [ + "number", + "null" + ] + }, + "ci": { + "type": [ + "object", + "null" + ], + "properties": { + "lo": { + "type": [ + "number", + "null" + ] + }, + "hi": { + "type": [ + "number", + "null" + ] + }, + "confidence": { + "type": [ + "number", + "null" + ] + }, + "method": { + "type": [ + "string", + "null" + ] + } + } + }, + "p_value": { + "type": [ + "number", + "null" + ] + }, + "p_correction": { + "type": [ + "string", + "null" + ] + }, + "verdict": { + "enum": [ + "improved", + "regressed", + "inconclusive" + ] + } + } + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py new file mode 100644 index 000000000..cba1f9af0 --- /dev/null +++ b/dokimi/dokimi/subjects/evidence.py @@ -0,0 +1,118 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Render a subject scorecard as a ``skill_evidence.v1`` document. + +The scorecard is dokimi's own record of a run: every cell, rollup path and warning. The evidence +document is what a publisher hands a client next to the skill: one row per (cell, arm) with the +measured effect against the baseline, keyed by the digest a ``.well-known/agent-skills`` index +carries for the skill, so a client can match it to the skill it installed. Nothing is measured +here: every number is read from the scorecard, and a value the scorecard does not hold is null, +never filled in. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import PurePath +from typing import Any + +from dokimi import __version__ + +#: The arm metric the awareness summary carries per arm (``summary.per_arm[arm]``). +METRIC = "avg_aggregate" + + +def build_skill_evidence(scorecard: dict[str, Any], *, source: str | None = None) -> dict[str, Any]: + """A ``skill_evidence.v1`` payload from a ``subject_scorecard`` payload. + + ``source`` names the scorecard file it was read from; only its file name is kept, since the + directory is the producer's disk layout and not part of the result. + """ + subject = scorecard.get("subject") or {} + suite = scorecard.get("suite") or {} + warnings = list(scorecard.get("warnings") or []) + results: list[dict[str, Any]] = [] + for cell in scorecard.get("cells") or []: + per_arm = (cell.get("summary") or {}).get("per_arm") or {} + baseline = _baseline_arm(cell, per_arm) + if baseline is None: + warnings.append(f"matrix cell {cell.get('matrix_cell')!r}: no single baseline arm, skipped") + continue + results.extend(_result(cell, arm, row) for arm, row in per_arm.items() if arm != baseline) + if subject.get("distribution_digest") is None: + warnings.append("subject has no distribution_digest (not a SKILL.md, a skill with resources, " + "or a non-local ref): a client cannot match this document by digest") + if subject.get("kind") != "skill": + warnings.append(f"subject kind is {subject.get('kind')!r}, not 'skill'") + if (scorecard.get("work") or {}).get("status") == "run": + warnings.append("task-suite (work) results are not in this document; only the awareness arms are") + return { + "schema_version": "skill_evidence.v1", + "generated_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z", + "producer": {"name": "dokimi", "version": __version__}, + "subject": { + "id": subject.get("id"), + "kind": subject.get("kind"), + "ref": subject.get("ref"), + "digest": subject.get("distribution_digest"), + "content_digest": subject.get("ref_digest"), + }, + "eval_set": { + "products": list(suite.get("products") or []), + "questions": list(suite.get("questions") or []), + "digest": suite.get("question_set_hash"), + }, + "baseline": scorecard.get("baseline"), + "measured_at": scorecard.get("generated_at"), + "source": {"kind": scorecard.get("schema_version"), "path": source and PurePath(source).name}, + "results": results, + "warnings": sorted(set(warnings)), + } + + +def _baseline_arm(cell: dict[str, Any], per_arm: dict[str, Any]) -> str | None: + """The cell's baseline arm: as recorded, else the one arm with no delta at all, else None. + + ``delta_vs_baseline`` is also None for an arm with no paired question (every answer failed), + so the fallback for a cell recorded before ``baseline_arm`` holds only when one arm qualifies. + """ + if cell.get("baseline_arm") in per_arm: + return cell["baseline_arm"] + candidates = [name for name, row in per_arm.items() + if row.get("delta_vs_baseline") is None and not row.get("significance")] + return candidates[0] if len(candidates) == 1 else None + + +def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, Any]: + # Every statistic is the paired test's, so n, delta, CI and p describe the same questions. + # The per-arm means are left out: their difference is not the delta once pairs drop out. + sig = row.get("significance") or {} + delta = sig.get("delta_mean") if sig else None + return { + "product": cell.get("product"), + "matrix_cell": cell.get("matrix_cell"), + "model": cell.get("model"), + "provider": cell.get("provider"), + "harness": cell.get("harness"), + # Absent until a run records the effort it applied; null means "not recorded", not "default". + "effort": cell.get("effort"), + "arm": arm, + "metric": METRIC, + "n": sig.get("n"), + "delta": delta, + "ci": None if sig.get("ci_lo") is None else { + "lo": sig.get("ci_lo"), "hi": sig.get("ci_hi"), + "confidence": sig.get("ci_confidence"), "method": sig.get("ci_method"), + }, + "p_value": sig.get("p_adjusted"), + "p_correction": sig.get("correction"), + "verdict": _verdict(delta, sig), + } + + +def _verdict(delta: float | None, sig: dict[str, Any]) -> str: + """``improved``/``regressed`` only on a corrected significant delta; else ``inconclusive``.""" + if delta is None or not sig.get("significant_adjusted"): + return "inconclusive" + return "improved" if delta > 0 else "regressed" if delta < 0 else "inconclusive" diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index ebb7abda5..831e41eb0 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -57,6 +57,7 @@ def build_subject_scorecard( "model": cell.get("model"), "provider": cell.get("provider"), "harness": cell.get("harness"), + "baseline_arm": cell.get("baseline_arm"), "plugin_set": cell.get("plugin_set"), "summary": cell.get("summary", {}), "cost_summary": cell.get("cost_summary", {}), diff --git a/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json b/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json new file mode 100644 index 000000000..34d1d2b53 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json @@ -0,0 +1,75 @@ +{ + "baseline": "baseline", + "eval_set": { + "digest": "sha256:7b0379b853ffc6170161dede74ec099e8819110188311444256faab6eea2d06e", + "products": [ + "oneTBB" + ], + "questions": [ + "questions.json" + ] + }, + "generated_at": "2026-10-31T10:05:00Z", + "measured_at": "2026-10-04T17:47:19.175356Z", + "producer": { + "name": "dokimi", + "version": "0.0.0" + }, + "results": [ + { + "arm": "skill:s", + "ci": { + "confidence": 0.95, + "hi": 13.5, + "lo": 10.5, + "method": "student_t" + }, + "delta": 12.0, + "effort": null, + "harness": "arms-runner", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "model": "m", + "n": 6, + "p_correction": "holm", + "p_value": 0.001, + "product": "oneTBB", + "provider": "p", + "verdict": "improved" + }, + { + "arm": "placebo", + "ci": { + "confidence": 0.95, + "hi": 2.5, + "lo": -0.5, + "method": "student_t" + }, + "delta": 1.0, + "effort": null, + "harness": "arms-runner", + "matrix_cell": "c1", + "metric": "avg_aggregate", + "model": "m", + "n": 6, + "p_correction": "holm", + "p_value": 0.4, + "product": "oneTBB", + "provider": "p", + "verdict": "inconclusive" + } + ], + "schema_version": "skill_evidence.v1", + "source": { + "kind": "subject_scorecard.v2", + "path": "subject-scorecard.json" + }, + "subject": { + "content_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "id": "s", + "kind": "skill", + "ref": "skills/s" + }, + "warnings": [] +} diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 9b5b9997d..0d2344c61 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -34,6 +34,7 @@ "matrix_rollup.v1": ({}, "oneTBB", None, "2026-10-04T17:47:16.294238Z"), "plugin_delta.v1": ({}, "oneTBB", "m", None), "subject_scorecard.v2": ({}, "docs-subject", None, "2026-10-04T17:47:19.175356Z"), + "skill_evidence.v1": ({}, "s", None, "2026-10-31T10:05:00Z"), "novelty.v1": ({}, "skill:/tmp/run/accel-tuning/SKILL.md", "answerer-m", "2026-10-04T17:47:16.841483Z"), "task_runs.v1": ({}, None, "haiku", "2026-01-01T00:00:00Z"), diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 07ad7a9f5..b81e20895 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -36,6 +36,7 @@ "repo_readiness.v1": {"files": 2, "claims": 52}, "result.v1": {"cells": 6}, "subject_scorecard.v2": {"cells": 0}, + "skill_evidence.v1": {"results": 2, "warnings": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, "task_inspect.v1": {"items": 4}, "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py new file mode 100644 index 000000000..915e4ed5e --- /dev/null +++ b/dokimi/tests/test_subjects_evidence.py @@ -0,0 +1,160 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""skill_evidence.v1: a subject scorecard rendered for publishing next to the skill.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from artifact_fixtures import artifact_fixture +from dokimi.artifacts import load_artifact, save_artifact +from dokimi.subjects.evidence import build_skill_evidence + +DIGEST = "sha256:" + "a" * 64 +FIXTURE = Path(__file__).parent / "fixtures" / "artifacts" / "subject_scorecard.v2.json" + + +def _significance(delta: float, *, significant: bool) -> dict: + return {"ci_confidence": 0.95, "ci_lo": delta - 1.5, "ci_hi": delta + 1.5, "ci_method": "student_t", + "correction": "holm", "delta_mean": delta, "n": 6, "p_adjusted": 0.001 if significant else 0.4, + "significant_adjusted": significant} + + +def _scorecard(**subject) -> dict: + per_arm = { + "baseline": {"avg_aggregate": 50.0, "delta_vs_baseline": None, "n": 6}, + "skill:s": {"avg_aggregate": 62.0, "delta_vs_baseline": 12.0, "n": 6, + "significance": _significance(12.0, significant=True)}, + "placebo": {"avg_aggregate": 51.0, "delta_vs_baseline": 1.0, "n": 6, + "significance": _significance(1.0, significant=False)}, + } + # The committed fixture, so the base is a scorecard a producer wrote; only subject and cells change. + base = json.loads(FIXTURE.read_text(encoding="utf-8")) + base["subject"] = {"id": "s", "kind": "skill", "ref": "skills/s", "ref_digest": "sha256:" + "b" * 64, + "distribution_digest": DIGEST, "members": [], **subject} + base["cells"] = [{"product": "oneTBB", "questions": "q.json", "matrix_cell": "c1", "artifact": "c1.json", + "model": "m", "provider": "p", "harness": "arms-runner", "baseline_arm": "baseline", + "plugin_set": "none", + "summary": {"per_arm": per_arm}, "cost_summary": {}}] + return artifact_fixture("subject_scorecard", base, closed=False) + + +def test_each_non_baseline_arm_is_one_result_read_from_the_scorecard(tmp_path: Path): + evidence = build_skill_evidence(_scorecard(), source="sc.json") + + assert evidence["subject"]["digest"] == DIGEST + assert evidence["eval_set"]["digest"] == _scorecard()["suite"]["question_set_hash"] + by_arm = {row["arm"]: row for row in evidence["results"]} + assert set(by_arm) == {"skill:s", "placebo"} + skill = by_arm["skill:s"] + assert (skill["model"], skill["harness"], skill["n"]) == ("m", "arms-runner", 6) + assert (skill["n"], skill["delta"]) == (6, 12.0) + assert skill["ci"] == {"lo": 10.5, "hi": 13.5, "confidence": 0.95, "method": "student_t"} + assert (skill["p_value"], skill["p_correction"], skill["verdict"]) == (0.001, "holm", "improved") + assert by_arm["placebo"]["verdict"] == "inconclusive" + # Not recorded by the run, so not filled in. + assert skill["effort"] is None + save_artifact("skill_evidence", evidence, tmp_path / "e.json") # validates against the schema + + +@pytest.mark.parametrize("delta, significant, verdict", [ + (-3.0, True, "regressed"), (-3.0, False, "inconclusive"), (0.0, True, "inconclusive"), +]) +def test_only_a_corrected_significant_delta_gets_a_direction(delta, significant, verdict): + scorecard = _scorecard() + row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] + row["delta_vs_baseline"] = delta + row["significance"] = _significance(delta, significant=significant) + + by_arm = {r["arm"]: r for r in build_skill_evidence(scorecard)["results"]} + assert by_arm["skill:s"]["verdict"] == verdict + + +def test_a_subject_with_no_distribution_digest_is_flagged_not_guessed(): + evidence = build_skill_evidence(_scorecard(distribution_digest=None)) + + assert evidence["subject"]["digest"] is None + assert any("distribution_digest" in w for w in evidence["warnings"]) + + +def test_subjects_evidence_writes_the_document_next_to_the_scorecard(tmp_path: Path, monkeypatch): + from dokimi.cli import main + + scorecard = tmp_path / "subject-scorecard.json" + save_artifact("subject_scorecard", _scorecard(), scorecard) + + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard)]) + main() + + out = load_artifact("skill_evidence", tmp_path / "skill-evidence.json") + assert out["source"] == {"kind": "subject_scorecard.v2", "path": "subject-scorecard.json"} + assert len(out["results"]) == 2 + + +def test_subjects_evidence_refuses_a_file_that_is_not_a_scorecard(tmp_path: Path, monkeypatch): + from dokimi.cli import main + + bad = tmp_path / "x.json" + bad.write_text(json.dumps({"schema_version": "subject_scorecard.v2"}), encoding="utf-8") + with pytest.raises(SystemExit) as exc: + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad)]) + main() + assert exc.value.code == 2 + + +def test_n_and_delta_are_the_paired_tests_not_the_arms_own_counts(): + scorecard = _scorecard() + row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] + row["n"], row["delta_vs_baseline"] = 10, 20.0 # the arm alone; 3 pairs behind the test + row["significance"] = {**_significance(4.0, significant=True), "n": 3} + + skill = {r["arm"]: r for r in build_skill_evidence(scorecard)["results"]}["skill:s"] + assert (skill["n"], skill["delta"]) == (3, 4.0) + + +def test_the_recorded_baseline_wins_over_an_arm_that_has_no_pairs(): + scorecard = _scorecard() + per_arm = scorecard["cells"][0]["summary"]["per_arm"] + # Every answer of this arm failed: no delta, no significance, like a baseline row; listed first. + scorecard["cells"][0]["summary"]["per_arm"] = {"broken": {"avg_aggregate": None, "delta_vs_baseline": None, + "n": 0}, **per_arm} + + arms = {r["arm"] for r in build_skill_evidence(scorecard)["results"]} + assert arms == {"broken", "skill:s", "placebo"} + + del scorecard["cells"][0]["baseline_arm"] # older cell: two candidates, so the cell is skipped + evidence = build_skill_evidence(scorecard) + assert evidence["results"] == [] + assert any("no single baseline arm" in w for w in evidence["warnings"]) + + +def test_subjects_evidence_refuses_to_overwrite_the_scorecard_or_publish_nothing(tmp_path: Path, monkeypatch): + from dokimi.cli import main + + scorecard = tmp_path / "sc.json" + save_artifact("subject_scorecard", _scorecard(), scorecard) + before = scorecard.read_bytes() + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard), "--out", str(scorecard)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 and scorecard.read_bytes() == before + + empty = _scorecard() + empty["cells"] = [] + save_artifact("subject_scorecard", empty, scorecard) + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 and not (tmp_path / "skill-evidence.json").exists() + + +def test_the_rollup_and_scorecard_cells_carry_the_baseline_arm(tmp_path: Path): + from dokimi.eval.matrix_rollup import build_matrix_rollup + + rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, + cell_artifacts=[("c1", tmp_path / "c1.json", {"baseline_arm": "none"})]) + assert rollup["cells"][0]["baseline_arm"] == "none" From d0f17047fd3a1405a8debfaff4ae78610c373f5c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 00:39:56 -0700 Subject: [PATCH 3/9] feat(subjects): skill evidence results name the judge model, null until a run records it Signed-off-by: Nikolay Petrov --- dokimi/dokimi/schemas/skill_evidence.v1.json | 6 ++++++ dokimi/dokimi/subjects/evidence.py | 1 + dokimi/tests/fixtures/artifacts/skill_evidence.v1.json | 2 ++ dokimi/tests/test_subjects_evidence.py | 2 +- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/schemas/skill_evidence.v1.json b/dokimi/dokimi/schemas/skill_evidence.v1.json index bc1554cda..c52232b84 100644 --- a/dokimi/dokimi/schemas/skill_evidence.v1.json +++ b/dokimi/dokimi/schemas/skill_evidence.v1.json @@ -166,6 +166,12 @@ "null" ] }, + "judge_model": { + "type": [ + "string", + "null" + ] + }, "arm": { "type": "string" }, diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index cba1f9af0..34a28ba55 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -97,6 +97,7 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, An "harness": cell.get("harness"), # Absent until a run records the effort it applied; null means "not recorded", not "default". "effort": cell.get("effort"), + "judge_model": cell.get("judge_model"), "arm": arm, "metric": METRIC, "n": sig.get("n"), diff --git a/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json b/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json index 34d1d2b53..1c82de3ee 100644 --- a/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json +++ b/dokimi/tests/fixtures/artifacts/skill_evidence.v1.json @@ -26,6 +26,7 @@ }, "delta": 12.0, "effort": null, + "judge_model": null, "harness": "arms-runner", "matrix_cell": "c1", "metric": "avg_aggregate", @@ -47,6 +48,7 @@ }, "delta": 1.0, "effort": null, + "judge_model": null, "harness": "arms-runner", "matrix_cell": "c1", "metric": "avg_aggregate", diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 915e4ed5e..877ed24c4 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -57,7 +57,7 @@ def test_each_non_baseline_arm_is_one_result_read_from_the_scorecard(tmp_path: P assert (skill["p_value"], skill["p_correction"], skill["verdict"]) == (0.001, "holm", "improved") assert by_arm["placebo"]["verdict"] == "inconclusive" # Not recorded by the run, so not filled in. - assert skill["effort"] is None + assert skill["effort"] is None and skill["judge_model"] is None save_artifact("skill_evidence", evidence, tmp_path / "e.json") # validates against the schema From 778a1ac59dfb5fb88fd6a5e55d3b6ab1d2e205d6 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 13:39:58 +0000 Subject: [PATCH 4/9] Exit 2 when a subject scorecard has wrong-typed nested values Signed-off-by: Anton Volkov --- dokimi/dokimi/commands/subjects.py | 5 ++++- dokimi/tests/test_subjects_evidence.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index 6c97db706..71c8f3566 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -175,7 +175,10 @@ def cmd_subjects_evidence(args: argparse.Namespace) -> None: scorecard = load_artifact("subject_scorecard", Path(args.scorecard)) except (OSError, ValueError, ArtifactValidationError) as exc: usage_exit(f"cannot read subject scorecard {args.scorecard}: {exc}") - evidence = build_skill_evidence(scorecard, source=str(args.scorecard)) + try: + evidence = build_skill_evidence(scorecard, source=str(args.scorecard)) + except (AttributeError, TypeError) as exc: + usage_exit(f"{args.scorecard} is not a well-formed subject scorecard: {exc}") if not evidence["results"]: usage_exit(f"{args.scorecard} holds no awareness arm with a baseline to compare against: " "nothing to publish") diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 877ed24c4..8cecdac73 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -106,6 +106,17 @@ def test_subjects_evidence_refuses_a_file_that_is_not_a_scorecard(tmp_path: Path assert exc.value.code == 2 +def test_subjects_evidence_refuses_a_scorecard_with_a_wrong_typed_cell(tmp_path: Path, monkeypatch): + from dokimi.cli import main + + bad = tmp_path / "x.json" + bad.write_text(json.dumps({**_scorecard(), "cells": [None]}), encoding="utf-8") + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + + def test_n_and_delta_are_the_paired_tests_not_the_arms_own_counts(): scorecard = _scorecard() row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] From 4eefca5b9784f7aa70e9cfd032ed1dd1983a6e4b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:36:27 -0800 Subject: [PATCH 5/9] fix(subjects): non-local refs get no digest, boolean flag gives the verdict distribution_digest looks a ref up on disk only for a local ref, so a scheme ref such as context7:/org/lib can no longer collide with a directory of that name. A verdict needs significant_adjusted to be the boolean true; the string "false" gives inconclusive. The baseline_arm of a scorecard cell and the wrong-typed-cell exits are pinned by tests. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/subjects/evidence.py | 2 +- dokimi/dokimi/subjects/scorecard.py | 10 ++++++- dokimi/tests/test_subjects_digest.py | 12 ++++++++ dokimi/tests/test_subjects_evidence.py | 39 ++++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/subjects/evidence.py b/dokimi/dokimi/subjects/evidence.py index 34a28ba55..8f76cf478 100644 --- a/dokimi/dokimi/subjects/evidence.py +++ b/dokimi/dokimi/subjects/evidence.py @@ -114,6 +114,6 @@ def _result(cell: dict[str, Any], arm: str, row: dict[str, Any]) -> dict[str, An def _verdict(delta: float | None, sig: dict[str, Any]) -> str: """``improved``/``regressed`` only on a corrected significant delta; else ``inconclusive``.""" - if delta is None or not sig.get("significant_adjusted"): + if delta is None or sig.get("significant_adjusted") is not True: # a string "false" is not a verdict return "inconclusive" return "improved" if delta > 0 else "regressed" if delta < 0 else "inconclusive" diff --git a/dokimi/dokimi/subjects/scorecard.py b/dokimi/dokimi/subjects/scorecard.py index 831e41eb0..cf00efb9d 100644 --- a/dokimi/dokimi/subjects/scorecard.py +++ b/dokimi/dokimi/subjects/scorecard.py @@ -6,6 +6,7 @@ from __future__ import annotations import hashlib +import re from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -141,6 +142,10 @@ def _digest_values(values: tuple[str, ...]) -> str: return "sha256:" + h.hexdigest() +#: A URI scheme (``context7:``, ``https:``). Two characters or more: a Windows drive letter is not a scheme. +_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]+:") + + def distribution_digest(ref: str | None, *, base: Path) -> str | None: """The digest a ``.well-known/agent-skills`` index would publish for *ref*, or None. @@ -154,7 +159,10 @@ def distribution_digest(ref: str | None, *, base: Path) -> str | None: """ if not ref: return None - path = Path(ref.removeprefix("local:")) + local = ref.removeprefix("local:") + if _SCHEME.match(local): # a non-local reference names no file here, even one that has its name + return None + path = Path(local) if not path.is_absolute(): path = base / path if path.is_dir(): diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index e29923b9c..9a36cfeb3 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -6,8 +6,11 @@ from __future__ import annotations import hashlib +import sys from pathlib import Path +import pytest + from dokimi.subjects.scorecard import distribution_digest @@ -48,3 +51,12 @@ def test_the_scorecard_carries_the_distribution_digest(tmp_path: Path, monkeypat assert scorecard["subject"]["distribution_digest"] == "sha256:" + hashlib.sha256(b"body").hexdigest() validate_artifact("subject_scorecard", scorecard) + + +@pytest.mark.skipif(sys.platform == "win32", reason="a colon is not allowed in a Windows file name") +def test_a_non_local_ref_is_not_looked_up_on_disk(tmp_path: Path): + # A directory named like the ref's scheme holds a SKILL.md; the ref is not a local path, so no digest. + (tmp_path / "context7:" / "org" / "lib").mkdir(parents=True) + (tmp_path / "context7:" / "org" / "lib" / "SKILL.md").write_bytes(b"body") + assert distribution_digest("context7:/org/lib", base=tmp_path) is None + assert distribution_digest("local:context7:/org/lib", base=tmp_path) is None diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 8cecdac73..f68092d5d 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -106,11 +106,18 @@ def test_subjects_evidence_refuses_a_file_that_is_not_a_scorecard(tmp_path: Path assert exc.value.code == 2 -def test_subjects_evidence_refuses_a_scorecard_with_a_wrong_typed_cell(tmp_path: Path, monkeypatch): +@pytest.mark.parametrize("cells", [ + [None], + [{"summary": []}], + [{"summary": {"per_arm": []}}], + [{"summary": {"per_arm": {"skill:s": 5}}}], + [{"summary": {"per_arm": {"skill:s": {"significance": "x"}}}}], +]) +def test_subjects_evidence_refuses_a_scorecard_with_a_wrong_typed_cell(tmp_path: Path, monkeypatch, cells): from dokimi.cli import main bad = tmp_path / "x.json" - bad.write_text(json.dumps({**_scorecard(), "cells": [None]}), encoding="utf-8") + bad.write_text(json.dumps({**_scorecard(), "cells": cells}), encoding="utf-8") monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad)]) with pytest.raises(SystemExit) as exc: main() @@ -163,9 +170,35 @@ def test_subjects_evidence_refuses_to_overwrite_the_scorecard_or_publish_nothing assert exc.value.code == 2 and not (tmp_path / "skill-evidence.json").exists() -def test_the_rollup_and_scorecard_cells_carry_the_baseline_arm(tmp_path: Path): +def test_the_rollup_cells_carry_the_baseline_arm(tmp_path: Path): from dokimi.eval.matrix_rollup import build_matrix_rollup rollup = build_matrix_rollup(library_name="l", out_dir=tmp_path, compute_plugin_deltas=False, cell_artifacts=[("c1", tmp_path / "c1.json", {"baseline_arm": "none"})]) assert rollup["cells"][0]["baseline_arm"] == "none" + + +def test_a_significance_flag_that_is_not_a_boolean_gives_no_direction(): + scorecard = _scorecard() + row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] + row["significance"] = {**_significance(12.0, significant=True), "significant_adjusted": "false"} + + by_arm = {r["arm"]: r for r in build_skill_evidence(scorecard)["results"]} + assert by_arm["skill:s"]["verdict"] == "inconclusive" + + +def test_the_scorecard_cells_carry_the_baseline_arm_of_the_rollup(tmp_path: Path): + from dokimi.subjects import build_subject_scorecard, load_subject + + (tmp_path / "subject.toml").write_text( + '[subject]\nid = "s"\nkind = "doc-source"\nref = "local:docs"\n\n' + '[suite]\nproducts = ["oneTBB"]\nquestions = ["questions.json"]\n', encoding="utf-8") + rollup = {"cells": [{"matrix_cell": "c1", "artifact": "c1.json", "model": "m", "provider": "p", + "harness": "arms-runner", "baseline_arm": "none", "plugin_set": "none", + "summary": {}, "cost_summary": {}}], "plugin_deltas": [], "warnings": []} + scorecard = build_subject_scorecard( + load_subject(tmp_path / "subject.toml"), + awareness_runs=[{"product": "oneTBB", "questions": "questions.json", "matrix_rollup": "m.json", + "rollup": rollup}], + out_dir=tmp_path) + assert scorecard["cells"][0]["baseline_arm"] == "none" From ac2b33c6d28dfd78a6df6720c5aef13e7a197b30 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:05:34 -0800 Subject: [PATCH 6/9] fix(subjects): a value the evidence schema rejects exits 2, not a traceback A scorecard whose nested significance n is a string passes the scorecard schema and renders, then save_artifact raised ArtifactValidationError past both the output guard and the CLI handler. It now goes through usage_exit like the other input diagnostics. Regression test: red before. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/subjects.py | 7 +++++-- dokimi/tests/test_subjects_evidence.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index 71c8f3566..4a5c50a8b 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -185,8 +185,11 @@ def cmd_subjects_evidence(args: argparse.Namespace) -> None: out = Path(args.out) if args.out else Path(args.scorecard).with_name("skill-evidence.json") if out.resolve() == Path(args.scorecard).resolve(): usage_exit("--out names the scorecard itself; pick another file") - with exit_on_unwritable(out): - save_artifact("skill_evidence", evidence, out) + try: + with exit_on_unwritable(out): + save_artifact("skill_evidence", evidence, out) + except ArtifactValidationError as exc: + usage_exit(f"{args.scorecard} holds a value the evidence schema rejects: {exc}") for warning in evidence["warnings"]: print(f"WARN {warning}", file=sys.stderr) print(f"OK Saved skill evidence: {out} ({len(evidence['results'])} results)") diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index f68092d5d..316c3cbe5 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -124,6 +124,22 @@ def test_subjects_evidence_refuses_a_scorecard_with_a_wrong_typed_cell(tmp_path: assert exc.value.code == 2 +def test_subjects_evidence_refuses_a_nested_statistic_the_evidence_schema_rejects(tmp_path: Path, monkeypatch): + # The scorecard schema accepts a string n and rendering copies it through; only the evidence + # schema rejects it at save time. That must be an input diagnostic (exit 2), not a traceback. + from dokimi.cli import main + + card = _scorecard() + card["cells"][0]["summary"]["per_arm"]["skill:s"]["significance"]["n"] = "6" + bad = tmp_path / "x.json" + bad.write_text(json.dumps(card), encoding="utf-8") + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad)]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + assert not (tmp_path / "skill-evidence.json").exists() + + def test_n_and_delta_are_the_paired_tests_not_the_arms_own_counts(): scorecard = _scorecard() row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] From 842720f3342fbb1c64e99eeb0c709d095ce28f64 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:15:19 -0800 Subject: [PATCH 7/9] fix(subjects): evidence --out refuses a symlink loop and a .rejected sibling of the scorecard Path.resolve() raises RuntimeError on a symlink loop before Python 3.13, which escaped as a traceback. save_artifact also keeps and removes ".rejected", so an --out whose sibling is the scorecard would delete or overwrite it. Both now exit 2; tests red before the change. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/subjects.py | 11 +++++++++- dokimi/tests/test_subjects_evidence.py | 29 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index 4a5c50a8b..0705713ae 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -183,8 +183,17 @@ def cmd_subjects_evidence(args: argparse.Namespace) -> None: usage_exit(f"{args.scorecard} holds no awareness arm with a baseline to compare against: " "nothing to publish") out = Path(args.out) if args.out else Path(args.scorecard).with_name("skill-evidence.json") - if out.resolve() == Path(args.scorecard).resolve(): + try: + out_real = out.resolve() + # save_artifact keeps and removes ".rejected"; that sibling must not be the scorecard. + stale_real = out.with_name(out.name + ".rejected").resolve() + card_real = Path(args.scorecard).resolve() + except (OSError, RuntimeError) as exc: # a symlink loop raises RuntimeError before Python 3.13 + usage_exit(f"cannot resolve --out {out}: {exc}") + if out_real == card_real: usage_exit("--out names the scorecard itself; pick another file") + if stale_real == card_real: + usage_exit(f"{out}.rejected is the scorecard; a save to {out} would delete or overwrite it; pick another --out") try: with exit_on_unwritable(out): save_artifact("skill_evidence", evidence, out) diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index 316c3cbe5..b5a059ea1 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import sys from pathlib import Path import pytest @@ -140,6 +141,34 @@ def test_subjects_evidence_refuses_a_nested_statistic_the_evidence_schema_reject assert not (tmp_path / "skill-evidence.json").exists() +def test_subjects_evidence_refuses_an_out_whose_rejected_sibling_is_the_scorecard(tmp_path: Path, monkeypatch): + # save_artifact removes ".rejected" on a valid save; here that would delete the scorecard. + from dokimi.cli import main + + card = tmp_path / "sc.json.rejected" + card.write_text(json.dumps(_scorecard()), encoding="utf-8") + before = card.read_bytes() + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(card), "--out", str(tmp_path / "sc.json")]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + assert card.read_bytes() == before + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlinks need privileges on Windows") +def test_subjects_evidence_refuses_an_out_that_is_a_symlink_loop(tmp_path: Path, monkeypatch): + # resolve() raises RuntimeError on a symlink loop before Python 3.13; that must be a diagnostic. + from dokimi.cli import main + + bad = tmp_path / "sc.json" + bad.write_text(json.dumps(_scorecard()), encoding="utf-8") + (tmp_path / "loop").symlink_to(tmp_path / "loop") + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(bad), "--out", str(tmp_path / "loop")]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + + def test_n_and_delta_are_the_paired_tests_not_the_arms_own_counts(): scorecard = _scorecard() row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"] From 541b9283f6330655c046fb257f150a62a888ce2b Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 20:51:47 +0000 Subject: [PATCH 8/9] Pin that a one-letter drive letter is not a ref scheme Signed-off-by: Anton Volkov --- dokimi/tests/test_subjects_digest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_subjects_digest.py b/dokimi/tests/test_subjects_digest.py index 9a36cfeb3..ce2dd207d 100644 --- a/dokimi/tests/test_subjects_digest.py +++ b/dokimi/tests/test_subjects_digest.py @@ -11,7 +11,7 @@ import pytest -from dokimi.subjects.scorecard import distribution_digest +from dokimi.subjects.scorecard import _SCHEME, distribution_digest def test_distribution_digest_matches_a_well_known_skill_md_entry(tmp_path: Path): @@ -60,3 +60,8 @@ def test_a_non_local_ref_is_not_looked_up_on_disk(tmp_path: Path): (tmp_path / "context7:" / "org" / "lib" / "SKILL.md").write_bytes(b"body") assert distribution_digest("context7:/org/lib", base=tmp_path) is None assert distribution_digest("local:context7:/org/lib", base=tmp_path) is None + + +def test_a_one_letter_drive_letter_is_not_a_scheme(): + assert _SCHEME.match("C:/skills/x") is None + assert _SCHEME.match("ab:/skills/x") is not None From e5fa6df5971e114213c353ba852058a68a8bd3cc Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 13:03:09 -0800 Subject: [PATCH 9/9] fix(subjects): evidence --out "." or "/" exits 2, not a traceback with_name() on a path with no file name raises ValueError, which the --out guard did not catch. It now exits 2 with a usage diagnostic. Tests: --out . and --out / refused (red before). The drive-letter scheme pin is Anton Volkov's test on the branch (541b9283). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/subjects.py | 3 ++- dokimi/tests/test_subjects_evidence.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/subjects.py b/dokimi/dokimi/commands/subjects.py index 0705713ae..7f4081104 100644 --- a/dokimi/dokimi/commands/subjects.py +++ b/dokimi/dokimi/commands/subjects.py @@ -188,7 +188,8 @@ def cmd_subjects_evidence(args: argparse.Namespace) -> None: # save_artifact keeps and removes ".rejected"; that sibling must not be the scorecard. stale_real = out.with_name(out.name + ".rejected").resolve() card_real = Path(args.scorecard).resolve() - except (OSError, RuntimeError) as exc: # a symlink loop raises RuntimeError before Python 3.13 + # A symlink loop raises RuntimeError before Python 3.13; "." or "/" has no name, so with_name raises ValueError. + except (OSError, RuntimeError, ValueError) as exc: usage_exit(f"cannot resolve --out {out}: {exc}") if out_real == card_real: usage_exit("--out names the scorecard itself; pick another file") diff --git a/dokimi/tests/test_subjects_evidence.py b/dokimi/tests/test_subjects_evidence.py index b5a059ea1..15f0c2625 100644 --- a/dokimi/tests/test_subjects_evidence.py +++ b/dokimi/tests/test_subjects_evidence.py @@ -169,6 +169,22 @@ def test_subjects_evidence_refuses_an_out_that_is_a_symlink_loop(tmp_path: Path, assert exc.value.code == 2 +@pytest.mark.skipif(sys.platform == "win32", reason="the root is not a bare name there") +@pytest.mark.parametrize("out", [".", "/"]) +def test_subjects_evidence_refuses_an_out_with_no_file_name(tmp_path: Path, monkeypatch, out): + # Path("." or "/").with_name(...) raises ValueError; that must be a usage diagnostic, not a traceback. + from dokimi.cli import main + + scorecard = tmp_path / "sc.json" + scorecard.write_text(json.dumps(_scorecard()), encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("sys.argv", ["dokimi", "subjects", "evidence", str(scorecard), "--out", out]) + with pytest.raises(SystemExit) as exc: + main() + assert exc.value.code == 2 + assert not (tmp_path / "skill-evidence.json").exists() + + def test_n_and_delta_are_the_paired_tests_not_the_arms_own_counts(): scorecard = _scorecard() row = scorecard["cells"][0]["summary"]["per_arm"]["skill:s"]