feat(subjects): subjects evidence renders a scorecard as skill_evidence.v1

dokimi/changelog.d/20261031_130000_napetrov_skill-evidence.md

@@ -0,0 +1,9 @@
+<!--
+Copyright (C) 2026 Intel Corporation
+SPDX-License-Identifier: Apache-2.0
+-->
+
+### Added
+
+- **`subjects evidence SCORECARD`: a subject scorecard as a `skill_evidence.v1` document** — the file a publisher puts next to a skill so a client can read what was measured: one result per (matrix cell, non-baseline arm) with model, provider, harness, effort, and the paired test of the arm against the cell's recorded baseline arm on `avg_aggregate` (`n` pairs, delta, confidence interval, corrected p-value) and a verdict (`improved` or `regressed` only on a delta that is significant after correction, `inconclusive` otherwise). It is keyed by `subject.digest`, the scorecard's `distribution_digest`, which is what a `.well-known/agent-skills` index entry carries for the skill. Free: it reads the saved scorecard and makes no model call. Every value comes from the scorecard; one it does not hold, such as `effort` before runs record it, is `null`. Writes `skill-evidence.json` next to the scorecard unless `--out` names a file; exits 2 when the scorecard has nothing to publish or `--out` is the scorecard itself. The schema is experimental until the agentskills evidence proposal settles.
+- **Matrix rollup and subject scorecard cells record `baseline_arm`**, so a reader no longer has to guess the baseline from the arm that has no delta (an arm whose every answer failed has none either).

dokimi/docs/reference/cli-interface.json

@@ -13050,6 +13050,7 @@
                 "'repo_environment'",
                 "'repo_readiness'",
                 "'result'",
+                "'skill_evidence'",
                 "'subject_scorecard'",
                 "'task_inspect'",
                 "'task_runs'",
@@ -13170,6 +13171,62 @@
       "path": "dokimi subjects",
       "subcommand_required": true,
       "subcommands": {
+        "evidence": {
+          "allow_abbrev": true,
+          "params": [
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "scorecard",
+              "kind": "argument",
+              "nargs": null,
+              "position": 0,
+              "required": true,
+              "type": null
+            },
+            {
+              "action": "_HelpAction",
+              "choices": null,
+              "const": null,
+              "default": "==SUPPRESS==",
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": true,
+              "key": "--help/-h",
+              "kind": "option",
+              "nargs": 0,
+              "position": null,
+              "required": false,
+              "type": null
+            },
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "--out",
+              "kind": "option",
+              "nargs": null,
+              "position": null,
+              "required": false,
+              "type": null
+            }
+          ],
+          "path": "dokimi subjects evidence",
+          "subcommand_required": null,
+          "subcommands": {}
+        },
         "list": {
           "allow_abbrev": true,
           "params": [

dokimi/dokimi/artifacts.py

@@ -47,6 +47,7 @@
     "matrix_rollup": "matrix_rollup.v1",
     "plugin_delta": "plugin_delta.v1",
     "subject_scorecard": "subject_scorecard.v2",
+    "skill_evidence": "skill_evidence.v1",
     "novelty": "novelty.v1",
     "task_runs": "task_runs.v1",
     "explore_runs": "explore_runs.v1",

dokimi/dokimi/cli_routes.py

@@ -99,6 +99,7 @@ class Tombstone:
     Leaf(("subjects", "list"), FREE),
     Leaf(("subjects", "show"), FREE),
     Leaf(("subjects", "run"), PAID),
+    Leaf(("subjects", "evidence"), FREE),
     Leaf(("tasks", "list"), FREE),
     Leaf(("tasks", "inspect"), FREE),
     Leaf(("tasks", "harnesses"), FREE),

dokimi/dokimi/commands/subjects.py

@@ -166,6 +166,45 @@ def cmd_subjects_run(args: argparse.Namespace) -> None:
     print(f"OK Saved subject report:    {out_md}")
 
 
+def cmd_subjects_evidence(args: argparse.Namespace) -> None:
+    """Render a saved subject scorecard as a skill_evidence.v1 document; no model calls."""
+    from dokimi.artifacts import ArtifactValidationError, load_artifact, save_artifact
+    from dokimi.subjects.evidence import build_skill_evidence
+
+    try:
+        scorecard = load_artifact("subject_scorecard", Path(args.scorecard))
+    except (OSError, ValueError, ArtifactValidationError) as exc:
+        usage_exit(f"cannot read subject scorecard {args.scorecard}: {exc}")
+    try:
+        evidence = build_skill_evidence(scorecard, source=str(args.scorecard))
+    except (AttributeError, TypeError) as exc:
+        usage_exit(f"{args.scorecard} is not a well-formed subject scorecard: {exc}")
+    if not evidence["results"]:
+        usage_exit(f"{args.scorecard} holds no awareness arm with a baseline to compare against: "
+                   "nothing to publish")
+    out = Path(args.out) if args.out else Path(args.scorecard).with_name("skill-evidence.json")
+    try:
+        out_real = out.resolve()
+        # save_artifact keeps and removes "<out>.rejected"; that sibling must not be the scorecard.
+        stale_real = out.with_name(out.name + ".rejected").resolve()
+        card_real = Path(args.scorecard).resolve()
+    # A symlink loop raises RuntimeError before Python 3.13; "." or "/" has no name, so with_name raises ValueError.
+    except (OSError, RuntimeError, ValueError) as exc:
+        usage_exit(f"cannot resolve --out {out}: {exc}")
+    if out_real == card_real:
+        usage_exit("--out names the scorecard itself; pick another file")
+    if stale_real == card_real:
+        usage_exit(f"{out}.rejected is the scorecard; a save to {out} would delete or overwrite it; pick another --out")
+    try:
+        with exit_on_unwritable(out):
+            save_artifact("skill_evidence", evidence, out)
+    except ArtifactValidationError as exc:
+        usage_exit(f"{args.scorecard} holds a value the evidence schema rejects: {exc}")
+    for warning in evidence["warnings"]:
+        print(f"WARN {warning}", file=sys.stderr)
+    print(f"OK Saved skill evidence: {out} ({len(evidence['results'])} results)")
+
+
 def register(sub, positive_int) -> None:
     """Add the `subjects` subcommand group."""
     subjects_p = sub.add_parser(
@@ -264,6 +303,15 @@ def register(sub, positive_int) -> None:
     )
     run_p.set_defaults(func=cmd_subjects_run)
 
+    evidence_p = subjects_sub.add_parser(
+        "evidence",
+        help="Render a subject scorecard as a skill_evidence.v1 document (no model calls)",
+    )
+    evidence_p.add_argument("scorecard", help="subject_scorecard JSON written by `subjects run`")
+    evidence_p.add_argument("--out", default=None,
+                            help="Output file (default: skill-evidence.json next to the scorecard)")
+    evidence_p.set_defaults(func=cmd_subjects_evidence)
+
 
 def _run_plan(descriptor, args: argparse.Namespace) -> tuple[Path, str, list[str]]:
     """The output dir, baseline arm name and per-product questions, or exit 2 when the descriptor cannot run.

dokimi/dokimi/eval/matrix_rollup.py

@@ -30,6 +30,7 @@ def build_matrix_rollup(
             "harness": data.get("harness"),
             "plugin_set": data.get("plugin_set", "none"),
             "plugin_set_id": data.get("plugin_set_id"),
+            "baseline_arm": data.get("baseline_arm"),
             "summary": data.get("summary", {}),
             "cost_summary": data.get("cost_summary", {}),
         }

dokimi/dokimi/runs/kinds.py

@@ -45,6 +45,8 @@ class KindFields:
     "plugin_delta": KindFields(subject=("library_name",), model=("model",), counts=("arms",)),
     "subject_scorecard": KindFields(subject=("subject.id",), time=("generated_at",),
                                     counts=("cells",)),
+    "skill_evidence": KindFields(subject=("subject.id",), time=("generated_at",),
+                                 counts=("results", "warnings")),
     "novelty": KindFields(subject=("artifact.ref",), model=("model",), time=("generated_at",),
                           counts=("conflicts", "warnings")),
     "task_runs": KindFields(model=("model",), time=("generated_at",),

dokimi/dokimi/schemas/matrix_rollup.v1.json

@@ -74,6 +74,12 @@
           },
           "cost_summary": {
             "type": "object"
+          },
+          "baseline_arm": {
+            "type": [
+              "string",
+              "null"
+            ]
           }
         },
         "additionalProperties": true