diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f13b08c9..dc4abe517 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -487,5 +487,6 @@ jobs: cd "$RUNNER_TEMP" && python "$smoke" - name: Install test deps run: python -m pip install --require-hashes -r requirements.lock.txt + # test_runs_scan.py holds the only junction test of the runs-root containment; it runs on Windows only. - name: Run a few fast tests - run: pytest -q tests/test_cli_help.py tests/test_console_encoding.py + run: pytest -q tests/test_cli_help.py tests/test_console_encoding.py tests/test_runs_scan.py tests/test_runs_show.py diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index fbaf738fb..b36efa758 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 36 + max_docs: 37 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md new file mode 100644 index 000000000..e5d054fed --- /dev/null +++ b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md @@ -0,0 +1,29 @@ + + +### Added + +- **`dokimi runs list` lists the artifacts under the runs root** — the root is + `results/` in the working directory, or `DOKIMI_RUNS_DIR`. One row per artifact: + its `schema_version`, subject, model, recorded time and path, newest first; + `--kind` (repeatable) and `--subject` (an exact match) filter the rows, and + `--json` prints them with the fields `schema_version`, `kind`, `run_id` (always + `null` until artifacts carry one), `subject`, `model`, `generated_at`, `path` + and `size_bytes`. A file is an artifact when its first 256 bytes open with + the `schema_version` of a registered kind, as `save_artifact` writes it; every other file is counted by why it + was skipped, and a file that cannot be read is listed with its reason. A + missing `results/` prints "no runs" and exits 0; a `DOKIMI_RUNS_DIR` that does + not exist or is not a directory exits 2. The command only reads. +- **`dokimi runs show PATH` shows one artifact** — `PATH` as `runs list` prints + it (relative to the root), or an absolute or working-directory path inside the + root once symlinks are resolved (neither command reads a file outside the + root). It prints the artifact's kind, its fields, the length of the lists it + holds (answers, evaluations, cells, ...) and the files next to it, and checks + it against its schema: an artifact that fails the check is shown with + "schema: invalid, N errors" and every error, and the command still exits 0. + A path outside the root, a missing file, a directory, or a file that is not a + readable artifact of a registered kind exits 2. `--json` prints the same + fields as a `runs list` row plus `schema`, `schema_valid`, `schema_errors`, + `contents` and the neighbours. diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 68bf82147..4133acaa6 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -294,6 +294,9 @@ code. - [quickstart.md](quickstart.md) — end-to-end LLM evaluation pipeline: personas → questions → answers → judge → report. Includes troubleshooting and a cost estimate. +- [runs.md](runs.md) — what the runs have written: `cli.py runs list` lists + every artifact under `results/` (or `DOKIMI_RUNS_DIR`) and counts what it + skipped, `cli.py runs show` checks one against its schema. No LLM call. - [user-flow.md](user-flow.md) — the canonical end-to-end path: pick a target, preflight, run, then pin the question set so runs stay comparable. - [benchmarking-and-comparison.md](benchmarking-and-comparison.md) — how to diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 39584b12d..192d0e3e1 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -70,7 +70,7 @@ points up: | Layer | Modules | | --- | --- | | front end | `cli`, `commands`, `__main__` (the argparse front end; no layer below imports it) | -| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` | +| tracks | `runner`, `runs`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` | | evaluation | `eval`, `personas`, `questions`, `treatments`, `plugins` | | components | `ingest`, `metrics`, `gate`, `mcp`, `skills`, `agent_profiles` | | foundation | `__init__` (the `dokimi` package itself), `_version`, `artifacts`, `records`, `llm`, `process`, `container_policy`, `caching`, `config`, `registry`, `intents`, `defaults`, `console`, `utils`, `metrics/usage.py` | diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index 8f97a7ba5..b2dd290ed 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -218,7 +218,8 @@ at such a checkout. Everything the pipeline produces — answers, eval scores, benchmark snapshots, reports — stays under the git-ignored `answers/`, `eval/`, `baselines/`, `reports/` and `results/` directories. See -[`data/README.md`](../data/README.md). +[`data/README.md`](../data/README.md). `python cli.py runs list` lists the +artifacts under `results/` ([runs.md](runs.md)). ## Cost estimate diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 24c817a3d..c9eb437d1 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -12820,6 +12820,179 @@ "subcommand_required": null, "subcommands": {} }, + "runs": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi runs", + "subcommand_required": true, + "subcommands": { + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": [ + "'answers'", + "'arms'", + "'cli_routes'", + "'eval'", + "'explore_runs'", + "'guidance_reach'", + "'matrix_rollup'", + "'novelty'", + "'plan'", + "'plugin_delta'", + "'questions'", + "'repo_budget'", + "'repo_environment'", + "'repo_readiness'", + "'result'", + "'subject_scorecard'", + "'task_inspect'", + "'task_runs'", + "'tool_preference'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--kind", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--subject", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi runs list", + "subcommand_required": null, + "subcommands": {} + }, + "show": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "path", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi runs show", + "subcommand_required": null, + "subcommands": {} + } + } + }, "subjects": { "allow_abbrev": true, "params": [ diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md new file mode 100644 index 000000000..140f941dd --- /dev/null +++ b/dokimi/docs/runs.md @@ -0,0 +1,118 @@ +--- +depends_on: + - dokimi/dokimi/runs/** + - dokimi/dokimi/commands/runs.py +--- + + +# The runs root: `runs list` and `runs show` + +Most commands that run an evaluation write their artifacts under `results/` by +default (a few write elsewhere, listed below). `runs list` shows what is there +and `runs show` opens one artifact. Both only read files: no model call, no +network access, nothing written. + +The runs root is `results/` in the working directory. Set `DOKIMI_RUNS_DIR` to +read another directory. + +## `runs list` + +```text +$ python cli.py runs list +SCHEMA SUBJECT MODEL TIME (UTC) PATH +arms.v1 onetbb gpt-6.1-sol 2026-10-29 14:02 arms/onetbb.json +plugin_delta.v1 onetbb gpt-6.1-sol 2026-10-28 10:11* arms/plugin_delta.json +eval.v1 - claude-sonnet-5-5 2026-10-28 10:05 onetbb_r7/eval/onetbb.json +answers.v1 onetbb gpt-6.1-sol 2026-10-28 09:40 onetbb_r7/answers/onetbb.json +novelty.v1 skill:skills/onetbb-tuning claude-sonnet-5-5 2026-10-27 18:30 novelty/skill_skills_onetbb-tuning/novelty.json + +5 artifacts under results/ (* the file's mtime: the artifact records no time) +skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_version (pairwise.v0: 1), 1 not .json +``` + +- **Each row is one artifact file.** A file is an artifact when its first 256 + bytes open with the `schema_version` of a registered kind: `save_artifact` + has written it as the first key since + [#27](https://github.com/intel/dokimi/pull/27), well inside that. Superseded + versions such as `questions.v1` count too. Files that open with anything + else are skipped without being read in full, so a large file of another kind + costs nothing. +- **Subject, model and time** come from the artifact's own fields, and which + fields depends on the kind: `product` of a questions set, `library_name` of + an arms run, `artifact.ref` of a novelty run, `repo.id` of a reach run, + `judge_model` and `evaluated_at` of an eval. An artifact that records no + subject shows `-` (an eval, a task run, a questions set from `questions + generate`, which records the model but not the product); nothing is guessed + from the path. Rows are newest first. When an artifact records no time, its + file's mtime is used for the order and shown with `*`. +- **Nothing is dropped silently.** The summary counts the files skipped, by + reason: JSON that does not open with a `schema_version` (other output, and + artifacts saved before #27, which did not put it first), JSON of an unknown + `schema_version` (named), `*.tmp` and `*.rejected` files left by + `save_artifact`, other files, entries that are not regular files (a FIFO), + symlinked directories and Windows junctions, which are not walked, and + symlinks to files outside the root, which are not read. A file or directory + that cannot be read is + listed as unreadable, as is a file that opens with a registered + `schema_version` but does not parse (a file of no registered kind is never + parsed). The text summary names the first one's reason; `--json` lists all. +- `--kind KIND` (repeatable, for example `--kind arms --kind eval`) and + `--subject NAME` (an exact, case-sensitive match) filter the rows. +- `--json` prints `root`, `artifacts`, `total` (before filtering), `skipped`, + `unknown_versions` and `unreadable`. Each artifact has `schema_version`, + `kind`, `run_id`, `subject`, `model`, `generated_at` (the time the artifact + records, `evaluated_at` for an eval; `null` when none), `path` (relative to + the root) and `size_bytes`. + +**What it does not show yet.** No artifact records a `run_id` yet +([#277](https://github.com/intel/dokimi/issues/277)), so rows are not grouped +into runs and `run_id` is `null`. Files outside the root are not listed: + +- `repo run` writes its batches to `repo-runs/`, and its files carry no + `schema_version`; +- `answers generate` and `eval score` write to `answers/` and `eval/` unless + `--run-id` is given, and `eval panel-score` writes to `eval/`; +- `questions generate` and `questions refine` write to `questions/`, + `repo reach` to `reach-out/`, `tools preference` to `tool_preference.json` + in the working directory; +- an output flag can point anywhere. + +## `runs show` + +```text +$ python cli.py runs show arms/onetbb.json +path: arms/onetbb.json +kind: arms (arms.v1) +schema: valid +subject: onetbb +model: gpt-6.1-sol +time: 2026-10-29T14:02:11Z +run_id: - (no artifact records one yet) +size: 3,503 bytes +contents: arms: 4, answers: 0 +next to it: onetbb.md, plugin_delta.json (plugin_delta.v1) +``` + +The path is the one `runs list` prints, relative to the root. An absolute +path, or one from the working directory (`results/arms/onetbb.json`), works +too if it lies inside the root once symlinks are resolved: neither command +reads anything outside the root. The artifact is checked against the schema of +the version it declares. An artifact that fails the check prints +`schema: invalid, N errors` and each error, and the command still exits 0: +`runs show` reads, it does not gate. `contents` gives the length of the lists +the kind holds, and `next to it` the first 20 entries of the artifact's +directory. `--json` prints the fields of a `runs list` row plus `root`, +`schema`, `schema_valid`, `schema_errors`, `contents`, `siblings`, +`siblings_total` and `siblings_error`. + +## Exit codes + +| Case | Exit | +| --- | --- | +| Listed or shown, including an artifact that fails its schema | 0 | +| `results/` does not exist (`runs list` prints "no runs") | 0 | +| `DOKIMI_RUNS_DIR` does not exist, or the root is not a directory | 2 | +| `runs show`: no root, a path outside the root (or a symlink that leads out of it), a missing file, a directory, or a file that is not a readable artifact of a registered kind | 2 | diff --git a/dokimi/dokimi/__init__.py b/dokimi/dokimi/__init__.py index eabdfaaaa..c8cccae2e 100644 --- a/dokimi/dokimi/__init__.py +++ b/dokimi/dokimi/__init__.py @@ -25,6 +25,8 @@ directories; [`task_formats`][dokimi.task_formats] names the format of other eval-task files and directories (detection only). - [`context`][dokimi.context] reads what a run's context window held. +- [`runs`][dokimi.runs] lists what the runs wrote: the artifacts under the runs + root (`results/`, or `DOKIMI_RUNS_DIR`). - Content novelty: [`novelty`][dokimi.novelty]. Repositories as subjects: [`repos`][dokimi.repos]. Subject scorecards, one per run: [`subjects`][dokimi.subjects]. diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 5551c9279..225112db7 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -33,6 +33,7 @@ repo, report, run, + runs, subjects, tasks, tools, @@ -143,6 +144,7 @@ def positive_int(value: str) -> int: repo.register(sub, positive_int) # repo (repositories as subjects) tools.register(sub, positive_int) # tools (tool-choice probes) context.register(sub, positive_int) # context (context window of a run) + runs.register(sub, positive_int) # runs (the artifacts under the runs root) backends.register(sub, positive_int) # backends (CLI providers) data.register(sub, positive_int) # data (schema; M2 adds export, import and models) diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index 6d1b464a2..47008bcc9 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -75,6 +75,7 @@ class Tombstone: Root("ingest", "Analysis and data"), Root("tools", "Analysis and data"), Root("context", "Analysis and data"), + Root("runs", "Analysis and data"), Root("backends", "Analysis and data"), Root("data", "Analysis and data"), ) @@ -154,6 +155,8 @@ class Tombstone: Leaf(("tools", "preference"), PAID), Leaf(("context", "trace"), FREE), Leaf(("context", "consistency"), FREE), + Leaf(("runs", "list"), FREE), + Leaf(("runs", "show"), FREE), Leaf(("backends",), FREE), Leaf(("data", "schema", "list"), FREE), Leaf(("data", "schema", "show"), FREE), diff --git a/dokimi/dokimi/commands/runs.py b/dokimi/dokimi/commands/runs.py new file mode 100644 index 000000000..97ab0927e --- /dev/null +++ b/dokimi/dokimi/commands/runs.py @@ -0,0 +1,182 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""runs subcommand group: the artifacts under the runs root (#278). Reads only; never spends.""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone + +from dokimi.artifacts import SCHEMA_VERSIONS +from dokimi.runs.scan import ( + SKIP_REASONS, + Artifact, + ScanResult, + plain, + runs_root, + scan, + select, +) +from dokimi.runs.show import Shown, show +from dokimi.utils import InputError + + +def _untimed(artifact: Artifact) -> bool: + return artifact.recorded_at is None and not artifact.generated_at + + +def _when(artifact: Artifact) -> str: + """The recorded time in UTC to the minute; the file's mtime, marked ``*``, when it has none.""" + stamp = artifact.recorded_at + if stamp: + return stamp.strftime("%Y-%m-%d %H:%M") + if artifact.generated_at: # not ISO 8601, or out of range: shown as written + return artifact.generated_at + return _mtime(artifact) + "*" + + +def _mtime(artifact: Artifact) -> str: + """The file's mtime in UTC to the minute; ``@`` when it is past what a date holds.""" + try: + return datetime.fromtimestamp(artifact.mtime, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + except (ValueError, OverflowError, OSError): # tmpfs keeps mtimes past the year 9999 + return f"@{artifact.mtime:.0f}" + + +def _count(n: int, noun: str) -> str: + return f"{n} {noun}" if n == 1 else f"{n} {noun}s" + + +def _table(artifacts: list[Artifact]) -> list[str]: + rows: list[tuple[str, ...]] = [("SCHEMA", "SUBJECT", "MODEL", "TIME (UTC)", "PATH")] + rows += [tuple(plain(cell) for cell in (a.schema_version, a.subject or "-", a.model or "-", + _when(a), a.path.as_posix())) + for a in artifacts] + widths = [max(len(row[i]) for row in rows) for i in range(4)] + return [" ".join(row[i].ljust(w) for i, w in enumerate(widths)) + " " + row[4] + for row in rows] + + +def _summary(result: ScanResult, shown: list[Artifact], root: str) -> list[str]: + total = _count(len(result.artifacts), "artifact") + lines = [f"{len(shown)} of {total} under {root} match" if len(shown) != len(result.artifacts) + else f"{total} under {root}"] + if any(_untimed(a) for a in shown): + lines[0] += " (* the file's mtime: the artifact records no time)" + labels = {**SKIP_REASONS, "unknown_schema_version": SKIP_REASONS["unknown_schema_version"] + + " (" + ", ".join(f"{plain(v)}: {n}" for v, n in sorted(result.unknown_versions.items())) + + ")"} + skipped = [f"{result.skipped[reason]} {label}" for reason, label in labels.items() + if result.skipped[reason]] + if skipped: + lines.append("skipped: " + ", ".join(skipped)) + if result.unreadable: + first = result.unreadable[0] + lines.append(f"unreadable: {len(result.unreadable)} " + f"(first: {plain(first.path.as_posix())}: {plain(first.reason)})") + return lines + + +def cmd_runs_list(args: argparse.Namespace) -> None: + """One row per artifact under the runs root, newest first, and the count of what was skipped.""" + root = runs_root() + exists = root.exists() + result = scan(root.path) if exists else ScanResult(root=root.path) + shown = select(result.artifacts, args.kind, args.subject) + if args.json: + print(json.dumps({ + "root": root.path.as_posix(), "artifacts": [a.to_json() for a in shown], + "total": len(result.artifacts), "skipped": dict(sorted(result.skipped.items())), + "unknown_versions": dict(sorted(result.unknown_versions.items())), + "unreadable": [u.to_json() for u in result.unreadable], + }, indent=2)) + return + shown_root = plain(root.path.as_posix().rstrip("/") + "/") + if not exists: + print(f"no runs under {shown_root} (it does not exist; DOKIMI_RUNS_DIR names another root)") + return + if shown: + print("\n".join(_table(shown))) + print() + print("\n".join(_summary(result, shown, shown_root))) + + +def _show_json(shown: Shown, root: str) -> dict: + return {**shown.artifact.to_json(), "root": root, "schema": shown.schema, + "schema_valid": not shown.errors, "schema_errors": shown.errors, + "contents": shown.contents, + "siblings": [{"name": s.name, "schema_version": s.schema_version} + for s in shown.siblings], + "siblings_total": shown.siblings_total, "siblings_error": shown.siblings_error} + + +def _show_text(shown: Shown) -> list[str]: + a = shown.artifact + errors = _count(len(shown.errors), "error") + when = a.generated_at or f"- ({_mtime(a)} UTC, the file's mtime)" + beside = [s.name + (f" ({s.schema_version})" if s.schema_version else "") + for s in shown.siblings] + if shown.siblings_total > len(shown.siblings): + beside.append(f"and {shown.siblings_total - len(shown.siblings)} more") + rows = [ + ("path", a.path.as_posix()), + ("kind", f"{a.kind} ({a.schema_version})"), + ("schema", f"invalid, {errors}" if shown.errors else "valid"), + ("subject", a.subject or "-"), + ("model", a.model or "-"), + ("time", when), + ("run_id", "- (no artifact records one yet)"), + ("size", f"{a.size_bytes:,} bytes"), + ("contents", ", ".join(f"{key}: {n}" for key, n in shown.contents.items()) or "-"), + ("next to it", shown.siblings_error or ", ".join(beside) or "-"), + ] + lines = [f"{label + ':':<12}{plain(value)}" for label, value in rows] + lines[3:3] = [f" - {plain(e)}" for e in shown.errors] + return lines + + +def cmd_runs_show(args: argparse.Namespace) -> None: + """One artifact: its kind, schema check, fields, contents and neighbours. A failed schema + check is printed, and the exit code stays 0.""" + root = runs_root() + if not root.exists(): + raise InputError(f"no runs root: {plain(root.path.as_posix())}/ does not exist " + f"(DOKIMI_RUNS_DIR names another root)") + shown = show(root.path, args.path) + if args.json: + print(json.dumps(_show_json(shown, root.path.as_posix()), indent=2)) + else: + print("\n".join(_show_text(shown))) + + +def register(sub: argparse._SubParsersAction, positive_int) -> None: + runs_p = sub.add_parser( + "runs", help="The artifacts under the runs root: results/, or DOKIMI_RUNS_DIR") + runs_sub = runs_p.add_subparsers(dest="runs_cmd", required=True) + + list_p = runs_sub.add_parser( + "list", + help="One row per artifact under the runs root (kind, subject, model, time, path), " + "newest first, and the count of every file skipped. Reads only.", + ) + list_p.add_argument("--kind", action="append", choices=sorted(SCHEMA_VERSIONS), metavar="KIND", + help="Only artifacts of this kind (eval, arms, novelty, ...); repeatable") + list_p.add_argument("--subject", + help="Only artifacts whose subject is exactly this (a product, a repo id, " + "a skill ref)") + list_p.add_argument("--json", action="store_true", + help="Print the artifacts and the skip counts as JSON") + list_p.set_defaults(func=cmd_runs_list) + + show_p = runs_sub.add_parser( + "show", + help="One artifact: its kind, whether it meets its schema (a failed check is printed and " + "exits 0), subject, model, time, what it counts and the files next to it. Reads only.", + ) + show_p.add_argument("path", help="The artifact: its path as runs list prints it (relative to " + "the root), or an absolute path inside the root") + show_p.add_argument("--json", action="store_true", + help="Print the artifact's fields, schema errors and neighbours as JSON") + show_p.set_defaults(func=cmd_runs_show) diff --git a/dokimi/dokimi/runs/__init__.py b/dokimi/dokimi/runs/__init__.py new file mode 100644 index 000000000..9158e79b3 --- /dev/null +++ b/dokimi/dokimi/runs/__init__.py @@ -0,0 +1,10 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The runs root (``results/``, or ``DOKIMI_RUNS_DIR``) and the artifacts under it (#278).""" + +from __future__ import annotations + +from dokimi.runs.scan import Artifact, RunsRoot, ScanResult, Unreadable, runs_root, scan + +__all__ = ["Artifact", "RunsRoot", "ScanResult", "Unreadable", "runs_root", "scan"] diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py new file mode 100644 index 000000000..cb8e49a39 --- /dev/null +++ b/dokimi/dokimi/runs/kinds.py @@ -0,0 +1,98 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Where each artifact kind keeps its subject, model and time, and what its contents count. + +No field is shared by every kind (a questions set names its ``product``, an arms run its +``library_name``, a novelty run its ``artifact.ref``), so :data:`FIELDS` names, per kind of +:data:`dokimi.artifacts.SCHEMA_VERSIONS`, the schema paths to read. A kind without such a field +reads as empty: nothing is guessed from the file's path. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from dokimi.artifacts import SCHEMA_VERSIONS, SUPERSEDED_VERSIONS + + +@dataclass(frozen=True) +class KindFields: + """Schema paths of one kind; each tuple is tried in order, the first that holds a value wins. + + A path is ``.``-joined keys; a key ending in ``[]`` steps into every item of a list, and the + distinct values found are all kept (several subjects of a plan, mixed products of answers). + ``counts`` are top-level lists whose length ``runs show`` prints. + """ + + subject: tuple[str, ...] = () + model: tuple[str, ...] = () + time: tuple[str, ...] = () + counts: tuple[str, ...] = () + + +FIELDS: dict[str, KindFields] = { + "questions": KindFields(subject=("product", "library"), model=("model",), + time=("generated_at",), counts=("questions",)), + "answers": KindFields(subject=("answers[].library_name",), model=("model",), + time=("generated_at",), counts=("answers",)), + "eval": KindFields(model=("judge_model",), time=("evaluated_at",), counts=("evaluations",)), + "arms": KindFields(subject=("library_name",), model=("model",), time=("generated_at",), + counts=("arms", "answers")), + "matrix_rollup": KindFields(subject=("library_name",), time=("generated_at",), + counts=("cells",)), + "plugin_delta": KindFields(subject=("library_name",), model=("model",), counts=("arms",)), + "subject_scorecard": KindFields(subject=("subject.id",), time=("generated_at",), + counts=("cells",)), + "novelty": KindFields(subject=("artifact.ref",), model=("model",), time=("generated_at",), + counts=("conflicts", "warnings")), + "task_runs": KindFields(model=("model",), time=("generated_at",), + counts=("tasks", "harnesses", "results")), + "explore_runs": KindFields(model=("model",), time=("generated_at",), + counts=("tasks", "arms", "results")), + "guidance_reach": KindFields(subject=("repo.id",), model=("model",), time=("generated_at",), + counts=("files", "scenarios", "records")), + "repo_readiness": KindFields(subject=("repo.id",), time=("generated_at",), + counts=("files", "claims")), + "tool_preference": KindFields(model=("model",), time=("generated_at",), + counts=("items", "results")), + "repo_environment": KindFields(subject=("repo.tree",), time=("generated_at",), + counts=("instructions", "findings")), + "plan": KindFields(subject=("subjects[].id",), model=("models[]",), time=("generated_at",), + counts=("cells",)), + "result": KindFields(time=("generated_at",), counts=("cells",)), + "repo_budget": KindFields(subject=("tree",), counts=("entries",)), + "task_inspect": KindFields(subject=("root",), counts=("items",)), + "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), +} + +#: Every schema_version an artifact may carry (current and superseded) -> its kind. +VERSION_KINDS: dict[str, str] = { + **{v: kind for kind, versions in SUPERSEDED_VERSIONS.items() for v in versions}, + **{v: kind for kind, v in SCHEMA_VERSIONS.items()}, +} + + +def _values(node: Any, keys: list[str]) -> list[str]: + if not keys: + return [node] if isinstance(node, str) and node else [] + key, rest = keys[0], keys[1:] + if key.endswith("[]"): + items = node.get(key[:-2]) if isinstance(node, dict) else None + return [v for item in items for v in _values(item, rest)] if isinstance(items, list) else [] + return _values(node.get(key), rest) if isinstance(node, dict) else [] + + +def read_field(data: dict, paths: tuple[str, ...]) -> tuple[str, ...]: + """The distinct non-empty strings at the first of ``paths`` that holds any, in order.""" + for path in paths: + found = tuple(dict.fromkeys(_values(data, path.split(".")))) + if found: + return found + return () + + +def contents(kind: str, data: dict) -> dict[str, int]: + """The length of each of the kind's ``counts`` lists that ``data`` holds.""" + return {key: len(data[key]) for key in FIELDS[kind].counts if isinstance(data.get(key), list)} diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py new file mode 100644 index 000000000..2426babdf --- /dev/null +++ b/dokimi/dokimi/runs/scan.py @@ -0,0 +1,297 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The runs root and the artifacts under it, each told apart by its first bytes. + +The root is ``results/`` (relative to the working directory, where most commands write by +default) unless ``DOKIMI_RUNS_DIR`` names another. :func:`scan` walks it once: a file is an +artifact when it opens with a ``schema_version`` of a registered kind, the way +:func:`dokimi.artifacts.save_artifact` writes it, so a large file of another kind is never +parsed just to be skipped. Everything else is counted by why it was skipped, never dropped +silently. Nothing here prints or exits. +""" + +from __future__ import annotations + +import json +import os +import re +import stat +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Mapping + +from dokimi.runs.kinds import FIELDS, VERSION_KINDS, read_field +from dokimi.utils import InputError + +ENV_VAR = "DOKIMI_RUNS_DIR" +DEFAULT_ROOT = Path("results") +#: ``save_artifact`` writes ``{"schema_version": ".vN", ...`` with ``indent=2``. +HEAD_BYTES = 256 +_HEAD = re.compile(rb'\A\s*\{\s*"schema_version"\s*:\s*"([^"\\\x00-\x1f]{1,100})"') + + +@dataclass(frozen=True) +class RunsRoot: + """The directory ``runs`` reads, and whether ``DOKIMI_RUNS_DIR`` chose it.""" + + path: Path + from_env: bool + + def exists(self) -> bool: + """True when the root is a directory; False when the default root is not there yet. + + Raises :class:`~dokimi.utils.InputError` when ``DOKIMI_RUNS_DIR`` names a missing path, + when the root is something other than a directory, or when it cannot be looked up. + """ + named = plain(f"{ENV_VAR}={self.path}" if self.from_env else str(self.path)) + try: # Python 3.11-3.13 raise PermissionError here for a parent that cannot be searched + if self.path.is_dir(): + return True + found = self.path.exists() + except OSError as exc: + raise InputError(f"runs root {named}: {read_error_reason(exc)}") from exc + if found: + raise InputError(f"runs root {named} is not a directory") + if self.from_env: + raise InputError(f"runs root {named} does not exist") + return False + + +def runs_root(environ: Mapping[str, str] | None = None) -> RunsRoot: + """``DOKIMI_RUNS_DIR`` when set and not empty, else ``results/``.""" + value = (os.environ if environ is None else environ).get(ENV_VAR, "") + return RunsRoot(Path(value), True) if value else RunsRoot(DEFAULT_ROOT, False) + + +@dataclass(frozen=True) +class Artifact: + """One artifact under the root: its kind and what it says it is about.""" + + path: Path # relative to the root + schema_version: str + subjects: tuple[str, ...] + models: tuple[str, ...] + generated_at: str | None + size_bytes: int + mtime: float + + @property + def kind(self) -> str: + return VERSION_KINDS[self.schema_version] + + @property + def subject(self) -> str | None: + return ", ".join(self.subjects) or None + + @property + def model(self) -> str | None: + return ", ".join(self.models) or None + + @property + def recorded_at(self) -> datetime | None: + """``generated_at`` in UTC (read as UTC if it names no zone); ``None`` if absent, not ISO + 8601 or out of range.""" + try: + stamp = datetime.fromisoformat(self.generated_at or "") + return (stamp if stamp.tzinfo else stamp.replace(tzinfo=timezone.utc)).astimezone( + timezone.utc) + except (ValueError, OverflowError): + return None + + def to_json(self) -> dict: + """The ``--json`` row. ``run_id`` is ``None``: no artifact records one yet (#277).""" + return {"schema_version": self.schema_version, "kind": self.kind, "run_id": None, + "subject": self.subject, "model": self.model, "generated_at": self.generated_at, + "path": self.path.as_posix(), "size_bytes": self.size_bytes} + + +@dataclass(frozen=True) +class Unreadable: + """A file or directory under the root that could not be read; ``schema_version`` if known.""" + + path: Path + reason: str + schema_version: str | None = None + + def to_json(self) -> dict: + return {"path": self.path.as_posix(), "schema_version": self.schema_version, + "reason": self.reason} + + +#: Why a file was skipped (the ``--json`` key) -> how ``runs list`` names it. +SKIP_REASONS = { + "no_leading_schema_version": "JSON with no leading schema_version", + "unknown_schema_version": "JSON of an unknown schema_version", + "temporary": "*.tmp (a save in progress or interrupted)", + "rejected": "*.rejected (payloads save_artifact refused)", + "not_json": "not .json", + "not_a_file": "not a regular file (a FIFO, socket or device)", + "linked_directory": "symlinked directories and junctions (not walked)", + "linked_outside": "symlinks to files outside the root (not read)", +} + + +def inside(root: Path | str, path: Path | str) -> bool: + """True when ``path`` is ``root`` or under it, as spelled (resolve both first for the target).""" + base, target = (os.path.normcase(os.path.abspath(p)) for p in (root, path)) + try: + return os.path.commonpath([base, target]) == base + except ValueError: # another drive on Windows + return False + + +@dataclass +class ScanResult: + """What :func:`scan` found: the artifacts, newest first, and every file it skipped.""" + + root: Path + artifacts: list[Artifact] = field(default_factory=list) + skipped: Counter[str] = field(default_factory=Counter) + unknown_versions: Counter[str] = field(default_factory=Counter) + unreadable: list[Unreadable] = field(default_factory=list) + + +def plain(text: str) -> str: + """``text`` with control and separator characters escaped (``\\n``, ``\\x1b``, ``\\u2028``), + so a file name or a field cannot split a printed line or reach the terminal as an escape.""" + return "".join(c if c.isprintable() else c.encode("unicode_escape").decode("ascii") + for c in text) + + +def head_version(path: Path) -> str | None: + """The ``schema_version`` a file opens with, read from its first bytes; ``None`` if none.""" + with open(path, "rb") as fh: + match = _HEAD.match(fh.read(HEAD_BYTES)) + return match.group(1).decode("utf-8", errors="replace") if match else None + + +def load_document(path: Path, version: str) -> dict: + """The JSON object at ``path``, whose first bytes declared ``version``. + + Raises ``OSError`` when it cannot be read and ``ValueError`` when it is not the JSON object + its first bytes promised. + """ + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("schema_version") != version: + raise ValueError(f"the document does not hold the schema_version {version} it opens with") + return data + + +def read_artifact(root: Path, path: Path, version: str, data: dict | None = None) -> Artifact: + """The artifact at ``path`` (under ``root``) of a known ``version``; ``data`` if already loaded. + + Raises as :func:`load_document` does. + """ + stat = path.stat() + data = load_document(path, version) if data is None else data + fields = FIELDS[VERSION_KINDS[version]] + times = read_field(data, fields.time) + return Artifact(path=path.relative_to(root), schema_version=version, + subjects=read_field(data, fields.subject), + models=read_field(data, fields.model), + generated_at=times[0] if times else None, + size_bytes=stat.st_size, mtime=stat.st_mtime) + + +def _skip_reason(name: str) -> str | None: + lower = name.lower() + if lower.endswith(".tmp"): + return "temporary" + if lower.endswith(".rejected"): + return "rejected" + return None if lower.endswith(".json") else "not_json" + + +def read_error_reason(exc: BaseException) -> str: + """Why a file could not be read, in one line.""" + if isinstance(exc, OSError) and exc.strerror: + return f"cannot read: {exc.strerror}" + if isinstance(exc, UnicodeDecodeError): + return "not UTF-8" + if isinstance(exc, json.JSONDecodeError): + return f"not valid JSON: {exc.msg} at line {exc.lineno}" + return str(exc) + + +def _scan_file(result: ScanResult, path: Path, real_root: str) -> None: + rel = path.relative_to(result.root) + skip = _skip_reason(path.name) + if skip: + result.skipped[skip] += 1 + return + try: + if path.is_symlink() and not inside(real_root, os.path.realpath(path)): + result.skipped["linked_outside"] += 1 + return + regular = stat.S_ISREG(path.stat().st_mode) # opening a FIFO would block + version = head_version(path) if regular else None + except OSError as exc: + result.unreadable.append(Unreadable(rel, read_error_reason(exc))) + return + if not regular: + result.skipped["not_a_file"] += 1 + elif version is None: + result.skipped["no_leading_schema_version"] += 1 + elif version not in VERSION_KINDS: + result.skipped["unknown_schema_version"] += 1 + result.unknown_versions[version] += 1 + else: + try: + result.artifacts.append(read_artifact(result.root, path, version)) + except (OSError, ValueError, RecursionError) as exc: + result.unreadable.append(Unreadable(rel, read_error_reason(exc), version)) + + +#: ``st_reparse_tag`` of a Windows directory junction (``stat.IO_REPARSE_TAG_MOUNT_POINT``). +_JUNCTION_TAG = 0xA0000003 + + +def _is_link(path: Path) -> bool: + """A symlink, or a Windows junction (os.walk descends into junctions on Python 3.11-3.13).""" + try: + if path.is_symlink(): + return True + return getattr(os.lstat(path), "st_reparse_tag", 0) == _JUNCTION_TAG + except OSError: # a directory listed but not searchable: os.walk reports it when it enters + return False + + +def newest_first(artifacts: list[Artifact]) -> list[Artifact]: + """By the time the artifact records (its file's mtime when it records none), then path.""" + def when(artifact: Artifact) -> float: + stamp = artifact.recorded_at + return stamp.timestamp() if stamp else artifact.mtime + + return sorted(artifacts, key=lambda a: (-when(a), a.path.as_posix())) + + +def scan(root: Path) -> ScanResult: + """Every artifact under ``root`` (a directory), newest first. Symlinked directories are not + walked, and a file symlink is read only when it resolves inside the root.""" + result = ScanResult(root=root) + real_root = os.path.realpath(root) + + def unlisted(exc: OSError) -> None: + where = Path(exc.filename).relative_to(root) if exc.filename else Path(".") + result.unreadable.append(Unreadable(where, f"cannot list: {exc.strerror or exc}")) + + for directory, dirnames, filenames in os.walk(root, onerror=unlisted): + base = Path(directory) + linked = {d for d in dirnames if _is_link(base / d)} + result.skipped["linked_directory"] += len(linked) + dirnames[:] = sorted(d for d in dirnames if d not in linked) + for name in sorted(filenames): + _scan_file(result, base / name, real_root) + result.artifacts = newest_first(result.artifacts) + result.skipped = +result.skipped # drop the zero counts + return result + + +def select(artifacts: list[Artifact], kinds: list[str] | None = None, + subject: str | None = None) -> list[Artifact]: + """The artifacts of one of ``kinds`` (all when none) that name exactly ``subject`` (if given).""" + return [a for a in artifacts + if (not kinds or a.kind in kinds) and (subject is None or subject in a.subjects)] diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py new file mode 100644 index 000000000..b407c51cc --- /dev/null +++ b/dokimi/dokimi/runs/show.py @@ -0,0 +1,137 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""One artifact under the runs root: its kind, whether it meets its schema, what it holds and +what lies next to it. A failed schema is reported, not raised: showing is not a gate.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dokimi.artifacts import schema_errors, schema_version_for +from dokimi.runs.kinds import VERSION_KINDS, contents +from dokimi.runs.scan import ( + Artifact, + head_version, + inside, + load_document, + plain, + read_artifact, + read_error_reason, +) +from dokimi.utils import InputError + +#: Entries of the artifact's directory that ``runs show`` names; the rest are only counted. +MAX_SIBLINGS = 20 + + +@dataclass(frozen=True) +class Sibling: + """An entry beside the artifact: a file (with its schema_version, if it opens with one) + or a directory (``name`` ends in ``/``).""" + + name: str + schema_version: str | None = None + + +@dataclass +class Shown: + """What ``runs show`` prints about one artifact.""" + + artifact: Artifact + schema: str # the schema version the document was checked against + errors: list[str] + contents: dict[str, int] + siblings: list[Sibling] = field(default_factory=list) + siblings_total: int = 0 + siblings_error: str | None = None + + +def resolve(root: Path, address: str) -> tuple[Path, Path]: + """The file ``address`` names, and the root its path is relative to (both absolute). + + ``address`` is a path relative to the root (as ``runs list`` prints it), or an absolute or + working-directory path; once symlinks are resolved it must lie inside the resolved root, so + nothing outside the root is read. Raises ``InputError`` otherwise. + """ + candidates = [root / address, Path(address)] + real_root = os.path.realpath(root) + label, shown_root = plain(address), plain(root.as_posix()) # for messages only + try: # Python 3.11-3.13 raise PermissionError here for a directory that cannot be searched + path = next((p for p in candidates if p.exists()), candidates[0]) + if not inside(real_root, os.path.realpath(path)): + where = "leads outside" if inside(root, path) else "is not under" # via a symlink + raise InputError(f"{label} {where} the runs root {shown_root}/") + if not path.exists(): + raise InputError(f"no file {label} under the runs root {shown_root}/") + if path.is_dir(): + raise InputError(f"{label} is a directory; runs show takes one artifact file") + if not path.is_file(): + raise InputError(f"{label} is not a regular file") + except OSError as exc: + raise InputError(f"{label}: {read_error_reason(exc)}") from exc + spelled, base = path.absolute(), root.absolute() # not abspath: `link/..` must stay as opened + if ".." not in spelled.parts + base.parts and inside(root, spelled): # the name list prints + return spelled, base + return Path(os.path.realpath(path)), Path(real_root) + + +def _version(path: Path, address: str) -> str: + label = plain(address) + try: + version = head_version(path) + except OSError as exc: + raise InputError(f"{label}: {read_error_reason(exc)}") from exc + if version is None: + raise InputError(f"{label} is not an artifact: it does not open with a schema_version") + if version not in VERSION_KINDS: + raise InputError(f"{label} is not an artifact of a registered kind: " + f"schema_version {plain(version)}") + return version + + +def _siblings(shown: Shown, path: Path, real_root: str) -> None: + if not inside(real_root, os.path.realpath(path.parent)): # reached through a link out and back + path = Path(os.path.realpath(path)) # list the artifact's own directory, inside the root + try: + with os.scandir(path.parent) as listing: + entries = sorted((e for e in listing if e.name != path.name), key=lambda e: e.name) + except OSError as exc: + shown.siblings_error = f"cannot list: {exc.strerror or exc}" + return + shown.siblings_total = len(entries) + for entry in entries[:MAX_SIBLINGS]: + try: + if entry.is_dir(): + shown.siblings.append(Sibling(entry.name + "/")) + continue + readable = (entry.name.lower().endswith(".json") and entry.is_file() # not a FIFO + and (not entry.is_symlink() + or inside(real_root, os.path.realpath(entry.path)))) + version = head_version(Path(entry.path)) if readable else None + except OSError: # a link loop, or an entry it may not stat: named, not read + version = None + shown.siblings.append(Sibling(entry.name, version)) + + +def show(root: Path, address: str) -> Shown: + """The artifact ``address`` names under ``root``, checked against its schema. + + Raises ``InputError`` when ``address`` is outside the root, missing, a directory, not an + artifact of a registered kind, or not readable as one. + """ + path, base = resolve(root, address) + version = _version(path, address) + kind = VERSION_KINDS[version] + try: + data = load_document(path, version) + artifact = read_artifact(base, path, version, data) + except (OSError, ValueError, RecursionError) as exc: + raise InputError(f"{plain(address)}: {read_error_reason(exc)}") from exc + schema = schema_version_for(kind, data) + shown = Shown(artifact=artifact, schema=schema, errors=schema_errors(schema, data), + contents=contents(kind, data)) + _siblings(shown, path, os.path.realpath(root)) + return shown diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 168ec6061..0f2d53b27 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -110,6 +110,7 @@ nav: - reviewing.md - external-content.md - quickstart.md + - runs.md - user-flow.md - benchmarking-and-comparison.md - adding-doc-source.md diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index be3e93b1e..f8e037fda 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -186,6 +186,7 @@ --static --stats-correction --strict +--subject # runs list (#278) adds it: the #94 §2.3 filter; no registry concept yet --subjects-root --swe-bench --target diff --git a/dokimi/tests/test_cli_routes.py b/dokimi/tests/test_cli_routes.py index 8e411ed6a..d9c63e75b 100644 --- a/dokimi/tests/test_cli_routes.py +++ b/dokimi/tests/test_cli_routes.py @@ -93,7 +93,7 @@ def test_help_lists_each_root_under_its_section(): ), **dict.fromkeys(("run", "compare", "library", "catalog", "novelty", "repo"), subjects), **dict.fromkeys(("personas", "questions", "eval"), evals), - **dict.fromkeys(("dataset", "ingest", "tools", "context", "backends", "data"), data), + **dict.fromkeys(("dataset", "ingest", "tools", "context", "runs", "backends", "data"), data), } diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index c9044be17..200cd63f2 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -51,8 +51,8 @@ "dokimi.config_check", "dokimi.context", "dokimi.coverage", "dokimi.dashboard", "dokimi.datasets_export", "dokimi.explore", "dokimi.external", "dokimi.harnesses", "dokimi.novelty", "dokimi.orchestrator", "dokimi.report", "dokimi.repos", - "dokimi.runner", "dokimi.schema_check", "dokimi.schema_compat", "dokimi.subjects", - "dokimi.task_formats", "dokimi.tool_preference", + "dokimi.runner", "dokimi.runs", "dokimi.schema_check", "dokimi.schema_compat", + "dokimi.subjects", "dokimi.task_formats", "dokimi.tool_preference", )), # The LLM evaluation engine: answering, judging, panels, arms, questions, personas. ("evaluation", ( diff --git a/dokimi/tests/test_runs_cli.py b/dokimi/tests/test_runs_cli.py new file mode 100644 index 000000000..b2cf2678d --- /dev/null +++ b/dokimi/tests/test_runs_cli.py @@ -0,0 +1,237 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`runs list` (#278): exit codes of the root, filters, order, the --json fields and the summary.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from dokimi import cli +from dokimi.artifacts import save_artifact + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts" + + +def _fixture(version: str) -> dict: + return json.loads((FIXTURES / f"{version}.json").read_text(encoding="utf-8")) + + +@pytest.fixture +def root(tmp_path, monkeypatch): + """An empty default root, ``results/`` in the working directory.""" + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + (tmp_path / "results").mkdir() + return tmp_path / "results" + + +@pytest.fixture +def filled(root): + save_artifact("arms", {**_fixture("arms.v1"), "generated_at": "2026-10-29T14:02:00Z"}, + root / "arms" / "onetbb.json") + save_artifact("plugin_delta", _fixture("plugin_delta.v1"), root / "arms" / "plugin_delta.json") + save_artifact("eval", _fixture("eval.v1"), root / "onetbb_r1" / "eval" / "onetbb.json") + save_artifact("questions", {**_fixture("questions.v2"), "product": "oneTBB", + "generated_at": "2026-10-20T09:00:00+02:00"}, + root / "onetbb_r1" / "questions" / "oneTBB.json") + os.utime(root / "arms" / "plugin_delta.json", (1_780_000_000, 1_780_000_000)) # 2026-05-28 + (root / "arms" / "onetbb.md").write_text("# arms", encoding="utf-8") + (root / "pairwise.json").write_text(json.dumps({"schema_version": "pairwise.v0"}), + encoding="utf-8") + (root / "broken.json").write_text('{"schema_version": "eval.v1", "evaluations": [', + encoding="utf-8") + return root + + +def _list(monkeypatch, capsys, *argv: str): + monkeypatch.setattr("sys.argv", ["dokimi", "runs", "list", *argv]) + cli.main() + return capsys.readouterr() + + +def _paths(out: str) -> list[str]: + rows = out.split("\n\n")[0].splitlines()[1:] + return [row.split()[-1] for row in rows] + + +def test_a_missing_default_root_is_no_runs_and_exits_0(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + + out = _list(monkeypatch, capsys).out + assert out == "no runs under results/ (it does not exist; DOKIMI_RUNS_DIR names another root)\n" + data = json.loads(_list(monkeypatch, capsys, "--json").out) + assert (data["root"], data["artifacts"], data["total"]) == ("results", [], 0) + + +@pytest.mark.parametrize("make, message", [ + (lambda p: None, "does not exist"), + (lambda p: p.write_text("", encoding="utf-8"), "is not a directory"), +], ids=["missing", "a-file"]) +def test_a_bad_dokimi_runs_dir_exits_2_with_a_message(tmp_path, monkeypatch, capsys, make, message): + named = tmp_path / "named" + make(named) + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(named)) + + with pytest.raises(SystemExit) as exc: + _list(monkeypatch, capsys) + + assert exc.value.code == 2 + err = capsys.readouterr().err + assert err == f"Error: runs root DOKIMI_RUNS_DIR={named} {message}\n" + + +def test_dokimi_runs_dir_is_the_root_listed(tmp_path, monkeypatch, capsys, root): + other = tmp_path / "elsewhere" + save_artifact("eval", _fixture("eval.v1"), other / "e.json") + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(other)) + + out = _list(monkeypatch, capsys).out + + assert _paths(out) == ["e.json"] + assert f"1 artifact under {other.as_posix()}/" in out + + +def test_an_empty_root_lists_nothing_and_says_so(monkeypatch, capsys, root): + assert _list(monkeypatch, capsys).out == "0 artifacts under results/\n" + + +def test_rows_are_newest_first_by_recorded_time_then_mtime(monkeypatch, capsys, filled): + out = _list(monkeypatch, capsys).out + + assert _paths(out) == ["arms/onetbb.json", "onetbb_r1/questions/oneTBB.json", + "arms/plugin_delta.json", "onetbb_r1/eval/onetbb.json"] + lines = out.splitlines() + assert lines[0].split() == ["SCHEMA", "SUBJECT", "MODEL", "TIME", "(UTC)", "PATH"] + assert lines[2].split()[:5] == ["questions.v2", "oneTBB", "-", "2026-10-20", "07:00"] + assert lines[3].split()[3] == "2026-05-28" and lines[3].split()[4].endswith("*") + + +def test_kind_is_repeatable_and_subject_matches_exactly(monkeypatch, capsys, filled): + out = _list(monkeypatch, capsys, "--kind", "arms", "--kind", "eval").out + assert _paths(out) == ["arms/onetbb.json", "onetbb_r1/eval/onetbb.json"] + assert "2 of 4 artifacts under results/ match" in out + + out = _list(monkeypatch, capsys, "--subject", "oneTBB").out + assert _paths(out) == ["onetbb_r1/questions/oneTBB.json", "arms/plugin_delta.json"] + + out = _list(monkeypatch, capsys, "--subject", "oneTBB", "--kind", "plugin_delta").out + assert _paths(out) == ["arms/plugin_delta.json"] + + assert _list(monkeypatch, capsys, "--subject", "onet").out.startswith("0 of 4 artifacts") + + +def test_json_has_the_vocabulary_fields_and_every_count(monkeypatch, capsys, filled): + data = json.loads(_list(monkeypatch, capsys, "--json", "--kind", "arms").out) + + assert data["artifacts"] == [{ + "schema_version": "arms.v1", "kind": "arms", "run_id": None, "subject": "onetbb", + "model": "m", "generated_at": "2026-10-29T14:02:00Z", "path": "arms/onetbb.json", + "size_bytes": (filled / "arms" / "onetbb.json").stat().st_size, + }] + assert data["root"] == "results" and data["total"] == 4 + assert data["skipped"] == {"not_json": 1, "unknown_schema_version": 1} + assert data["unknown_versions"] == {"pairwise.v0": 1} + (bad,) = data["unreadable"] + assert (bad["path"], bad["schema_version"]) == ("broken.json", "eval.v1") + assert bad["reason"].startswith("not valid JSON: ") + + +def test_the_text_summary_counts_what_was_skipped_and_names_the_first_unreadable( + monkeypatch, capsys, filled): + (filled / "late.json").write_text(json.dumps({"x": 1, "schema_version": "eval.v1"}), + encoding="utf-8") + (filled / "arms" / "onetbb.json.rejected").write_text("{}", encoding="utf-8") + + summary = _list(monkeypatch, capsys).out.split("\n\n")[1].splitlines() + + assert summary == [ + "4 artifacts under results/ (* the file's mtime: the artifact records no time)", + "skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_version " + "(pairwise.v0: 1), 1 *.rejected (payloads save_artifact refused), 1 not .json", + "unreadable: 1 (first: broken.json: not valid JSON: Expecting value at line 1)", + ] + + +def test_the_mtime_footnote_is_there_only_when_a_shown_row_has_a_star(monkeypatch, capsys, filled): + out = _list(monkeypatch, capsys, "--kind", "arms").out + + assert out.split("\n\n")[1].splitlines()[0] == "1 of 4 artifacts under results/ match" + + +@pytest.mark.skipif(os.name == "nt", reason="Windows file names cannot hold control characters") +def test_control_characters_in_names_and_fields_are_escaped(monkeypatch, capsys, root): + save_artifact("arms", {**_fixture("arms.v1"), "library_name": "evil\x1b[2J", + "generated_at": "2026-10-29T14:02:00Z"}, root / "two\nlines.json") + + rows = _list(monkeypatch, capsys).out.split("\n\n")[0].splitlines() + + assert len(rows) == 2 + assert rows[1].split()[1] == "evil\\x1b[2J" and rows[1].endswith("two\\nlines.json") + + +def test_an_unknown_version_with_a_line_separator_is_printed_escaped(monkeypatch, capsys, root): + (root / "x.json").write_text(json.dumps({"schema_version": "v\u2028forged"}, ensure_ascii=False), + encoding="utf-8") + + out = _list(monkeypatch, capsys).out + + assert "\u2028" not in out + assert "JSON of an unknown schema_version (v\\u2028forged: 1)" in out + + +def test_an_mtime_past_the_year_9999_is_shown_as_seconds(monkeypatch, capsys, root): + from dokimi.commands import runs as runs_cmd + from dokimi.runs.scan import Artifact, ScanResult + + far = Artifact(path=Path("p.json"), schema_version="plugin_delta.v1", subjects=(), models=(), + generated_at=None, size_bytes=1, mtime=253402300800.0) # tmpfs keeps such mtimes + monkeypatch.setattr(runs_cmd, "scan", lambda path: ScanResult(root=path, artifacts=[far])) + + rows = _list(monkeypatch, capsys).out.splitlines() + + assert rows[1].split()[:4] == ["plugin_delta.v1", "-", "-", "@253402300800*"] + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root searches every directory") +def test_a_root_under_a_directory_that_cannot_be_searched_exits_2(tmp_path, monkeypatch, capsys): + named = tmp_path / "locked" / "root" + named.mkdir(parents=True) + (tmp_path / "locked").chmod(0o600) + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(named)) + try: + with pytest.raises(SystemExit) as exc: + _list(monkeypatch, capsys) + finally: + (tmp_path / "locked").chmod(0o755) + + assert exc.value.code == 2 + # Python 3.11-3.13 raise PermissionError on the lookup; 3.14 reports the path as missing. + assert capsys.readouterr().err in ( + f"Error: runs root DOKIMI_RUNS_DIR={named}: cannot read: Permission denied\n", + f"Error: runs root DOKIMI_RUNS_DIR={named} does not exist\n") + + +def test_a_time_that_is_not_iso_8601_is_shown_as_written(monkeypatch, capsys, root): + save_artifact("eval", {**_fixture("eval.v1"), "evaluated_at": "yesterday"}, root / "e.json") + + rows = _list(monkeypatch, capsys).out.splitlines() + + assert rows[1].split()[:4] == ["eval.v1", "-", "demo-judge", "yesterday"] + + +@pytest.mark.skipif(os.name == "nt", reason="Windows paths cannot hold control characters") +def test_a_bad_root_is_named_escaped(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(tmp_path / "no\nwhere")) + + with pytest.raises(SystemExit): + _list(monkeypatch, capsys) + + err = capsys.readouterr().err + assert err == f"Error: runs root DOKIMI_RUNS_DIR={tmp_path}/no\\nwhere does not exist\n" diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py new file mode 100644 index 000000000..9b5b9997d --- /dev/null +++ b/dokimi/tests/test_runs_scan.py @@ -0,0 +1,346 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The runs root scan (#278): an artifact's kind from its first bytes, its fields, and the count +of every file skipped.""" + +from __future__ import annotations + +import importlib +import json +import os +import time +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from dokimi.artifacts import SCHEMA_VERSIONS, SUPERSEDED_VERSIONS, save_artifact +from dokimi.runs import kinds, runs_root, scan +from dokimi.runs.scan import HEAD_BYTES, RunsRoot +from dokimi.utils import InputError + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts" +# The module, not the scan() that dokimi.runs re-exports under the same name. +scan_module = importlib.import_module("dokimi.runs.scan") + +#: version -> (keys added to its fixture, expected subject, model, time). +EXPECTED: dict[str, tuple[dict, str | None, str | None, str | None]] = { + "questions.v2": ({"model": "gen-m", "generated_at": "2026-10-01T00:00:00Z"}, + "demo", "gen-m", "2026-10-01T00:00:00Z"), + "answers.v1": ({}, "demo", "demo-model", "2026-01-01T00:00:00Z"), + "eval.v1": ({}, None, "demo-judge", "2026-01-01T00:00:00Z"), + "arms.v1": ({"generated_at": "2026-10-02T00:00:00Z"}, "onetbb", "m", "2026-10-02T00:00:00Z"), + "matrix_rollup.v1": ({}, "oneTBB", None, "2026-10-04T17:47:16.294238Z"), + "plugin_delta.v1": ({}, "oneTBB", "m", None), + "subject_scorecard.v2": ({}, "docs-subject", None, "2026-10-04T17:47:19.175356Z"), + "novelty.v1": ({}, "skill:/tmp/run/accel-tuning/SKILL.md", "answerer-m", + "2026-10-04T17:47:16.841483Z"), + "task_runs.v1": ({}, None, "haiku", "2026-01-01T00:00:00Z"), + "explore_runs.v1": ({"model": "explorer-m"}, None, "explorer-m", "2026-10-04T17:47:15.816155Z"), + "guidance_reach.v1": ({}, "toy", "m", "2026-10-06T04:30:15.383519+00:00"), + "repo_readiness.v1": ({"generated_at": "2026-10-03T00:00:00Z"}, "toy", None, + "2026-10-03T00:00:00Z"), + "tool_preference.v1": ({}, None, "claude-haiku-5-5", "2026-10-14T00:00:00+00:00"), + "repo_environment.v1": ({}, "toy@4ff761e2d943", None, None), + "plan.v1": ({}, "skill:onetbb", "claude-sonnet-4-6", "2026-10-21T00:00:00Z"), + "result.v1": ({}, None, None, "2026-10-21T01:00:00Z"), + "repo_budget.v1": ({}, "toy", None, None), + "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), + "cli_routes.v1": ({}, None, None, None), +} + + +def _fixture(version: str) -> dict: + return json.loads((FIXTURES / f"{version}.json").read_text(encoding="utf-8")) + + +def _save(root: Path, version: str, rel: str, **extra) -> Path: + path = root / rel + save_artifact(kinds.VERSION_KINDS[version], {**_fixture(version), **extra}, path) + return path + + +def test_every_kind_has_a_row_in_the_fields_table_and_in_this_test(): + assert set(kinds.FIELDS) == set(SCHEMA_VERSIONS) + assert set(EXPECTED) == set(SCHEMA_VERSIONS.values()) + + +@pytest.mark.parametrize("version", sorted(EXPECTED)) +def test_each_kind_is_listed_with_the_subject_model_and_time_it_records(tmp_path, version): + extra, subject, model, time = EXPECTED[version] + path = _save(tmp_path, version, f"some/dir/{version}.json", **extra) + + result = scan(tmp_path) + + assert result.unreadable == [] and not result.skipped + (found,) = result.artifacts + assert found.to_json() == { + "schema_version": version, "kind": kinds.VERSION_KINDS[version], "run_id": None, + "subject": subject, "model": model, "generated_at": time, + "path": f"some/dir/{version}.json", "size_bytes": path.stat().st_size, + } + + +@pytest.mark.parametrize("version", [v for vs in SUPERSEDED_VERSIONS.values() for v in vs]) +def test_a_superseded_version_is_still_an_artifact(tmp_path, version): + data = _fixture(version) + payload = {"schema_version": version, **{k: v for k, v in data.items() if k != "schema_version"}} + (tmp_path / "old.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") + + (found,) = scan(tmp_path).artifacts + + assert (found.schema_version, found.kind) == (version, kinds.VERSION_KINDS[version]) + assert found.subject == {"questions.v1": "demo", "subject_scorecard.v1": "s"}[version] + + +def test_distinct_values_of_a_list_are_all_kept_and_joined(tmp_path): + data = _fixture("answers.v1") + first = data["answers"][0] + data["answers"] = [first, {**first, "question_id": "q2", "library_name": "other"}, + {**first, "question_id": "q3"}] + save_artifact("answers", data, tmp_path / "a.json") + + (found,) = scan(tmp_path).artifacts + + assert found.subjects == ("demo", "other") + assert found.subject == "demo, other" + + +def test_every_file_that_is_not_an_artifact_is_counted_by_reason(tmp_path): + _save(tmp_path, "eval.v1", "run/eval/p.json") + (tmp_path / "run" / "eval" / "p.json.123.456.tmp").write_text("{", encoding="utf-8") + (tmp_path / "run" / "eval" / "p.json.rejected").write_text( + json.dumps({"schema_version": "eval.v1"}), encoding="utf-8") + (tmp_path / "run" / "report.md").write_text("# report", encoding="utf-8") + (tmp_path / "dashboard.json").write_text(json.dumps({"libraries": []}), encoding="utf-8") + (tmp_path / "pairwise.json").write_text(json.dumps({"schema_version": "pairwise.v0"}), + encoding="utf-8") + (tmp_path / "checklist.json").write_text(json.dumps({"schema_version": "checklist.v0"}), + encoding="utf-8") + (tmp_path / "broken.json").write_text('{"schema_version": "answers.v1", "answers": [', + encoding="utf-8") + (tmp_path / "late.json").write_text(json.dumps({"evaluations": [], "schema_version": "eval.v1"}), + encoding="utf-8") + + result = scan(tmp_path) + + assert [a.path.as_posix() for a in result.artifacts] == ["run/eval/p.json"] + assert result.skipped == {"temporary": 1, "rejected": 1, "not_json": 1, + "no_leading_schema_version": 2, "unknown_schema_version": 2} + assert result.unknown_versions == {"pairwise.v0": 1, "checklist.v0": 1} + (bad,) = result.unreadable + assert (bad.path.as_posix(), bad.schema_version) == ("broken.json", "answers.v1") + assert bad.reason.startswith("not valid JSON: ") + + +def test_a_file_that_is_not_utf8_is_unreadable_with_its_reason(tmp_path): + (tmp_path / "x.json").write_bytes(b'{"schema_version": "eval.v1", "judge_model": "\xff"}') + + (bad,) = scan(tmp_path).unreadable + + assert (bad.reason, bad.schema_version) == ("not UTF-8", "eval.v1") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_symlinked_directory_is_counted_and_not_walked(tmp_path): + _save(tmp_path, "eval.v1", "run/eval.json") + (tmp_path / "run" / "loop").symlink_to(tmp_path, target_is_directory=True) + (tmp_path / "self.json").symlink_to(tmp_path / "self.json") + + result = scan(tmp_path) + + assert [a.path.as_posix() for a in result.artifacts] == ["run/eval.json"] + assert result.skipped == {"linked_directory": 1} + (bad,) = result.unreadable + assert bad.path.as_posix() == "self.json" and bad.reason.startswith("cannot read: ") + + +def test_the_kind_comes_from_the_first_bytes_and_the_rest_is_not_parsed_to_skip(tmp_path): + # The tail is broken: a skipped file was never parsed, a file of a known kind was. + tail = "x" * HEAD_BYTES + "{not json" + (tmp_path / "big.json").write_text('{"libraries": "' + tail, encoding="utf-8") + (tmp_path / "eval.json").write_text('{"schema_version": "eval.v1", "x": "' + tail, + encoding="utf-8") + + result = scan(tmp_path) + + assert result.skipped == {"no_leading_schema_version": 1} + (bad,) = result.unreadable + assert (bad.path.as_posix(), bad.schema_version) == ("eval.json", "eval.v1") + + +def test_a_head_that_names_one_version_and_a_body_that_holds_another_is_unreadable(tmp_path): + (tmp_path / "x.json").write_text('{"schema_version": "eval.v1", "schema_version": "arms.v1"}', + encoding="utf-8") + + (bad,) = scan(tmp_path).unreadable + + assert "does not hold the schema_version eval.v1" in bad.reason + + +def test_artifacts_are_newest_first_by_their_time_then_by_mtime(tmp_path): + _save(tmp_path, "eval.v1", "old.json", evaluated_at="2026-01-01T00:00:00Z") + _save(tmp_path, "eval.v1", "new.json", evaluated_at="2026-10-01T00:00:00+00:00") + _save(tmp_path, "eval.v1", "naive.json", evaluated_at="2026-06-01T00:00:00") + no_time = _save(tmp_path, "plugin_delta.v1", "no_time.json") + stamp = 1_780_000_000 # 2026-05-28, between old and naive + os.utime(no_time, (stamp, stamp)) + + paths = [a.path.as_posix() for a in scan(tmp_path).artifacts] + + assert paths == ["new.json", "naive.json", "no_time.json", "old.json"] + + +@pytest.mark.skipif(not hasattr(time, "tzset"), reason="TZ is read by time.tzset, POSIX only") +def test_a_time_that_names_no_zone_is_utc_whatever_the_local_zone(tmp_path, monkeypatch): + _save(tmp_path, "eval.v1", "e.json", evaluated_at="2026-06-01T09:30:00") + monkeypatch.setenv("TZ", "Asia/Kolkata") + time.tzset() + try: + (found,) = scan(tmp_path).artifacts + finally: + monkeypatch.undo() + time.tzset() + + assert found.recorded_at == datetime(2026, 6, 1, 9, 30, tzinfo=timezone.utc) + + +def test_dokimi_runs_dir_overrides_the_default_root(tmp_path): + assert runs_root({}) == RunsRoot(Path("results"), from_env=False) + assert runs_root({"DOKIMI_RUNS_DIR": ""}) == RunsRoot(Path("results"), from_env=False) + assert runs_root({"DOKIMI_RUNS_DIR": str(tmp_path)}) == RunsRoot(tmp_path, from_env=True) + + +def test_a_missing_default_root_is_no_runs_and_a_bad_named_root_is_an_input_error(tmp_path): + assert RunsRoot(tmp_path / "results", from_env=False).exists() is False + assert RunsRoot(tmp_path, from_env=True).exists() is True + with pytest.raises(InputError, match="DOKIMI_RUNS_DIR=.* does not exist"): + RunsRoot(tmp_path / "nope", from_env=True).exists() + (tmp_path / "file").write_text("", encoding="utf-8") + with pytest.raises(InputError, match="is not a directory"): + RunsRoot(tmp_path / "file", from_env=True).exists() + with pytest.raises(InputError, match="is not a directory"): + RunsRoot(tmp_path / "file", from_env=False).exists() + + +def test_upper_case_suffixes_are_matched_too(tmp_path): + _save(tmp_path, "eval.v1", "E.JSON") + (tmp_path / "x.json.1.2.TMP").write_text("{", encoding="utf-8") + + result = scan(tmp_path) + + assert [a.path.as_posix() for a in result.artifacts] == ["E.JSON"] + assert result.skipped == {"temporary": 1} + + +def test_a_schema_version_past_the_first_bytes_is_not_a_leading_one(tmp_path): + assert HEAD_BYTES < 1000 + padded = "{" + " " * 1000 + '"schema_version": "eval.v1", "evaluations": []}' + (tmp_path / "padded.json").write_text(padded, encoding="utf-8") + + assert scan(tmp_path).skipped == {"no_leading_schema_version": 1} + + +def test_a_document_nested_too_deep_to_parse_is_unreadable(tmp_path, monkeypatch): + # How deep json can go depends on the interpreter (3.14 bounds it by C stack), so the + # RecursionError is raised for the parse instead of found by nesting. + def too_deep(path, version): + raise RecursionError("maximum recursion depth exceeded while decoding a JSON array") + + _save(tmp_path, "eval.v1", "deep.json") + monkeypatch.setattr(scan_module, "load_document", too_deep) + + (bad,) = scan(tmp_path).unreadable + + assert (bad.path.as_posix(), bad.schema_version) == ("deep.json", "eval.v1") + assert bad.reason == "maximum recursion depth exceeded while decoding a JSON array" + + +def test_a_time_out_of_range_falls_back_to_the_mtime(tmp_path): + early = _save(tmp_path, "eval.v1", "early.json", evaluated_at="0001-01-01T00:00:00+05:00") + late = _save(tmp_path, "eval.v1", "late.json", evaluated_at="9999-12-31T23:00:00-05:00") + os.utime(early, (1_780_000_000, 1_780_000_000)) + os.utime(late, (1_790_000_000, 1_790_000_000)) + + found = scan(tmp_path).artifacts + + assert [(a.path.as_posix(), a.recorded_at) for a in found] == [("late.json", None), + ("early.json", None)] + + +def test_unreadable_files_come_in_path_order(tmp_path): + names = [f"{d}/{i:02}.json" for d in ("b", "a") for i in range(20, 0, -1)] + for rel in names: + (tmp_path / rel).parent.mkdir(exist_ok=True) + (tmp_path / rel).write_text('{"schema_version": "eval.v1", ', encoding="utf-8") + + paths = [u.path.as_posix() for u in scan(tmp_path).unreadable] + + assert paths == sorted(names) + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFOs are POSIX") +def test_a_fifo_is_skipped_without_being_opened(tmp_path): + os.mkfifo(tmp_path / "pipe.json") # opening it for reading would block the scan + + assert scan(tmp_path).skipped == {"not_a_file": 1} + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root reads every directory") +@pytest.mark.parametrize("mode", [0o000, 0o444], ids=["unreadable", "not-searchable"]) +def test_a_directory_that_cannot_be_walked_is_unreadable_not_a_crash(tmp_path, mode): + _save(tmp_path, "eval.v1", "ok.json") + (tmp_path / "locked" / "sub").mkdir(parents=True) + (tmp_path / "locked").chmod(mode) + try: + result = scan(tmp_path) + finally: + (tmp_path / "locked").chmod(0o755) + + assert [a.path.as_posix() for a in result.artifacts] == ["ok.json"] + (bad,) = result.unreadable + expected = "locked" if mode == 0o000 else "locked/sub" + assert (bad.path.as_posix(), bad.reason) == (expected, "cannot list: Permission denied") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_file_symlink_is_read_only_when_it_resolves_inside_the_root(tmp_path): + root = tmp_path / "results" + _save(root, "eval.v1", "arms/real.json") + _save(tmp_path, "eval.v1", "outside/eval.json") + (root / "latest.json").symlink_to(root / "arms" / "real.json") + (root / "external.json").symlink_to(tmp_path / "outside" / "eval.json") + + result = scan(root) + + assert sorted(a.path.as_posix() for a in result.artifacts) == ["arms/real.json", "latest.json"] + assert result.skipped == {"linked_outside": 1} + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_root_that_is_itself_a_symlink_is_read(tmp_path): + _save(tmp_path, "eval.v1", "store/e.json") + (tmp_path / "store" / "latest.json").symlink_to(tmp_path / "store" / "e.json") + (tmp_path / "results").symlink_to(tmp_path / "store", target_is_directory=True) + + result = scan(tmp_path / "results") + + assert sorted(a.path.as_posix() for a in result.artifacts) == ["e.json", "latest.json"] + assert not result.skipped + + +@pytest.mark.skipif(os.name != "nt", reason="junctions are a Windows directory link") +def test_a_junction_is_counted_and_not_walked(tmp_path): + import _winapi + + root = tmp_path / "results" + _save(root, "eval.v1", "in.json") + _save(tmp_path, "eval.v1", "outside/eval.json") + _winapi.CreateJunction(str(tmp_path / "outside"), str(root / "external")) + + result = scan(root) + + assert [a.path.as_posix() for a in result.artifacts] == ["in.json"] + assert result.skipped == {"linked_directory": 1} diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py new file mode 100644 index 000000000..07ad7a9f5 --- /dev/null +++ b/dokimi/tests/test_runs_show.py @@ -0,0 +1,408 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`runs show` (#278): one artifact's fields, schema check, contents and neighbours; a failed +schema check exits 0, an address that is not an artifact under the root exits 2.""" + +from __future__ import annotations + +import dataclasses +import json +import os +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from dokimi import cli +from dokimi.artifacts import SCHEMA_DIR, SCHEMA_VERSIONS, save_artifact +from dokimi.runs import kinds +from dokimi.runs.show import MAX_SIBLINGS, Shown, show + +#: What ``contents`` counts in each kind's fixture (the length of its top-level lists). +CONTENTS = { + "answers.v1": {"answers": 1}, + "arms.v1": {"arms": 4, "answers": 0}, + "eval.v1": {"evaluations": 1}, + "explore_runs.v1": {"tasks": 1, "arms": 2, "results": 2}, + "guidance_reach.v1": {"files": 5, "scenarios": 5, "records": 25}, + "matrix_rollup.v1": {"cells": 2}, + "novelty.v1": {"conflicts": 0, "warnings": 2}, + "plan.v1": {"cells": 6}, + "plugin_delta.v1": {"arms": 2}, + "questions.v2": {"questions": 3}, + "repo_budget.v1": {"entries": 9}, + "repo_environment.v1": {"instructions": 4, "findings": 13}, + "repo_readiness.v1": {"files": 2, "claims": 52}, + "result.v1": {"cells": 6}, + "subject_scorecard.v2": {"cells": 0}, + "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "task_inspect.v1": {"items": 4}, + "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, + "tool_preference.v1": {"items": 16, "results": 16}, +} + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts" + + +def _fixture(version: str) -> dict: + return json.loads((FIXTURES / f"{version}.json").read_text(encoding="utf-8")) + + +@pytest.fixture +def root(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + root = tmp_path / "results" + save_artifact("arms", {**_fixture("arms.v1"), "generated_at": "2026-10-29T14:02:00Z"}, + root / "arms" / "onetbb.json") + save_artifact("plugin_delta", _fixture("plugin_delta.v1"), root / "arms" / "plugin_delta.json") + (root / "arms" / "onetbb.md").write_text("# arms", encoding="utf-8") + (root / "arms" / "onetbb-matrix").mkdir() + return root + + +def _show(monkeypatch, capsys, *argv: str): + monkeypatch.setattr("sys.argv", ["dokimi", "runs", "show", *argv]) + cli.main() + return capsys.readouterr() + + +def _fails(monkeypatch, capsys, *argv: str) -> str: + with pytest.raises(SystemExit) as exc: + _show(monkeypatch, capsys, *argv) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert err.startswith("Error: ") and "Traceback" not in err + return err + + +@pytest.mark.parametrize("kind", sorted(kinds.FIELDS)) +def test_every_count_is_a_top_level_list_of_the_kinds_schema(kind): + schema = json.loads((SCHEMA_DIR / f"{SCHEMA_VERSIONS[kind]}.json").read_text("utf-8")) + for key in kinds.FIELDS[kind].counts: + declared = schema["properties"][key].get("type") + assert declared == "array" or "array" in (declared or []), (kind, key) + + +def test_a_valid_artifact_shows_its_fields_contents_and_neighbours(monkeypatch, capsys, root): + out = _show(monkeypatch, capsys, "arms/onetbb.json").out + + size = (root / "arms" / "onetbb.json").stat().st_size + assert out.splitlines() == [ + "path: arms/onetbb.json", + "kind: arms (arms.v1)", + "schema: valid", + "subject: onetbb", + "model: m", + "time: 2026-10-29T14:02:00Z", + "run_id: - (no artifact records one yet)", + f"size: {size:,} bytes", + "contents: arms: 4, answers: 0", + "next to it: onetbb-matrix/, onetbb.md, plugin_delta.json (plugin_delta.v1)", + ] + + +def test_the_path_may_be_absolute_or_from_the_working_directory(monkeypatch, capsys, root): + first = _show(monkeypatch, capsys, "arms/onetbb.json").out + assert _show(monkeypatch, capsys, str(root / "arms" / "onetbb.json")).out == first + assert _show(monkeypatch, capsys, "results/arms/onetbb.json").out == first + + +def test_an_artifact_that_fails_its_schema_lists_the_errors_and_exits_0(monkeypatch, capsys, root): + data = json.loads((root / "arms" / "onetbb.json").read_text(encoding="utf-8")) + data["arms"], data["total_questions"] = "baseline", "six" + (root / "arms" / "bad.json").write_text(json.dumps(data, indent=2), encoding="utf-8") + + lines = _show(monkeypatch, capsys, "arms/bad.json").out.splitlines() # no SystemExit: 0 + + assert lines[2:5] == [ + "schema: invalid, 2 errors", + " - arms: 'baseline' is not of type 'array'", + " - total_questions: 'six' is not of type 'integer'", + ] + assert lines[5] == "subject: onetbb" + + +def test_a_time_the_artifact_does_not_record_is_shown_as_the_files_mtime(monkeypatch, capsys, root): + stamp = 1_780_000_000 + os.utime(root / "arms" / "plugin_delta.json", (stamp, stamp)) + out = _show(monkeypatch, capsys, "arms/plugin_delta.json").out + + (time_line,) = [line for line in out.splitlines() if line.startswith("time:")] + when = datetime.fromtimestamp(stamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + assert time_line == f"time: - ({when} UTC, the file's mtime)" + + +@pytest.mark.parametrize("address, message", [ + ("arms/nope.json", "no file arms/nope.json under the runs root results/"), + ("arms", "arms is a directory; runs show takes one artifact file"), + ("arms/onetbb.md", "arms/onetbb.md is not an artifact: it does not open with a schema_version"), + ("../outside.json", "../outside.json is not under the runs root results/"), + ("pairwise.json", "pairwise.json is not an artifact of a registered kind: " + "schema_version pairwise.v0"), + ("broken.json", "broken.json: not valid JSON: Expecting value at line 1"), +]) +def test_an_address_that_is_not_an_artifact_under_the_root_exits_2(monkeypatch, capsys, root, + address, message): + (root.parent / "outside.json").write_text(json.dumps({"schema_version": "eval.v1"}), + encoding="utf-8") + (root / "pairwise.json").write_text(json.dumps({"schema_version": "pairwise.v0"}), + encoding="utf-8") + (root / "broken.json").write_text('{"schema_version": "eval.v1", "x": [', encoding="utf-8") + + assert _fails(monkeypatch, capsys, address) == f"Error: {message}\n" + + +def test_an_absolute_path_outside_the_root_exits_2(monkeypatch, capsys, root, tmp_path): + other = tmp_path / "eval.json" + save_artifact("eval", _fixture("eval.v1"), other) + + assert "is not under the runs root" in _fails(monkeypatch, capsys, str(other)) + + +def test_show_without_a_root_exits_2(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + + err = _fails(monkeypatch, capsys, "x.json") + + assert err.startswith("Error: no runs root: results/ does not exist") + + +def test_json_has_the_list_fields_the_schema_check_and_the_neighbours(monkeypatch, capsys, root): + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + + assert data == { + "schema_version": "arms.v1", "kind": "arms", "run_id": None, "subject": "onetbb", + "model": "m", "generated_at": "2026-10-29T14:02:00Z", "path": "arms/onetbb.json", + "size_bytes": (root / "arms" / "onetbb.json").stat().st_size, "root": "results", + "schema": "arms.v1", "schema_valid": True, "schema_errors": [], + "contents": {"arms": 4, "answers": 0}, + "siblings": [{"name": "onetbb-matrix/", "schema_version": None}, + {"name": "onetbb.md", "schema_version": None}, + {"name": "plugin_delta.json", "schema_version": "plugin_delta.v1"}], + "siblings_total": 3, "siblings_error": None, + } + + +def test_json_of_an_invalid_artifact_carries_its_errors(monkeypatch, capsys, root): + (root / "arms" / "bad.json").write_text(json.dumps({"schema_version": "arms.v1"}), + encoding="utf-8") + + data = json.loads(_show(monkeypatch, capsys, "arms/bad.json", "--json").out) + + assert data["schema_valid"] is False + assert sorted(data["schema_errors"]) == [": 'answers' is a required property", + ": 'arms' is a required property"] + + +def test_a_crowded_directory_names_the_first_neighbours_and_counts_the_rest(monkeypatch, capsys, + root): + for i in range(MAX_SIBLINGS + 5): + (root / "arms" / f"z{i:02}.log").write_text("", encoding="utf-8") + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + text = _show(monkeypatch, capsys, "arms/onetbb.json").out + + assert len(data["siblings"]) == MAX_SIBLINGS and data["siblings_total"] == MAX_SIBLINGS + 8 + assert text.rstrip().endswith(", and 8 more") + + +def test_a_superseded_version_is_checked_against_its_own_schema(monkeypatch, capsys, root): + old = {"schema_version": "questions.v1", + **{k: v for k, v in _fixture("questions.v1").items() if k != "schema_version"}} + (root / "q.json").write_text(json.dumps(old, indent=2), encoding="utf-8") + + data = json.loads(_show(monkeypatch, capsys, "q.json", "--json").out) + + assert (data["schema"], data["schema_valid"], data["kind"]) == ("questions.v1", True, + "questions") + + +def test_the_count_is_kept_for_every_list_of_every_kind(tmp_path): + assert set(CONTENTS) == set(SCHEMA_VERSIONS.values()) + for version, expected in CONTENTS.items(): + save_artifact(kinds.VERSION_KINDS[version], _fixture(version), tmp_path / f"{version}.json") + assert show(tmp_path, f"{version}.json").contents == expected, version + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFOs are POSIX") +def test_a_fifo_is_named_as_a_neighbour_and_refused_as_an_address(monkeypatch, capsys, root): + os.mkfifo(root / "arms" / "pipe.json") # opening it would block + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + assert {"name": "pipe.json", "schema_version": None} in data["siblings"] + assert _fails(monkeypatch, capsys, "arms/pipe.json") == "Error: arms/pipe.json is not a regular file\n" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_neighbour_that_cannot_be_stated_is_named_not_a_crash(monkeypatch, capsys, root): + (root / "arms" / "loop").symlink_to(root / "arms" / "loop") + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + + assert {"name": "loop", "schema_version": None} in data["siblings"] + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root lists every directory") +def test_a_directory_that_cannot_be_listed_still_shows_its_artifact(monkeypatch, capsys, root): + (root / "arms").chmod(0o311) # files can be opened by name, the directory cannot be listed + try: + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + text = _show(monkeypatch, capsys, "arms/onetbb.json").out + finally: + (root / "arms").chmod(0o755) + + assert (data["siblings"], data["siblings_error"]) == ([], "cannot list: Permission denied") + assert text.splitlines()[-1] == "next to it: cannot list: Permission denied" + + +def test_control_characters_in_fields_are_escaped(monkeypatch, capsys, root): + save_artifact("arms", {**_fixture("arms.v1"), "library_name": "evil\x1b[2J\nnext"}, + root / "evil.json") + + lines = _show(monkeypatch, capsys, "evil.json").out.splitlines() + + assert "subject: evil\\x1b[2J\\nnext" in lines and len(lines) == 10 + + +def test_an_unknown_version_is_named_escaped_in_the_error(monkeypatch, capsys, root): + (root / "x.json").write_text(json.dumps({"schema_version": "v\u2028forged"}, ensure_ascii=False), + encoding="utf-8") + + err = _fails(monkeypatch, capsys, "x.json") + + assert err == ("Error: x.json is not an artifact of a registered kind: " + "schema_version v\\u2028forged\n") + + +def test_an_mtime_past_the_year_9999_is_shown_as_seconds(monkeypatch, capsys, root): + from dokimi.commands import runs as runs_cmd + + real = show(root, "arms/plugin_delta.json") + far = Shown(artifact=dataclasses.replace(real.artifact, mtime=253402300800.0), + schema=real.schema, errors=[], contents=real.contents) + monkeypatch.setattr(runs_cmd, "show", lambda root, address: far) + + out = _show(monkeypatch, capsys, "arms/plugin_delta.json").out + + assert "time: - (@253402300800 UTC, the file's mtime)" in out.splitlines() + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root searches every directory") +def test_an_address_in_a_directory_that_cannot_be_searched_exits_2(monkeypatch, capsys, root): + (root / "arms").chmod(0o600) + try: + err = _fails(monkeypatch, capsys, "arms/onetbb.json") + finally: + (root / "arms").chmod(0o755) + + # Python 3.11-3.13 raise PermissionError on the lookup; 3.14 reports the file as missing. + assert err in ("Error: arms/onetbb.json: cannot read: Permission denied\n", + "Error: no file arms/onetbb.json under the runs root results/\n") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +@pytest.mark.parametrize("address", ["linked/eval.json", "external.json"]) +def test_a_symlink_that_leads_outside_the_root_is_not_read(monkeypatch, capsys, root, address): + save_artifact("eval", _fixture("eval.v1"), root.parent / "ext" / "eval.json") + (root / "linked").symlink_to(root.parent / "ext", target_is_directory=True) + (root / "external.json").symlink_to(root.parent / "ext" / "eval.json") + + err = _fails(monkeypatch, capsys, address) + + assert err == f"Error: {address} leads outside the runs root results/\n" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_symlink_inside_the_root_is_shown_by_its_own_name(monkeypatch, capsys, root): + (root / "latest.json").symlink_to(root / "arms" / "onetbb.json") + + data = json.loads(_show(monkeypatch, capsys, "latest.json", "--json").out) + + assert (data["path"], data["kind"]) == ("latest.json", "arms") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_the_real_path_of_a_symlinked_root_is_inside_it(monkeypatch, capsys, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + save_artifact("eval", _fixture("eval.v1"), tmp_path / "store" / "e.json") + (tmp_path / "results").symlink_to(tmp_path / "store", target_is_directory=True) + + data = json.loads(_show(monkeypatch, capsys, str(tmp_path / "store" / "e.json"), "--json").out) + + assert data["path"] == "e.json" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_neighbour_that_links_outside_the_root_is_named_not_read(monkeypatch, capsys, root): + save_artifact("eval", _fixture("eval.v1"), root.parent / "ext" / "eval.json") + (root / "arms" / "ext.json").symlink_to(root.parent / "ext" / "eval.json") + (root / "arms" / "in.json").symlink_to(root / "arms" / "plugin_delta.json") + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + + assert {"name": "ext.json", "schema_version": None} in data["siblings"] + assert {"name": "in.json", "schema_version": "plugin_delta.v1"} in data["siblings"] + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root reads every file") +def test_an_artifact_that_cannot_be_opened_exits_2(monkeypatch, capsys, root): + (root / "arms" / "onetbb.json").chmod(0o000) + try: + err = _fails(monkeypatch, capsys, "arms/onetbb.json") + finally: + (root / "arms" / "onetbb.json").chmod(0o644) + + assert err == "Error: arms/onetbb.json: cannot read: Permission denied\n" + + +@pytest.mark.skipif(os.name == "nt", reason="Windows paths cannot hold control characters") +def test_a_rejected_address_is_named_escaped(monkeypatch, capsys, root): + err = _fails(monkeypatch, capsys, "missing\nforged\x1b[2J.json") + + assert err == "Error: no file missing\\nforged\\x1b[2J.json under the runs root results/\n" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_dot_dot_after_a_symlink_is_the_file_the_filesystem_opens(monkeypatch, capsys, root): + save_artifact("eval", _fixture("eval.v1"), root / "nested" / "eval.json") + save_artifact("plugin_delta", _fixture("plugin_delta.v1"), root / "eval.json") + (root / "nested" / "deep").mkdir() + (root / "link").symlink_to(root / "nested" / "deep", target_is_directory=True) + + data = json.loads(_show(monkeypatch, capsys, "link/../eval.json", "--json").out) + + assert (data["kind"], data["path"]) == ("eval", "nested/eval.json") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_neighbours_are_never_listed_from_outside_the_root(monkeypatch, capsys, root): + save_artifact("eval", _fixture("eval.v1"), root.parent / "ext" / "secret.json") + (root / "linked").symlink_to(root.parent / "ext", target_is_directory=True) + (root.parent / "ext" / "return.json").symlink_to(root / "arms" / "onetbb.json") + + data = json.loads(_show(monkeypatch, capsys, "linked/return.json", "--json").out) + + assert data["kind"] == "arms" + assert [s["name"] for s in data["siblings"]] == ["onetbb-matrix/", "onetbb.md", + "plugin_delta.json"] + + +def test_an_absolute_address_under_a_root_named_with_dot_dot(tmp_path, monkeypatch, capsys): + save_artifact("eval", _fixture("eval.v1"), tmp_path / "results" / "e.json") + (tmp_path / "work").mkdir() + monkeypatch.chdir(tmp_path / "work") + monkeypatch.setenv("DOKIMI_RUNS_DIR", os.path.join("..", "results")) + + by_path = json.loads(_show(monkeypatch, capsys, str(tmp_path / "results" / "e.json"), + "--json").out) + by_name = json.loads(_show(monkeypatch, capsys, "e.json", "--json").out) + + assert (by_path["path"], by_path["kind"]) == (by_name["path"], by_name["kind"]) == ("e.json", + "eval")