From d3551fd139d74dc80ab4deb1a3d469eac3a6eed8 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 06:26:46 -0700 Subject: [PATCH 01/18] feat(runs): scan the runs root and tell each artifact's kind from its first bytes New package dokimi/runs, in the tracks layer. runs_root() is DOKIMI_RUNS_DIR when set, else results/ in the working directory, where most commands write by default. A missing default root means no runs yet; a DOKIMI_RUNS_DIR that does not exist, or a root that is not a directory, is an InputError. scan() walks the root once. save_artifact writes schema_version as the first key, so a file whose first 256 bytes open with the schema_version of a registered kind (current or superseded) is an artifact and is parsed; any other file is counted by why it was skipped (no leading schema_version, an unknown one, *.tmp, *.rejected, not .json, not a regular file such as a FIFO, a symlinked directory) and never parsed. A file that cannot be read, or whose body does not hold the version its head names, is listed as unreadable with its reason. Artifacts come newest first by the time they record, by mtime when they record none. No field is shared by every kind, so dokimi/runs/kinds.py names per kind where the subject, the model and the time are; a kind without one reads as empty, never as a guess from the path. A test fails when a kind of SCHEMA_VERSIONS has no row. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/docs/architecture.md | 2 +- dokimi/dokimi/__init__.py | 2 + dokimi/dokimi/runs/__init__.py | 10 + dokimi/dokimi/runs/kinds.py | 79 ++++++ dokimi/dokimi/runs/scan.py | 247 +++++++++++++++++++ dokimi/tests/test_import_architecture.py | 4 +- dokimi/tests/test_runs_scan.py | 295 +++++++++++++++++++++++ 7 files changed, 636 insertions(+), 3 deletions(-) create mode 100644 dokimi/dokimi/runs/__init__.py create mode 100644 dokimi/dokimi/runs/kinds.py create mode 100644 dokimi/dokimi/runs/scan.py create mode 100644 dokimi/tests/test_runs_scan.py diff --git a/dokimi/docs/architecture.md b/dokimi/docs/architecture.md index 97d3e30e7..9e5cc9d4a 100644 --- a/dokimi/docs/architecture.md +++ b/dokimi/docs/architecture.md @@ -70,7 +70,7 @@ points up: | Layer | Modules | | --- | --- | | front end | `cli`, `commands`, `__main__` (the argparse front end; no layer below imports it) | -| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` | +| tracks | `runner`, `runs`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` | | evaluation | `eval`, `personas`, `questions`, `treatments`, `plugins` | | components | `ingest`, `metrics`, `gate`, `mcp`, `skills`, `agent_profiles` | | foundation | `__init__` (the `dokimi` package itself), `_version`, `artifacts`, `records`, `llm`, `process`, `container_policy`, `caching`, `config`, `registry`, `intents`, `defaults`, `console`, `utils`, `metrics/usage.py` | diff --git a/dokimi/dokimi/__init__.py b/dokimi/dokimi/__init__.py index eabdfaaaa..c8cccae2e 100644 --- a/dokimi/dokimi/__init__.py +++ b/dokimi/dokimi/__init__.py @@ -25,6 +25,8 @@ directories; [`task_formats`][dokimi.task_formats] names the format of other eval-task files and directories (detection only). - [`context`][dokimi.context] reads what a run's context window held. +- [`runs`][dokimi.runs] lists what the runs wrote: the artifacts under the runs + root (`results/`, or `DOKIMI_RUNS_DIR`). - Content novelty: [`novelty`][dokimi.novelty]. Repositories as subjects: [`repos`][dokimi.repos]. Subject scorecards, one per run: [`subjects`][dokimi.subjects]. diff --git a/dokimi/dokimi/runs/__init__.py b/dokimi/dokimi/runs/__init__.py new file mode 100644 index 000000000..9158e79b3 --- /dev/null +++ b/dokimi/dokimi/runs/__init__.py @@ -0,0 +1,10 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The runs root (``results/``, or ``DOKIMI_RUNS_DIR``) and the artifacts under it (#278).""" + +from __future__ import annotations + +from dokimi.runs.scan import Artifact, RunsRoot, ScanResult, Unreadable, runs_root, scan + +__all__ = ["Artifact", "RunsRoot", "ScanResult", "Unreadable", "runs_root", "scan"] diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py new file mode 100644 index 000000000..58babefc1 --- /dev/null +++ b/dokimi/dokimi/runs/kinds.py @@ -0,0 +1,79 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Where each artifact kind keeps its subject, model and time. + +No field is shared by every kind (a questions set names its ``product``, an arms run its +``library_name``, a novelty run its ``artifact.ref``), so :data:`FIELDS` names, per kind of +:data:`dokimi.artifacts.SCHEMA_VERSIONS`, the schema paths to read. A kind without such a field +reads as empty: nothing is guessed from the file's path. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from dokimi.artifacts import SCHEMA_VERSIONS, SUPERSEDED_VERSIONS + + +@dataclass(frozen=True) +class KindFields: + """Schema paths of one kind; each tuple is tried in order, the first that holds a value wins. + + A path is ``.``-joined keys; a key ending in ``[]`` steps into every item of a list, and the + distinct values found are all kept (several subjects of a plan, mixed products of answers). + """ + + subject: tuple[str, ...] = () + model: tuple[str, ...] = () + time: tuple[str, ...] = () + + +FIELDS: dict[str, KindFields] = { + "questions": KindFields(subject=("product", "library"), model=("model",), + time=("generated_at",)), + "answers": KindFields(subject=("answers[].library_name",), model=("model",), + time=("generated_at",)), + "eval": KindFields(model=("judge_model",), time=("evaluated_at",)), + "arms": KindFields(subject=("library_name",), model=("model",), time=("generated_at",)), + "matrix_rollup": KindFields(subject=("library_name",), time=("generated_at",)), + "plugin_delta": KindFields(subject=("library_name",), model=("model",)), + "subject_scorecard": KindFields(subject=("subject.id",), time=("generated_at",)), + "novelty": KindFields(subject=("artifact.ref",), model=("model",), time=("generated_at",)), + "task_runs": KindFields(model=("model",), time=("generated_at",)), + "explore_runs": KindFields(model=("model",), time=("generated_at",)), + "guidance_reach": KindFields(subject=("repo.id",), model=("model",), time=("generated_at",)), + "repo_readiness": KindFields(subject=("repo.id",), time=("generated_at",)), + "tool_preference": KindFields(model=("model",), time=("generated_at",)), + "repo_environment": KindFields(subject=("repo.tree",), time=("generated_at",)), + "plan": KindFields(subject=("subjects[].id",), model=("models[]",), time=("generated_at",)), + "result": KindFields(time=("generated_at",)), + "repo_budget": KindFields(subject=("tree",)), + "task_inspect": KindFields(subject=("root",)), +} + +#: Every schema_version an artifact may carry (current and superseded) -> its kind. +VERSION_KINDS: dict[str, str] = { + **{v: kind for kind, versions in SUPERSEDED_VERSIONS.items() for v in versions}, + **{v: kind for kind, v in SCHEMA_VERSIONS.items()}, +} + + +def _values(node: Any, keys: list[str]) -> list[str]: + if not keys: + return [node] if isinstance(node, str) and node else [] + key, rest = keys[0], keys[1:] + if key.endswith("[]"): + items = node.get(key[:-2]) if isinstance(node, dict) else None + return [v for item in items for v in _values(item, rest)] if isinstance(items, list) else [] + return _values(node.get(key), rest) if isinstance(node, dict) else [] + + +def read_field(data: dict, paths: tuple[str, ...]) -> tuple[str, ...]: + """The distinct non-empty strings at the first of ``paths`` that holds any, in order.""" + for path in paths: + found = tuple(dict.fromkeys(_values(data, path.split(".")))) + if found: + return found + return () diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py new file mode 100644 index 000000000..f5dd3f943 --- /dev/null +++ b/dokimi/dokimi/runs/scan.py @@ -0,0 +1,247 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The runs root and the artifacts under it, each told apart by its first bytes. + +The root is ``results/`` (relative to the working directory, where most commands write by +default) unless ``DOKIMI_RUNS_DIR`` names another. :func:`scan` walks it once: a file is an +artifact when it opens with a ``schema_version`` of a registered kind, the way +:func:`dokimi.artifacts.save_artifact` writes it, so a large file of another kind is never +parsed just to be skipped. Everything else is counted by why it was skipped, never dropped +silently. Nothing here prints or exits. +""" + +from __future__ import annotations + +import json +import os +import re +import stat +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Mapping + +from dokimi.runs.kinds import FIELDS, VERSION_KINDS, read_field +from dokimi.utils import InputError + +ENV_VAR = "DOKIMI_RUNS_DIR" +DEFAULT_ROOT = Path("results") +#: ``save_artifact`` writes ``{"schema_version": ".vN", ...`` with ``indent=2``. +HEAD_BYTES = 256 +_HEAD = re.compile(rb'\A\s*\{\s*"schema_version"\s*:\s*"([^"\\\x00-\x1f]{1,100})"') + + +@dataclass(frozen=True) +class RunsRoot: + """The directory ``runs`` reads, and whether ``DOKIMI_RUNS_DIR`` chose it.""" + + path: Path + from_env: bool + + def exists(self) -> bool: + """True when the root is a directory; False when the default root is not there yet. + + Raises :class:`~dokimi.utils.InputError` when ``DOKIMI_RUNS_DIR`` names a missing path, + or when the root is something other than a directory. + """ + if self.path.is_dir(): + return True + named = f"{ENV_VAR}={self.path}" if self.from_env else str(self.path) + if self.path.exists(): + raise InputError(f"runs root {named} is not a directory") + if self.from_env: + raise InputError(f"runs root {named} does not exist") + return False + + +def runs_root(environ: Mapping[str, str] | None = None) -> RunsRoot: + """``DOKIMI_RUNS_DIR`` when set and not empty, else ``results/``.""" + value = (os.environ if environ is None else environ).get(ENV_VAR, "") + return RunsRoot(Path(value), True) if value else RunsRoot(DEFAULT_ROOT, False) + + +@dataclass(frozen=True) +class Artifact: + """One artifact under the root: its kind and what it says it is about.""" + + path: Path # relative to the root + schema_version: str + subjects: tuple[str, ...] + models: tuple[str, ...] + generated_at: str | None + size_bytes: int + mtime: float + + @property + def kind(self) -> str: + return VERSION_KINDS[self.schema_version] + + @property + def subject(self) -> str | None: + return ", ".join(self.subjects) or None + + @property + def model(self) -> str | None: + return ", ".join(self.models) or None + + @property + def recorded_at(self) -> datetime | None: + """``generated_at`` in UTC (read as UTC if it names no zone); ``None`` if absent, not ISO + 8601 or out of range.""" + try: + stamp = datetime.fromisoformat(self.generated_at or "") + return (stamp if stamp.tzinfo else stamp.replace(tzinfo=timezone.utc)).astimezone( + timezone.utc) + except (ValueError, OverflowError): + return None + + def to_json(self) -> dict: + """The ``--json`` row. ``run_id`` is ``None``: no artifact records one yet (#277).""" + return {"schema_version": self.schema_version, "kind": self.kind, "run_id": None, + "subject": self.subject, "model": self.model, "generated_at": self.generated_at, + "path": self.path.as_posix(), "size_bytes": self.size_bytes} + + +@dataclass(frozen=True) +class Unreadable: + """A file or directory under the root that could not be read; ``schema_version`` if known.""" + + path: Path + reason: str + schema_version: str | None = None + + def to_json(self) -> dict: + return {"path": self.path.as_posix(), "schema_version": self.schema_version, + "reason": self.reason} + + +#: Why a file was skipped (the ``--json`` key) -> how ``runs list`` names it. +SKIP_REASONS = { + "no_leading_schema_version": "JSON with no leading schema_version", + "unknown_schema_version": "JSON of an unknown schema_version", + "temporary": "*.tmp (a save in progress or interrupted)", + "rejected": "*.rejected (payloads save_artifact refused)", + "not_json": "not .json", + "not_a_file": "not a regular file (a FIFO, socket or device)", + "linked_directory": "symlinked directories (not walked)", +} + + +@dataclass +class ScanResult: + """What :func:`scan` found: the artifacts, newest first, and every file it skipped.""" + + root: Path + artifacts: list[Artifact] = field(default_factory=list) + skipped: Counter[str] = field(default_factory=Counter) + unknown_versions: Counter[str] = field(default_factory=Counter) + unreadable: list[Unreadable] = field(default_factory=list) + + +def head_version(path: Path) -> str | None: + """The ``schema_version`` a file opens with, read from its first bytes; ``None`` if none.""" + with open(path, "rb") as fh: + match = _HEAD.match(fh.read(HEAD_BYTES)) + return match.group(1).decode("utf-8", errors="replace") if match else None + + +def read_artifact(root: Path, path: Path, version: str) -> Artifact: + """Parse the artifact at ``path`` whose first bytes declared ``version`` (a known one). + + Raises ``OSError`` when it cannot be read and ``ValueError`` when it is not the JSON object + its first bytes promised. + """ + stat = path.stat() + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict) or data.get("schema_version") != version: + raise ValueError(f"the document does not hold the schema_version {version} it opens with") + fields = FIELDS[VERSION_KINDS[version]] + times = read_field(data, fields.time) + return Artifact(path=path.relative_to(root), schema_version=version, + subjects=read_field(data, fields.subject), + models=read_field(data, fields.model), + generated_at=times[0] if times else None, + size_bytes=stat.st_size, mtime=stat.st_mtime) + + +def _skip_reason(name: str) -> str | None: + lower = name.lower() + if lower.endswith(".tmp"): + return "temporary" + if lower.endswith(".rejected"): + return "rejected" + return None if lower.endswith(".json") else "not_json" + + +def _reason(exc: BaseException) -> str: + if isinstance(exc, OSError) and exc.strerror: + return f"cannot read: {exc.strerror}" + if isinstance(exc, UnicodeDecodeError): + return "not UTF-8" + if isinstance(exc, json.JSONDecodeError): + return f"not valid JSON: {exc.msg} at line {exc.lineno}" + return str(exc) + + +def _scan_file(result: ScanResult, path: Path) -> None: + rel = path.relative_to(result.root) + skip = _skip_reason(path.name) + if skip: + result.skipped[skip] += 1 + return + try: + regular = stat.S_ISREG(path.stat().st_mode) # opening a FIFO would block + version = head_version(path) if regular else None + except OSError as exc: + result.unreadable.append(Unreadable(rel, _reason(exc))) + return + if not regular: + result.skipped["not_a_file"] += 1 + elif version is None: + result.skipped["no_leading_schema_version"] += 1 + elif version not in VERSION_KINDS: + result.skipped["unknown_schema_version"] += 1 + result.unknown_versions[version] += 1 + else: + try: + result.artifacts.append(read_artifact(result.root, path, version)) + except (OSError, ValueError, RecursionError) as exc: + result.unreadable.append(Unreadable(rel, _reason(exc), version)) + + +def _is_link(path: Path) -> bool: + try: + return path.is_symlink() + except OSError: # a directory listed but not searchable: os.walk reports it when it enters + return False + + +def newest_first(artifacts: list[Artifact]) -> list[Artifact]: + """By the time the artifact records (its file's mtime when it records none), then path.""" + def when(artifact: Artifact) -> float: + stamp = artifact.recorded_at + return stamp.timestamp() if stamp else artifact.mtime + + return sorted(artifacts, key=lambda a: (-when(a), a.path.as_posix())) + + +def scan(root: Path) -> ScanResult: + """Every artifact under ``root`` (a directory), newest first; symlinked dirs are not walked.""" + result = ScanResult(root=root) + + def unlisted(exc: OSError) -> None: + where = Path(exc.filename).relative_to(root) if exc.filename else Path(".") + result.unreadable.append(Unreadable(where, f"cannot list: {exc.strerror or exc}")) + + for directory, dirnames, filenames in os.walk(root, onerror=unlisted): + base = Path(directory) # os.walk does not descend into a linked directory: count it + result.skipped["linked_directory"] += sum(_is_link(base / d) for d in dirnames) + dirnames.sort() + for name in sorted(filenames): + _scan_file(result, base / name) + result.artifacts = newest_first(result.artifacts) + result.skipped = +result.skipped # drop the zero counts + return result + diff --git a/dokimi/tests/test_import_architecture.py b/dokimi/tests/test_import_architecture.py index ebeb4eb10..c7eb575be 100644 --- a/dokimi/tests/test_import_architecture.py +++ b/dokimi/tests/test_import_architecture.py @@ -51,8 +51,8 @@ "dokimi.config_check", "dokimi.context", "dokimi.coverage", "dokimi.dashboard", "dokimi.datasets_export", "dokimi.explore", "dokimi.external", "dokimi.harnesses", "dokimi.novelty", "dokimi.orchestrator", "dokimi.report", "dokimi.repos", - "dokimi.runner", "dokimi.schema_check", "dokimi.schema_compat", "dokimi.subjects", - "dokimi.task_formats", "dokimi.tool_preference", + "dokimi.runner", "dokimi.runs", "dokimi.schema_check", "dokimi.schema_compat", + "dokimi.subjects", "dokimi.task_formats", "dokimi.tool_preference", )), # The LLM evaluation engine: answering, judging, panels, arms, questions, personas. ("evaluation", ( diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py new file mode 100644 index 000000000..928a3d642 --- /dev/null +++ b/dokimi/tests/test_runs_scan.py @@ -0,0 +1,295 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The runs root scan (#278): an artifact's kind from its first bytes, its fields, and the count +of every file skipped.""" + +from __future__ import annotations + +import json +import os +import time +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from dokimi.artifacts import SCHEMA_VERSIONS, SUPERSEDED_VERSIONS, save_artifact +from dokimi.runs import kinds, runs_root, scan +from dokimi.runs.scan import HEAD_BYTES, RunsRoot +from dokimi.utils import InputError + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts" + +#: version -> (keys added to its fixture, expected subject, model, time). +EXPECTED: dict[str, tuple[dict, str | None, str | None, str | None]] = { + "questions.v2": ({"model": "gen-m", "generated_at": "2026-10-01T00:00:00Z"}, + "demo", "gen-m", "2026-10-01T00:00:00Z"), + "answers.v1": ({}, "demo", "demo-model", "2026-01-01T00:00:00Z"), + "eval.v1": ({}, None, "demo-judge", "2026-01-01T00:00:00Z"), + "arms.v1": ({"generated_at": "2026-10-02T00:00:00Z"}, "onetbb", "m", "2026-10-02T00:00:00Z"), + "matrix_rollup.v1": ({}, "oneTBB", None, "2026-10-04T17:47:16.294238Z"), + "plugin_delta.v1": ({}, "oneTBB", "m", None), + "subject_scorecard.v2": ({}, "docs-subject", None, "2026-10-04T17:47:19.175356Z"), + "novelty.v1": ({}, "skill:/tmp/run/accel-tuning/SKILL.md", "answerer-m", + "2026-10-04T17:47:16.841483Z"), + "task_runs.v1": ({}, None, "haiku", "2026-01-01T00:00:00Z"), + "explore_runs.v1": ({"model": "explorer-m"}, None, "explorer-m", "2026-10-04T17:47:15.816155Z"), + "guidance_reach.v1": ({}, "toy", "m", "2026-10-06T04:30:15.383519+00:00"), + "repo_readiness.v1": ({"generated_at": "2026-10-03T00:00:00Z"}, "toy", None, + "2026-10-03T00:00:00Z"), + "tool_preference.v1": ({}, None, "claude-haiku-5-5", "2026-10-14T00:00:00+00:00"), + "repo_environment.v1": ({}, "toy@4ff761e2d943", None, None), + "plan.v1": ({}, "skill:onetbb", "claude-sonnet-4-6", "2026-10-21T00:00:00Z"), + "result.v1": ({}, None, None, "2026-10-21T01:00:00Z"), + "repo_budget.v1": ({}, "toy", None, None), + "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), +} + + +def _fixture(version: str) -> dict: + return json.loads((FIXTURES / f"{version}.json").read_text(encoding="utf-8")) + + +def _save(root: Path, version: str, rel: str, **extra) -> Path: + path = root / rel + save_artifact(kinds.VERSION_KINDS[version], {**_fixture(version), **extra}, path) + return path + + +def test_every_kind_has_a_row_in_the_fields_table_and_in_this_test(): + assert set(kinds.FIELDS) == set(SCHEMA_VERSIONS) + assert set(EXPECTED) == set(SCHEMA_VERSIONS.values()) + + +@pytest.mark.parametrize("version", sorted(EXPECTED)) +def test_each_kind_is_listed_with_the_subject_model_and_time_it_records(tmp_path, version): + extra, subject, model, time = EXPECTED[version] + path = _save(tmp_path, version, f"some/dir/{version}.json", **extra) + + result = scan(tmp_path) + + assert result.unreadable == [] and not result.skipped + (found,) = result.artifacts + assert found.to_json() == { + "schema_version": version, "kind": kinds.VERSION_KINDS[version], "run_id": None, + "subject": subject, "model": model, "generated_at": time, + "path": f"some/dir/{version}.json", "size_bytes": path.stat().st_size, + } + + +@pytest.mark.parametrize("version", [v for vs in SUPERSEDED_VERSIONS.values() for v in vs]) +def test_a_superseded_version_is_still_an_artifact(tmp_path, version): + data = _fixture(version) + payload = {"schema_version": version, **{k: v for k, v in data.items() if k != "schema_version"}} + (tmp_path / "old.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") + + (found,) = scan(tmp_path).artifacts + + assert (found.schema_version, found.kind) == (version, kinds.VERSION_KINDS[version]) + assert found.subject == {"questions.v1": "demo", "subject_scorecard.v1": "s"}[version] + + +def test_distinct_values_of_a_list_are_all_kept_and_joined(tmp_path): + data = _fixture("answers.v1") + first = data["answers"][0] + data["answers"] = [first, {**first, "question_id": "q2", "library_name": "other"}, + {**first, "question_id": "q3"}] + save_artifact("answers", data, tmp_path / "a.json") + + (found,) = scan(tmp_path).artifacts + + assert found.subjects == ("demo", "other") + assert found.subject == "demo, other" + + +def test_every_file_that_is_not_an_artifact_is_counted_by_reason(tmp_path): + _save(tmp_path, "eval.v1", "run/eval/p.json") + (tmp_path / "run" / "eval" / "p.json.123.456.tmp").write_text("{", encoding="utf-8") + (tmp_path / "run" / "eval" / "p.json.rejected").write_text( + json.dumps({"schema_version": "eval.v1"}), encoding="utf-8") + (tmp_path / "run" / "report.md").write_text("# report", encoding="utf-8") + (tmp_path / "dashboard.json").write_text(json.dumps({"libraries": []}), encoding="utf-8") + (tmp_path / "pairwise.json").write_text(json.dumps({"schema_version": "pairwise.v0"}), + encoding="utf-8") + (tmp_path / "checklist.json").write_text(json.dumps({"schema_version": "checklist.v0"}), + encoding="utf-8") + (tmp_path / "broken.json").write_text('{"schema_version": "answers.v1", "answers": [', + encoding="utf-8") + (tmp_path / "late.json").write_text(json.dumps({"evaluations": [], "schema_version": "eval.v1"}), + encoding="utf-8") + + result = scan(tmp_path) + + assert [a.path.as_posix() for a in result.artifacts] == ["run/eval/p.json"] + assert result.skipped == {"temporary": 1, "rejected": 1, "not_json": 1, + "no_leading_schema_version": 2, "unknown_schema_version": 2} + assert result.unknown_versions == {"pairwise.v0": 1, "checklist.v0": 1} + (bad,) = result.unreadable + assert (bad.path.as_posix(), bad.schema_version) == ("broken.json", "answers.v1") + assert bad.reason.startswith("not valid JSON: ") + + +def test_a_file_that_is_not_utf8_is_unreadable_with_its_reason(tmp_path): + (tmp_path / "x.json").write_bytes(b'{"schema_version": "eval.v1", "judge_model": "\xff"}') + + (bad,) = scan(tmp_path).unreadable + + assert (bad.reason, bad.schema_version) == ("not UTF-8", "eval.v1") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_symlinked_directory_is_counted_and_not_walked(tmp_path): + _save(tmp_path, "eval.v1", "run/eval.json") + (tmp_path / "run" / "loop").symlink_to(tmp_path, target_is_directory=True) + (tmp_path / "self.json").symlink_to(tmp_path / "self.json") + + result = scan(tmp_path) + + assert [a.path.as_posix() for a in result.artifacts] == ["run/eval.json"] + assert result.skipped == {"linked_directory": 1} + (bad,) = result.unreadable + assert bad.path.as_posix() == "self.json" and bad.reason.startswith("cannot read: ") + + +def test_the_kind_comes_from_the_first_bytes_and_the_rest_is_not_parsed_to_skip(tmp_path): + # The tail is broken: a skipped file was never parsed, a file of a known kind was. + tail = "x" * HEAD_BYTES + "{not json" + (tmp_path / "big.json").write_text('{"libraries": "' + tail, encoding="utf-8") + (tmp_path / "eval.json").write_text('{"schema_version": "eval.v1", "x": "' + tail, + encoding="utf-8") + + result = scan(tmp_path) + + assert result.skipped == {"no_leading_schema_version": 1} + (bad,) = result.unreadable + assert (bad.path.as_posix(), bad.schema_version) == ("eval.json", "eval.v1") + + +def test_a_head_that_names_one_version_and_a_body_that_holds_another_is_unreadable(tmp_path): + (tmp_path / "x.json").write_text('{"schema_version": "eval.v1", "schema_version": "arms.v1"}', + encoding="utf-8") + + (bad,) = scan(tmp_path).unreadable + + assert "does not hold the schema_version eval.v1" in bad.reason + + +def test_artifacts_are_newest_first_by_their_time_then_by_mtime(tmp_path): + _save(tmp_path, "eval.v1", "old.json", evaluated_at="2026-01-01T00:00:00Z") + _save(tmp_path, "eval.v1", "new.json", evaluated_at="2026-10-01T00:00:00+00:00") + _save(tmp_path, "eval.v1", "naive.json", evaluated_at="2026-06-01T00:00:00") + no_time = _save(tmp_path, "plugin_delta.v1", "no_time.json") + stamp = 1_780_000_000 # 2026-05-28, between old and naive + os.utime(no_time, (stamp, stamp)) + + paths = [a.path.as_posix() for a in scan(tmp_path).artifacts] + + assert paths == ["new.json", "naive.json", "no_time.json", "old.json"] + + +@pytest.mark.skipif(not hasattr(time, "tzset"), reason="TZ is read by time.tzset, POSIX only") +def test_a_time_that_names_no_zone_is_utc_whatever_the_local_zone(tmp_path, monkeypatch): + _save(tmp_path, "eval.v1", "e.json", evaluated_at="2026-06-01T09:30:00") + monkeypatch.setenv("TZ", "Asia/Kolkata") + time.tzset() + try: + (found,) = scan(tmp_path).artifacts + finally: + monkeypatch.undo() + time.tzset() + + assert found.recorded_at == datetime(2026, 6, 1, 9, 30, tzinfo=timezone.utc) + + +def test_dokimi_runs_dir_overrides_the_default_root(tmp_path): + assert runs_root({}) == RunsRoot(Path("results"), from_env=False) + assert runs_root({"DOKIMI_RUNS_DIR": ""}) == RunsRoot(Path("results"), from_env=False) + assert runs_root({"DOKIMI_RUNS_DIR": str(tmp_path)}) == RunsRoot(tmp_path, from_env=True) + + +def test_a_missing_default_root_is_no_runs_and_a_bad_named_root_is_an_input_error(tmp_path): + assert RunsRoot(tmp_path / "results", from_env=False).exists() is False + assert RunsRoot(tmp_path, from_env=True).exists() is True + with pytest.raises(InputError, match="DOKIMI_RUNS_DIR=.* does not exist"): + RunsRoot(tmp_path / "nope", from_env=True).exists() + (tmp_path / "file").write_text("", encoding="utf-8") + with pytest.raises(InputError, match="is not a directory"): + RunsRoot(tmp_path / "file", from_env=True).exists() + with pytest.raises(InputError, match="is not a directory"): + RunsRoot(tmp_path / "file", from_env=False).exists() + + +def test_upper_case_suffixes_are_matched_too(tmp_path): + _save(tmp_path, "eval.v1", "E.JSON") + (tmp_path / "x.json.1.2.TMP").write_text("{", encoding="utf-8") + + result = scan(tmp_path) + + assert [a.path.as_posix() for a in result.artifacts] == ["E.JSON"] + assert result.skipped == {"temporary": 1} + + +def test_a_schema_version_past_the_first_bytes_is_not_a_leading_one(tmp_path): + assert HEAD_BYTES < 1000 + padded = "{" + " " * 1000 + '"schema_version": "eval.v1", "evaluations": []}' + (tmp_path / "padded.json").write_text(padded, encoding="utf-8") + + assert scan(tmp_path).skipped == {"no_leading_schema_version": 1} + + +def test_a_document_nested_too_deep_to_parse_is_unreadable(tmp_path): + deep = '{"schema_version": "eval.v1", "x": ' + "[" * 100_000 + "]" * 100_000 + "}" + (tmp_path / "deep.json").write_text(deep, encoding="utf-8") + + (bad,) = scan(tmp_path).unreadable + + assert bad.schema_version == "eval.v1" and "recursion" in bad.reason + + +def test_a_time_out_of_range_falls_back_to_the_mtime(tmp_path): + early = _save(tmp_path, "eval.v1", "early.json", evaluated_at="0001-01-01T00:00:00+05:00") + late = _save(tmp_path, "eval.v1", "late.json", evaluated_at="9999-12-31T23:00:00-05:00") + os.utime(early, (1_780_000_000, 1_780_000_000)) + os.utime(late, (1_790_000_000, 1_790_000_000)) + + found = scan(tmp_path).artifacts + + assert [(a.path.as_posix(), a.recorded_at) for a in found] == [("late.json", None), + ("early.json", None)] + + +def test_unreadable_files_come_in_path_order(tmp_path): + names = [f"{d}/{i:02}.json" for d in ("b", "a") for i in range(20, 0, -1)] + for rel in names: + (tmp_path / rel).parent.mkdir(exist_ok=True) + (tmp_path / rel).write_text('{"schema_version": "eval.v1", ', encoding="utf-8") + + paths = [u.path.as_posix() for u in scan(tmp_path).unreadable] + + assert paths == sorted(names) + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFOs are POSIX") +def test_a_fifo_is_skipped_without_being_opened(tmp_path): + os.mkfifo(tmp_path / "pipe.json") # opening it for reading would block the scan + + assert scan(tmp_path).skipped == {"not_a_file": 1} + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root reads every directory") +@pytest.mark.parametrize("mode", [0o000, 0o444], ids=["unreadable", "not-searchable"]) +def test_a_directory_that_cannot_be_walked_is_unreadable_not_a_crash(tmp_path, mode): + _save(tmp_path, "eval.v1", "ok.json") + (tmp_path / "locked" / "sub").mkdir(parents=True) + (tmp_path / "locked").chmod(mode) + try: + result = scan(tmp_path) + finally: + (tmp_path / "locked").chmod(0o755) + + assert [a.path.as_posix() for a in result.artifacts] == ["ok.json"] + (bad,) = result.unreadable + expected = "locked" if mode == 0o000 else "locked/sub" + assert (bad.path.as_posix(), bad.reason) == (expected, "cannot list: Permission denied") From 60a1424e14d46dbf82012383bab4ac2fb2d72b89 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 06:31:52 -0700 Subject: [PATCH 02/18] feat(runs): runs list dokimi runs list prints one row per artifact under the runs root: schema_version, subject, model, recorded time (UTC, to the minute; the file's mtime marked * when the artifact records none) and the path relative to the root, newest first. Under the table a summary counts the artifacts, every file skipped by reason (unknown schema_versions by name) and the unreadable files with the first one's reason. --kind KIND (repeatable, a kind of SCHEMA_VERSIONS) and --subject (an exact match on one of the artifact's subjects) filter the rows. --json prints the rows with the #94 field names (schema_version, kind, run_id, subject, model, generated_at, path, size_bytes; run_id is null until #277) plus root, total, skipped, unknown_versions and unreadable. A missing default results/ prints "no runs" and exits 0. A DOKIMI_RUNS_DIR that does not exist or is not a directory, or a results/ that is not a directory, exits 2 through InputError. --subject is a new local flag, allowlisted with its reason; --kind was already on the list. The CLI snapshot is regenerated. Refs #278 Signed-off-by: Vladislav Perevezentsev --- ...030_120000_vlad-perevezentsev_runs-list.md | 18 ++ dokimi/docs/reference/cli-interface.json | 116 ++++++++++++ dokimi/dokimi/cli.py | 2 + dokimi/dokimi/commands/runs.py | 112 +++++++++++ dokimi/dokimi/runs/scan.py | 6 + dokimi/tests/data/cli_flags_allowlist.txt | 1 + dokimi/tests/test_runs_cli.py | 175 ++++++++++++++++++ 7 files changed, 430 insertions(+) create mode 100644 dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md create mode 100644 dokimi/dokimi/commands/runs.py create mode 100644 dokimi/tests/test_runs_cli.py diff --git a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md new file mode 100644 index 000000000..8d342d8d1 --- /dev/null +++ b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md @@ -0,0 +1,18 @@ + + +### Added + +- **`dokimi runs list` lists the artifacts under the runs root** โ€” the root is + `results/` in the working directory, or `DOKIMI_RUNS_DIR`. One row per artifact: + its `schema_version`, subject, model, recorded time and path, newest first; + `--kind` (repeatable) and `--subject` (an exact match) filter the rows, and + `--json` prints them with the fields `schema_version`, `kind`, `run_id` (always + `null` until artifacts carry one), `subject`, `model`, `generated_at`, `path` + and `size_bytes`. A file is an artifact when it opens with the + `schema_version` of a registered kind; every other file is counted by why it + was skipped, and a file that cannot be read is listed with its reason. A + missing `results/` prints "no runs" and exits 0; a `DOKIMI_RUNS_DIR` that does + not exist or is not a directory exits 2. The command only reads. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 3778ecf3d..c18684a80 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -12597,6 +12597,122 @@ "subcommand_required": null, "subcommands": {} }, + "runs": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi runs", + "subcommand_required": true, + "subcommands": { + "list": { + "allow_abbrev": true, + "params": [ + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_AppendAction", + "choices": [ + "'answers'", + "'arms'", + "'eval'", + "'explore_runs'", + "'guidance_reach'", + "'matrix_rollup'", + "'novelty'", + "'plan'", + "'plugin_delta'", + "'questions'", + "'repo_budget'", + "'repo_environment'", + "'repo_readiness'", + "'result'", + "'subject_scorecard'", + "'task_inspect'", + "'task_runs'", + "'tool_preference'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--kind", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--subject", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi runs list", + "subcommand_required": null, + "subcommands": {} + } + } + }, "subjects": { "allow_abbrev": true, "params": [ diff --git a/dokimi/dokimi/cli.py b/dokimi/dokimi/cli.py index 32ee6e64c..15a677a9b 100755 --- a/dokimi/dokimi/cli.py +++ b/dokimi/dokimi/cli.py @@ -31,6 +31,7 @@ repo, report, run, + runs, subjects, tasks, tools, @@ -81,6 +82,7 @@ def positive_int(value: str) -> int: repo.register(sub, positive_int) # repo (repositories as subjects) tools.register(sub, positive_int) # tools (tool-choice probes) context.register(sub, positive_int) # context (context window of a run) + runs.register(sub, positive_int) # runs (the artifacts under the runs root) return p diff --git a/dokimi/dokimi/commands/runs.py b/dokimi/dokimi/commands/runs.py new file mode 100644 index 000000000..3887a4c91 --- /dev/null +++ b/dokimi/dokimi/commands/runs.py @@ -0,0 +1,112 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""runs subcommand group: the artifacts under the runs root (#278). Reads only; never spends.""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone + +from dokimi.artifacts import SCHEMA_VERSIONS +from dokimi.runs.scan import SKIP_REASONS, Artifact, ScanResult, runs_root, scan, select + + +def _plain(text: str) -> str: + """``text`` with control characters escaped: a file name or field cannot split a row or + reach the terminal as an escape sequence.""" + return "".join(c if c.isprintable() else c.encode("unicode_escape").decode("ascii") + for c in text) + + +def _untimed(artifact: Artifact) -> bool: + return artifact.recorded_at is None and not artifact.generated_at + + +def _when(artifact: Artifact) -> str: + """The recorded time in UTC to the minute; the file's mtime, marked ``*``, when it has none.""" + stamp = artifact.recorded_at + if stamp: + return stamp.strftime("%Y-%m-%d %H:%M") + if artifact.generated_at: # not ISO 8601, or out of range: shown as written + return artifact.generated_at + return datetime.fromtimestamp(artifact.mtime, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + "*" + + +def _count(n: int, noun: str) -> str: + return f"{n} {noun}" if n == 1 else f"{n} {noun}s" + + +def _table(artifacts: list[Artifact]) -> list[str]: + rows: list[tuple[str, ...]] = [("SCHEMA", "SUBJECT", "MODEL", "TIME (UTC)", "PATH")] + rows += [tuple(_plain(cell) for cell in (a.schema_version, a.subject or "-", a.model or "-", + _when(a), a.path.as_posix())) + for a in artifacts] + widths = [max(len(row[i]) for row in rows) for i in range(4)] + return [" ".join(row[i].ljust(w) for i, w in enumerate(widths)) + " " + row[4] + for row in rows] + + +def _summary(result: ScanResult, shown: list[Artifact], root: str) -> list[str]: + total = _count(len(result.artifacts), "artifact") + lines = [f"{len(shown)} of {total} under {root} match" if len(shown) != len(result.artifacts) + else f"{total} under {root}"] + if any(_untimed(a) for a in shown): + lines[0] += " (* the file's mtime: the artifact records no time)" + labels = {**SKIP_REASONS, "unknown_schema_version": SKIP_REASONS["unknown_schema_version"] + + " (" + ", ".join(f"{v}: {n}" for v, n in sorted(result.unknown_versions.items())) + + ")"} + skipped = [f"{result.skipped[reason]} {label}" for reason, label in labels.items() + if result.skipped[reason]] + if skipped: + lines.append("skipped: " + ", ".join(skipped)) + if result.unreadable: + first = result.unreadable[0] + lines.append(f"unreadable: {len(result.unreadable)} " + f"(first: {_plain(first.path.as_posix())}: {_plain(first.reason)})") + return lines + + +def cmd_runs_list(args: argparse.Namespace) -> None: + """One row per artifact under the runs root, newest first, and the count of what was skipped.""" + root = runs_root() + exists = root.exists() + result = scan(root.path) if exists else ScanResult(root=root.path) + shown = select(result.artifacts, args.kind, args.subject) + if args.json: + print(json.dumps({ + "root": root.path.as_posix(), "artifacts": [a.to_json() for a in shown], + "total": len(result.artifacts), "skipped": dict(sorted(result.skipped.items())), + "unknown_versions": dict(sorted(result.unknown_versions.items())), + "unreadable": [u.to_json() for u in result.unreadable], + }, indent=2)) + return + shown_root = _plain(root.path.as_posix().rstrip("/") + "/") + if not exists: + print(f"no runs under {shown_root} (it does not exist; DOKIMI_RUNS_DIR names another root)") + return + if shown: + print("\n".join(_table(shown))) + print() + print("\n".join(_summary(result, shown, shown_root))) + + +def register(sub: argparse._SubParsersAction, positive_int) -> None: + runs_p = sub.add_parser( + "runs", help="The artifacts under the runs root: results/, or DOKIMI_RUNS_DIR") + runs_sub = runs_p.add_subparsers(dest="runs_cmd", required=True) + + list_p = runs_sub.add_parser( + "list", + help="One row per artifact under the runs root (kind, subject, model, time, path), " + "newest first, and the count of every file skipped. Reads only.", + ) + list_p.add_argument("--kind", action="append", choices=sorted(SCHEMA_VERSIONS), metavar="KIND", + help="Only artifacts of this kind (eval, arms, novelty, ...); repeatable") + list_p.add_argument("--subject", + help="Only artifacts whose subject is exactly this (a product, a repo id, " + "a skill ref)") + list_p.add_argument("--json", action="store_true", + help="Print the artifacts and the skip counts as JSON") + list_p.set_defaults(func=cmd_runs_list) diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py index f5dd3f943..ace7a690a 100644 --- a/dokimi/dokimi/runs/scan.py +++ b/dokimi/dokimi/runs/scan.py @@ -245,3 +245,9 @@ def unlisted(exc: OSError) -> None: result.skipped = +result.skipped # drop the zero counts return result + +def select(artifacts: list[Artifact], kinds: list[str] | None = None, + subject: str | None = None) -> list[Artifact]: + """The artifacts of one of ``kinds`` (all when none) that name exactly ``subject`` (if given).""" + return [a for a in artifacts + if (not kinds or a.kind in kinds) and (subject is None or subject in a.subjects)] diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index 6e9ed384d..aa78c59d7 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -185,6 +185,7 @@ --static --stats-correction --strict +--subject # runs list (#278) adds it: the #94 ยง2.3 filter; no registry concept yet --subjects-root --swe-bench --target diff --git a/dokimi/tests/test_runs_cli.py b/dokimi/tests/test_runs_cli.py new file mode 100644 index 000000000..7f32d6b4f --- /dev/null +++ b/dokimi/tests/test_runs_cli.py @@ -0,0 +1,175 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`runs list` (#278): exit codes of the root, filters, order, the --json fields and the summary.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest + +from dokimi import cli +from dokimi.artifacts import save_artifact + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts" + + +def _fixture(version: str) -> dict: + return json.loads((FIXTURES / f"{version}.json").read_text(encoding="utf-8")) + + +@pytest.fixture +def root(tmp_path, monkeypatch): + """An empty default root, ``results/`` in the working directory.""" + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + (tmp_path / "results").mkdir() + return tmp_path / "results" + + +@pytest.fixture +def filled(root): + save_artifact("arms", {**_fixture("arms.v1"), "generated_at": "2026-10-29T14:02:00Z"}, + root / "arms" / "onetbb.json") + save_artifact("plugin_delta", _fixture("plugin_delta.v1"), root / "arms" / "plugin_delta.json") + save_artifact("eval", _fixture("eval.v1"), root / "onetbb_r1" / "eval" / "onetbb.json") + save_artifact("questions", {**_fixture("questions.v2"), "product": "oneTBB", + "generated_at": "2026-10-20T09:00:00+02:00"}, + root / "onetbb_r1" / "questions" / "oneTBB.json") + os.utime(root / "arms" / "plugin_delta.json", (1_780_000_000, 1_780_000_000)) # 2026-05-28 + (root / "arms" / "onetbb.md").write_text("# arms", encoding="utf-8") + (root / "pairwise.json").write_text(json.dumps({"schema_version": "pairwise.v0"}), + encoding="utf-8") + (root / "broken.json").write_text('{"schema_version": "eval.v1", "evaluations": [', + encoding="utf-8") + return root + + +def _list(monkeypatch, capsys, *argv: str): + monkeypatch.setattr("sys.argv", ["dokimi", "runs", "list", *argv]) + cli.main() + return capsys.readouterr() + + +def _paths(out: str) -> list[str]: + rows = out.split("\n\n")[0].splitlines()[1:] + return [row.split()[-1] for row in rows] + + +def test_a_missing_default_root_is_no_runs_and_exits_0(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + + out = _list(monkeypatch, capsys).out + assert out == "no runs under results/ (it does not exist; DOKIMI_RUNS_DIR names another root)\n" + data = json.loads(_list(monkeypatch, capsys, "--json").out) + assert (data["root"], data["artifacts"], data["total"]) == ("results", [], 0) + + +@pytest.mark.parametrize("make, message", [ + (lambda p: None, "does not exist"), + (lambda p: p.write_text("", encoding="utf-8"), "is not a directory"), +], ids=["missing", "a-file"]) +def test_a_bad_dokimi_runs_dir_exits_2_with_a_message(tmp_path, monkeypatch, capsys, make, message): + named = tmp_path / "named" + make(named) + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(named)) + + with pytest.raises(SystemExit) as exc: + _list(monkeypatch, capsys) + + assert exc.value.code == 2 + err = capsys.readouterr().err + assert err == f"Error: runs root DOKIMI_RUNS_DIR={named} {message}\n" + + +def test_dokimi_runs_dir_is_the_root_listed(tmp_path, monkeypatch, capsys, root): + other = tmp_path / "elsewhere" + save_artifact("eval", _fixture("eval.v1"), other / "e.json") + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(other)) + + out = _list(monkeypatch, capsys).out + + assert _paths(out) == ["e.json"] + assert f"1 artifact under {other.as_posix()}/" in out + + +def test_an_empty_root_lists_nothing_and_says_so(monkeypatch, capsys, root): + assert _list(monkeypatch, capsys).out == "0 artifacts under results/\n" + + +def test_rows_are_newest_first_by_recorded_time_then_mtime(monkeypatch, capsys, filled): + out = _list(monkeypatch, capsys).out + + assert _paths(out) == ["arms/onetbb.json", "onetbb_r1/questions/oneTBB.json", + "arms/plugin_delta.json", "onetbb_r1/eval/onetbb.json"] + lines = out.splitlines() + assert lines[0].split() == ["SCHEMA", "SUBJECT", "MODEL", "TIME", "(UTC)", "PATH"] + assert lines[2].split()[:5] == ["questions.v2", "oneTBB", "-", "2026-10-20", "07:00"] + assert lines[3].split()[3] == "2026-05-28" and lines[3].split()[4].endswith("*") + + +def test_kind_is_repeatable_and_subject_matches_exactly(monkeypatch, capsys, filled): + out = _list(monkeypatch, capsys, "--kind", "arms", "--kind", "eval").out + assert _paths(out) == ["arms/onetbb.json", "onetbb_r1/eval/onetbb.json"] + assert "2 of 4 artifacts under results/ match" in out + + out = _list(monkeypatch, capsys, "--subject", "oneTBB").out + assert _paths(out) == ["onetbb_r1/questions/oneTBB.json", "arms/plugin_delta.json"] + + out = _list(monkeypatch, capsys, "--subject", "oneTBB", "--kind", "plugin_delta").out + assert _paths(out) == ["arms/plugin_delta.json"] + + assert _list(monkeypatch, capsys, "--subject", "onet").out.startswith("0 of 4 artifacts") + + +def test_json_has_the_vocabulary_fields_and_every_count(monkeypatch, capsys, filled): + data = json.loads(_list(monkeypatch, capsys, "--json", "--kind", "arms").out) + + assert data["artifacts"] == [{ + "schema_version": "arms.v1", "kind": "arms", "run_id": None, "subject": "onetbb", + "model": "m", "generated_at": "2026-10-29T14:02:00Z", "path": "arms/onetbb.json", + "size_bytes": (filled / "arms" / "onetbb.json").stat().st_size, + }] + assert data["root"] == "results" and data["total"] == 4 + assert data["skipped"] == {"not_json": 1, "unknown_schema_version": 1} + assert data["unknown_versions"] == {"pairwise.v0": 1} + (bad,) = data["unreadable"] + assert (bad["path"], bad["schema_version"]) == ("broken.json", "eval.v1") + assert bad["reason"].startswith("not valid JSON: ") + + +def test_the_text_summary_counts_what_was_skipped_and_names_the_first_unreadable( + monkeypatch, capsys, filled): + (filled / "late.json").write_text(json.dumps({"x": 1, "schema_version": "eval.v1"}), + encoding="utf-8") + (filled / "arms" / "onetbb.json.rejected").write_text("{}", encoding="utf-8") + + summary = _list(monkeypatch, capsys).out.split("\n\n")[1].splitlines() + + assert summary == [ + "4 artifacts under results/ (* the file's mtime: the artifact records no time)", + "skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_version " + "(pairwise.v0: 1), 1 *.rejected (payloads save_artifact refused), 1 not .json", + "unreadable: 1 (first: broken.json: not valid JSON: Expecting value at line 1)", + ] + + +def test_the_mtime_footnote_is_there_only_when_a_shown_row_has_a_star(monkeypatch, capsys, filled): + out = _list(monkeypatch, capsys, "--kind", "arms").out + + assert out.split("\n\n")[1].splitlines()[0] == "1 of 4 artifacts under results/ match" + + +@pytest.mark.skipif(os.name == "nt", reason="Windows file names cannot hold control characters") +def test_control_characters_in_names_and_fields_are_escaped(monkeypatch, capsys, root): + save_artifact("arms", {**_fixture("arms.v1"), "library_name": "evil\x1b[2J", + "generated_at": "2026-10-29T14:02:00Z"}, root / "two\nlines.json") + + rows = _list(monkeypatch, capsys).out.split("\n\n")[0].splitlines() + + assert len(rows) == 2 + assert rows[1].split()[1] == "evil\\x1b[2J" and rows[1].endswith("two\\nlines.json") From 1a94525d47144de205aec07643ca90bcc9574ae1 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 06:36:30 -0700 Subject: [PATCH 03/18] feat(runs): runs show dokimi runs show PATH prints one artifact: its path, kind and schema_version, the result of its schema check, subject, model, time (the file's mtime when the artifact records none), size, the length of the lists it holds and the entries next to it, each .json one with the schema_version it opens with (the first 20 named, the rest counted). PATH is the path runs list prints, relative to the root, or an absolute or working-directory path that lies inside the root. The document is checked against the schema of the version it declares, so a superseded version is checked against its own schema. A failed check prints "schema: invalid, N errors" and every error and exits 0: showing is not a gate. A path outside the root, a missing file, a directory, an entry that is not a regular file (a FIFO), a file that does not open with the schema_version of a registered kind, or one that does not parse, exits 2. Control characters in the printed fields are escaped. --json prints the runs list row fields plus root, schema, schema_valid, schema_errors, contents, siblings, siblings_total and siblings_error. The kinds table gains the lists to count per kind; a test checks that each is a top-level array of the kind's schema. Refs #278 Signed-off-by: Vladislav Perevezentsev --- ...030_120000_vlad-perevezentsev_runs-list.md | 10 + dokimi/docs/reference/cli-interface.json | 56 ++++ dokimi/dokimi/commands/runs.py | 61 ++++ dokimi/dokimi/runs/kinds.py | 56 ++-- dokimi/dokimi/runs/scan.py | 22 +- dokimi/dokimi/runs/show.py | 125 ++++++++ dokimi/tests/test_runs_show.py | 266 ++++++++++++++++++ 7 files changed, 571 insertions(+), 25 deletions(-) create mode 100644 dokimi/dokimi/runs/show.py create mode 100644 dokimi/tests/test_runs_show.py diff --git a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md index 8d342d8d1..6fdaa2ec4 100644 --- a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md +++ b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md @@ -16,3 +16,13 @@ SPDX-License-Identifier: Apache-2.0 was skipped, and a file that cannot be read is listed with its reason. A missing `results/` prints "no runs" and exits 0; a `DOKIMI_RUNS_DIR` that does not exist or is not a directory exits 2. The command only reads. +- **`dokimi runs show PATH` shows one artifact** โ€” `PATH` as `runs list` prints + it (relative to the root), or an absolute or working-directory path inside the + root. It prints the artifact's kind, its fields, the length of the lists it + holds (answers, evaluations, cells, ...) and the files next to it, and checks + it against its schema: an artifact that fails the check is shown with + "schema: invalid, N errors" and every error, and the command still exits 0. + A path outside the root, a missing file, a directory, or a file that is not a + readable artifact of a registered kind exits 2. `--json` prints the same + fields as a `runs list` row plus `schema`, `schema_valid`, `schema_errors`, + `contents` and the neighbours. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c18684a80..7f92a5338 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -12710,6 +12710,62 @@ "path": "dokimi runs list", "subcommand_required": null, "subcommands": {} + }, + "show": { + "allow_abbrev": true, + "params": [ + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "path", + "kind": "argument", + "nargs": null, + "position": 0, + "required": true, + "type": null + }, + { + "action": "_HelpAction", + "choices": null, + "const": null, + "default": "==SUPPRESS==", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--help/-h", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--json", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + } + ], + "path": "dokimi runs show", + "subcommand_required": null, + "subcommands": {} } } }, diff --git a/dokimi/dokimi/commands/runs.py b/dokimi/dokimi/commands/runs.py index 3887a4c91..f508e9b8a 100644 --- a/dokimi/dokimi/commands/runs.py +++ b/dokimi/dokimi/commands/runs.py @@ -11,6 +11,8 @@ from dokimi.artifacts import SCHEMA_VERSIONS from dokimi.runs.scan import SKIP_REASONS, Artifact, ScanResult, runs_root, scan, select +from dokimi.runs.show import Shown, show +from dokimi.utils import InputError def _plain(text: str) -> str: @@ -92,6 +94,54 @@ def cmd_runs_list(args: argparse.Namespace) -> None: print("\n".join(_summary(result, shown, shown_root))) +def _show_json(shown: Shown, root: str) -> dict: + return {**shown.artifact.to_json(), "root": root, "schema": shown.schema, + "schema_valid": not shown.errors, "schema_errors": shown.errors, + "contents": shown.contents, + "siblings": [{"name": s.name, "schema_version": s.schema_version} + for s in shown.siblings], + "siblings_total": shown.siblings_total, "siblings_error": shown.siblings_error} + + +def _show_text(shown: Shown) -> list[str]: + a = shown.artifact + errors = _count(len(shown.errors), "error") + when = a.generated_at or "- (" + _when(a).rstrip("*") + " UTC, the file's mtime)" + beside = [s.name + (f" ({s.schema_version})" if s.schema_version else "") + for s in shown.siblings] + if shown.siblings_total > len(shown.siblings): + beside.append(f"and {shown.siblings_total - len(shown.siblings)} more") + rows = [ + ("path", a.path.as_posix()), + ("kind", f"{a.kind} ({a.schema_version})"), + ("schema", f"invalid, {errors}" if shown.errors else "valid"), + ("subject", a.subject or "-"), + ("model", a.model or "-"), + ("time", when), + ("run_id", "- (no artifact records one yet)"), + ("size", f"{a.size_bytes:,} bytes"), + ("contents", ", ".join(f"{key}: {n}" for key, n in shown.contents.items()) or "-"), + ("next to it", shown.siblings_error or ", ".join(beside) or "-"), + ] + lines = [f"{label + ':':<12}{_plain(value)}" for label, value in rows] + lines[3:3] = [f" - {_plain(e)}" for e in shown.errors] + return lines + + +def cmd_runs_show(args: argparse.Namespace) -> None: + """One artifact: its kind, schema check, fields, contents and neighbours. A failed schema + check is printed, and the exit code stays 0.""" + root = runs_root() + if not root.exists(): + raise InputError(f"no runs root: {root.path.as_posix()}/ does not exist " + f"(DOKIMI_RUNS_DIR names another root)") + shown = show(root.path, args.path) + if args.json: + print(json.dumps(_show_json(shown, root.path.as_posix()), indent=2)) + else: + print("\n".join(_show_text(shown))) + + def register(sub: argparse._SubParsersAction, positive_int) -> None: runs_p = sub.add_parser( "runs", help="The artifacts under the runs root: results/, or DOKIMI_RUNS_DIR") @@ -110,3 +160,14 @@ def register(sub: argparse._SubParsersAction, positive_int) -> None: list_p.add_argument("--json", action="store_true", help="Print the artifacts and the skip counts as JSON") list_p.set_defaults(func=cmd_runs_list) + + show_p = runs_sub.add_parser( + "show", + help="One artifact: its kind, whether it meets its schema (a failed check is printed and " + "exits 0), subject, model, time, what it counts and the files next to it. Reads only.", + ) + show_p.add_argument("path", help="The artifact: its path as runs list prints it (relative to " + "the root), or an absolute path inside the root") + show_p.add_argument("--json", action="store_true", + help="Print the artifact's fields, schema errors and neighbours as JSON") + show_p.set_defaults(func=cmd_runs_show) diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index 58babefc1..5e80e4945 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Where each artifact kind keeps its subject, model and time. +"""Where each artifact kind keeps its subject, model and time, and what its contents count. No field is shared by every kind (a questions set names its ``product``, an arms run its ``library_name``, a novelty run its ``artifact.ref``), so :data:`FIELDS` names, per kind of @@ -23,34 +23,47 @@ class KindFields: A path is ``.``-joined keys; a key ending in ``[]`` steps into every item of a list, and the distinct values found are all kept (several subjects of a plan, mixed products of answers). + ``counts`` are top-level lists whose length ``runs show`` prints. """ subject: tuple[str, ...] = () model: tuple[str, ...] = () time: tuple[str, ...] = () + counts: tuple[str, ...] = () FIELDS: dict[str, KindFields] = { "questions": KindFields(subject=("product", "library"), model=("model",), - time=("generated_at",)), + time=("generated_at",), counts=("questions",)), "answers": KindFields(subject=("answers[].library_name",), model=("model",), - time=("generated_at",)), - "eval": KindFields(model=("judge_model",), time=("evaluated_at",)), - "arms": KindFields(subject=("library_name",), model=("model",), time=("generated_at",)), - "matrix_rollup": KindFields(subject=("library_name",), time=("generated_at",)), - "plugin_delta": KindFields(subject=("library_name",), model=("model",)), - "subject_scorecard": KindFields(subject=("subject.id",), time=("generated_at",)), - "novelty": KindFields(subject=("artifact.ref",), model=("model",), time=("generated_at",)), - "task_runs": KindFields(model=("model",), time=("generated_at",)), - "explore_runs": KindFields(model=("model",), time=("generated_at",)), - "guidance_reach": KindFields(subject=("repo.id",), model=("model",), time=("generated_at",)), - "repo_readiness": KindFields(subject=("repo.id",), time=("generated_at",)), - "tool_preference": KindFields(model=("model",), time=("generated_at",)), - "repo_environment": KindFields(subject=("repo.tree",), time=("generated_at",)), - "plan": KindFields(subject=("subjects[].id",), model=("models[]",), time=("generated_at",)), - "result": KindFields(time=("generated_at",)), - "repo_budget": KindFields(subject=("tree",)), - "task_inspect": KindFields(subject=("root",)), + time=("generated_at",), counts=("answers",)), + "eval": KindFields(model=("judge_model",), time=("evaluated_at",), counts=("evaluations",)), + "arms": KindFields(subject=("library_name",), model=("model",), time=("generated_at",), + counts=("arms", "answers")), + "matrix_rollup": KindFields(subject=("library_name",), time=("generated_at",), + counts=("cells",)), + "plugin_delta": KindFields(subject=("library_name",), model=("model",), counts=("arms",)), + "subject_scorecard": KindFields(subject=("subject.id",), time=("generated_at",), + counts=("cells",)), + "novelty": KindFields(subject=("artifact.ref",), model=("model",), time=("generated_at",), + counts=("conflicts", "warnings")), + "task_runs": KindFields(model=("model",), time=("generated_at",), + counts=("tasks", "harnesses", "results")), + "explore_runs": KindFields(model=("model",), time=("generated_at",), + counts=("tasks", "arms", "results")), + "guidance_reach": KindFields(subject=("repo.id",), model=("model",), time=("generated_at",), + counts=("files", "scenarios", "records")), + "repo_readiness": KindFields(subject=("repo.id",), time=("generated_at",), + counts=("files", "claims")), + "tool_preference": KindFields(model=("model",), time=("generated_at",), + counts=("items", "results")), + "repo_environment": KindFields(subject=("repo.tree",), time=("generated_at",), + counts=("instructions", "findings")), + "plan": KindFields(subject=("subjects[].id",), model=("models[]",), time=("generated_at",), + counts=("cells",)), + "result": KindFields(time=("generated_at",), counts=("cells",)), + "repo_budget": KindFields(subject=("tree",), counts=("entries",)), + "task_inspect": KindFields(subject=("root",), counts=("items",)), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. @@ -77,3 +90,8 @@ def read_field(data: dict, paths: tuple[str, ...]) -> tuple[str, ...]: if found: return found return () + + +def contents(kind: str, data: dict) -> dict[str, int]: + """The length of each of the kind's ``counts`` lists that ``data`` holds.""" + return {key: len(data[key]) for key in FIELDS[kind].counts if isinstance(data.get(key), list)} diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py index ace7a690a..1f3e480a0 100644 --- a/dokimi/dokimi/runs/scan.py +++ b/dokimi/dokimi/runs/scan.py @@ -147,16 +147,25 @@ def head_version(path: Path) -> str | None: return match.group(1).decode("utf-8", errors="replace") if match else None -def read_artifact(root: Path, path: Path, version: str) -> Artifact: - """Parse the artifact at ``path`` whose first bytes declared ``version`` (a known one). +def load_document(path: Path, version: str) -> dict: + """The JSON object at ``path``, whose first bytes declared ``version``. Raises ``OSError`` when it cannot be read and ``ValueError`` when it is not the JSON object its first bytes promised. """ - stat = path.stat() data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict) or data.get("schema_version") != version: raise ValueError(f"the document does not hold the schema_version {version} it opens with") + return data + + +def read_artifact(root: Path, path: Path, version: str, data: dict | None = None) -> Artifact: + """The artifact at ``path`` (under ``root``) of a known ``version``; ``data`` if already loaded. + + Raises as :func:`load_document` does. + """ + stat = path.stat() + data = load_document(path, version) if data is None else data fields = FIELDS[VERSION_KINDS[version]] times = read_field(data, fields.time) return Artifact(path=path.relative_to(root), schema_version=version, @@ -175,7 +184,8 @@ def _skip_reason(name: str) -> str | None: return None if lower.endswith(".json") else "not_json" -def _reason(exc: BaseException) -> str: +def read_error_reason(exc: BaseException) -> str: + """Why a file could not be read, in one line.""" if isinstance(exc, OSError) and exc.strerror: return f"cannot read: {exc.strerror}" if isinstance(exc, UnicodeDecodeError): @@ -195,7 +205,7 @@ def _scan_file(result: ScanResult, path: Path) -> None: regular = stat.S_ISREG(path.stat().st_mode) # opening a FIFO would block version = head_version(path) if regular else None except OSError as exc: - result.unreadable.append(Unreadable(rel, _reason(exc))) + result.unreadable.append(Unreadable(rel, read_error_reason(exc))) return if not regular: result.skipped["not_a_file"] += 1 @@ -208,7 +218,7 @@ def _scan_file(result: ScanResult, path: Path) -> None: try: result.artifacts.append(read_artifact(result.root, path, version)) except (OSError, ValueError, RecursionError) as exc: - result.unreadable.append(Unreadable(rel, _reason(exc), version)) + result.unreadable.append(Unreadable(rel, read_error_reason(exc), version)) def _is_link(path: Path) -> bool: diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py new file mode 100644 index 000000000..e1d0e5631 --- /dev/null +++ b/dokimi/dokimi/runs/show.py @@ -0,0 +1,125 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""One artifact under the runs root: its kind, whether it meets its schema, what it holds and +what lies next to it. A failed schema is reported, not raised: showing is not a gate.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dokimi.artifacts import schema_errors, schema_version_for +from dokimi.runs.kinds import VERSION_KINDS, contents +from dokimi.runs.scan import ( + Artifact, + head_version, + load_document, + read_artifact, + read_error_reason, +) +from dokimi.utils import InputError + +#: Entries of the artifact's directory that ``runs show`` names; the rest are only counted. +MAX_SIBLINGS = 20 + + +@dataclass(frozen=True) +class Sibling: + """An entry beside the artifact: a file (with its schema_version, if it opens with one) + or a directory (``name`` ends in ``/``).""" + + name: str + schema_version: str | None = None + + +@dataclass +class Shown: + """What ``runs show`` prints about one artifact.""" + + artifact: Artifact + schema: str # the schema version the document was checked against + errors: list[str] + contents: dict[str, int] + siblings: list[Sibling] = field(default_factory=list) + siblings_total: int = 0 + siblings_error: str | None = None + + +def _inside(root: Path, path: Path) -> bool: + base, target = (os.path.normcase(os.path.abspath(p)) for p in (root, path)) + try: + return os.path.commonpath([base, target]) == base + except ValueError: # another drive on Windows + return False + + +def resolve(root: Path, address: str) -> Path: + """The file ``address`` names: a path relative to the root (as ``runs list`` prints it), or + an absolute or working-directory path inside the root. Raises ``InputError`` otherwise.""" + candidates = [root / address, Path(address)] + path = next((p for p in candidates if p.exists()), candidates[0]) + if not _inside(root, path): + raise InputError(f"{address} is not under the runs root {root.as_posix()}/") + if not path.exists(): + raise InputError(f"no file {address} under the runs root {root.as_posix()}/") + if path.is_dir(): + raise InputError(f"{address} is a directory; runs show takes one artifact file") + if not path.is_file(): + raise InputError(f"{address} is not a regular file") + return Path(os.path.abspath(path)) + + +def _version(path: Path, address: str) -> str: + try: + version = head_version(path) + except OSError as exc: + raise InputError(f"{address}: cannot read: {exc.strerror or exc}") from exc + if version is None: + raise InputError(f"{address} is not an artifact: it does not open with a schema_version") + if version not in VERSION_KINDS: + raise InputError(f"{address} is not an artifact of a registered kind: " + f"schema_version {version}") + return version + + +def _siblings(shown: Shown, path: Path) -> None: + try: + with os.scandir(path.parent) as listing: + entries = sorted((e for e in listing if e.name != path.name), key=lambda e: e.name) + except OSError as exc: + shown.siblings_error = f"cannot list: {exc.strerror or exc}" + return + shown.siblings_total = len(entries) + for entry in entries[:MAX_SIBLINGS]: + try: + if entry.is_dir(): + shown.siblings.append(Sibling(entry.name + "/")) + continue + readable = entry.name.lower().endswith(".json") and entry.is_file() # not a FIFO + version = head_version(Path(entry.path)) if readable else None + except OSError: # a link loop, or an entry it may not stat: named, not read + version = None + shown.siblings.append(Sibling(entry.name, version)) + + +def show(root: Path, address: str) -> Shown: + """The artifact ``address`` names under ``root``, checked against its schema. + + Raises ``InputError`` when ``address`` is outside the root, missing, a directory, not an + artifact of a registered kind, or not readable as one. + """ + path = resolve(root, address) + version = _version(path, address) + kind = VERSION_KINDS[version] + try: + data = load_document(path, version) + artifact = read_artifact(Path(os.path.abspath(root)), path, version, data) + except (OSError, ValueError, RecursionError) as exc: + raise InputError(f"{address}: {read_error_reason(exc)}") from exc + schema = schema_version_for(kind, data) + shown = Shown(artifact=artifact, schema=schema, errors=schema_errors(schema, data), + contents=contents(kind, data)) + _siblings(shown, path) + return shown diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py new file mode 100644 index 000000000..9bc68d51a --- /dev/null +++ b/dokimi/tests/test_runs_show.py @@ -0,0 +1,266 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`runs show` (#278): one artifact's fields, schema check, contents and neighbours; a failed +schema check exits 0, an address that is not an artifact under the root exits 2.""" + +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from dokimi import cli +from dokimi.artifacts import SCHEMA_DIR, SCHEMA_VERSIONS, save_artifact +from dokimi.runs import kinds +from dokimi.runs.show import MAX_SIBLINGS, show + +#: What ``contents`` counts in each kind's fixture (the length of its top-level lists). +CONTENTS = { + "answers.v1": {"answers": 1}, + "arms.v1": {"arms": 4, "answers": 0}, + "eval.v1": {"evaluations": 1}, + "explore_runs.v1": {"tasks": 1, "arms": 2, "results": 2}, + "guidance_reach.v1": {"files": 5, "scenarios": 5, "records": 25}, + "matrix_rollup.v1": {"cells": 2}, + "novelty.v1": {"conflicts": 0, "warnings": 2}, + "plan.v1": {"cells": 6}, + "plugin_delta.v1": {"arms": 2}, + "questions.v2": {"questions": 3}, + "repo_budget.v1": {"entries": 9}, + "repo_environment.v1": {"instructions": 4, "findings": 13}, + "repo_readiness.v1": {"files": 2, "claims": 52}, + "result.v1": {"cells": 6}, + "subject_scorecard.v2": {"cells": 0}, + "task_inspect.v1": {"items": 4}, + "task_runs.v1": {"tasks": 6, "harnesses": 2, "results": 24}, + "tool_preference.v1": {"items": 16, "results": 16}, +} + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts" + + +def _fixture(version: str) -> dict: + return json.loads((FIXTURES / f"{version}.json").read_text(encoding="utf-8")) + + +@pytest.fixture +def root(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + root = tmp_path / "results" + save_artifact("arms", {**_fixture("arms.v1"), "generated_at": "2026-10-29T14:02:00Z"}, + root / "arms" / "onetbb.json") + save_artifact("plugin_delta", _fixture("plugin_delta.v1"), root / "arms" / "plugin_delta.json") + (root / "arms" / "onetbb.md").write_text("# arms", encoding="utf-8") + (root / "arms" / "onetbb-matrix").mkdir() + return root + + +def _show(monkeypatch, capsys, *argv: str): + monkeypatch.setattr("sys.argv", ["dokimi", "runs", "show", *argv]) + cli.main() + return capsys.readouterr() + + +def _fails(monkeypatch, capsys, *argv: str) -> str: + with pytest.raises(SystemExit) as exc: + _show(monkeypatch, capsys, *argv) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert err.startswith("Error: ") and "Traceback" not in err + return err + + +@pytest.mark.parametrize("kind", sorted(kinds.FIELDS)) +def test_every_count_is_a_top_level_list_of_the_kinds_schema(kind): + schema = json.loads((SCHEMA_DIR / f"{SCHEMA_VERSIONS[kind]}.json").read_text("utf-8")) + for key in kinds.FIELDS[kind].counts: + declared = schema["properties"][key].get("type") + assert declared == "array" or "array" in (declared or []), (kind, key) + + +def test_a_valid_artifact_shows_its_fields_contents_and_neighbours(monkeypatch, capsys, root): + out = _show(monkeypatch, capsys, "arms/onetbb.json").out + + size = (root / "arms" / "onetbb.json").stat().st_size + assert out.splitlines() == [ + "path: arms/onetbb.json", + "kind: arms (arms.v1)", + "schema: valid", + "subject: onetbb", + "model: m", + "time: 2026-10-29T14:02:00Z", + "run_id: - (no artifact records one yet)", + f"size: {size:,} bytes", + "contents: arms: 4, answers: 0", + "next to it: onetbb-matrix/, onetbb.md, plugin_delta.json (plugin_delta.v1)", + ] + + +def test_the_path_may_be_absolute_or_from_the_working_directory(monkeypatch, capsys, root): + first = _show(monkeypatch, capsys, "arms/onetbb.json").out + assert _show(monkeypatch, capsys, str(root / "arms" / "onetbb.json")).out == first + assert _show(monkeypatch, capsys, "results/arms/onetbb.json").out == first + + +def test_an_artifact_that_fails_its_schema_lists_the_errors_and_exits_0(monkeypatch, capsys, root): + data = json.loads((root / "arms" / "onetbb.json").read_text(encoding="utf-8")) + data["arms"], data["total_questions"] = "baseline", "six" + (root / "arms" / "bad.json").write_text(json.dumps(data, indent=2), encoding="utf-8") + + lines = _show(monkeypatch, capsys, "arms/bad.json").out.splitlines() # no SystemExit: 0 + + assert lines[2:5] == [ + "schema: invalid, 2 errors", + " - arms: 'baseline' is not of type 'array'", + " - total_questions: 'six' is not of type 'integer'", + ] + assert lines[5] == "subject: onetbb" + + +def test_a_time_the_artifact_does_not_record_is_shown_as_the_files_mtime(monkeypatch, capsys, root): + stamp = 1_780_000_000 + os.utime(root / "arms" / "plugin_delta.json", (stamp, stamp)) + out = _show(monkeypatch, capsys, "arms/plugin_delta.json").out + + (time_line,) = [line for line in out.splitlines() if line.startswith("time:")] + when = datetime.fromtimestamp(stamp, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + assert time_line == f"time: - ({when} UTC, the file's mtime)" + + +@pytest.mark.parametrize("address, message", [ + ("arms/nope.json", "no file arms/nope.json under the runs root results/"), + ("arms", "arms is a directory; runs show takes one artifact file"), + ("arms/onetbb.md", "arms/onetbb.md is not an artifact: it does not open with a schema_version"), + ("../outside.json", "../outside.json is not under the runs root results/"), + ("pairwise.json", "pairwise.json is not an artifact of a registered kind: " + "schema_version pairwise.v0"), + ("broken.json", "broken.json: not valid JSON: Expecting value at line 1"), +]) +def test_an_address_that_is_not_an_artifact_under_the_root_exits_2(monkeypatch, capsys, root, + address, message): + (root.parent / "outside.json").write_text(json.dumps({"schema_version": "eval.v1"}), + encoding="utf-8") + (root / "pairwise.json").write_text(json.dumps({"schema_version": "pairwise.v0"}), + encoding="utf-8") + (root / "broken.json").write_text('{"schema_version": "eval.v1", "x": [', encoding="utf-8") + + assert _fails(monkeypatch, capsys, address) == f"Error: {message}\n" + + +def test_an_absolute_path_outside_the_root_exits_2(monkeypatch, capsys, root, tmp_path): + other = tmp_path / "eval.json" + save_artifact("eval", _fixture("eval.v1"), other) + + assert "is not under the runs root" in _fails(monkeypatch, capsys, str(other)) + + +def test_show_without_a_root_exits_2(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + + err = _fails(monkeypatch, capsys, "x.json") + + assert err.startswith("Error: no runs root: results/ does not exist") + + +def test_json_has_the_list_fields_the_schema_check_and_the_neighbours(monkeypatch, capsys, root): + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + + assert data == { + "schema_version": "arms.v1", "kind": "arms", "run_id": None, "subject": "onetbb", + "model": "m", "generated_at": "2026-10-29T14:02:00Z", "path": "arms/onetbb.json", + "size_bytes": (root / "arms" / "onetbb.json").stat().st_size, "root": "results", + "schema": "arms.v1", "schema_valid": True, "schema_errors": [], + "contents": {"arms": 4, "answers": 0}, + "siblings": [{"name": "onetbb-matrix/", "schema_version": None}, + {"name": "onetbb.md", "schema_version": None}, + {"name": "plugin_delta.json", "schema_version": "plugin_delta.v1"}], + "siblings_total": 3, "siblings_error": None, + } + + +def test_json_of_an_invalid_artifact_carries_its_errors(monkeypatch, capsys, root): + (root / "arms" / "bad.json").write_text(json.dumps({"schema_version": "arms.v1"}), + encoding="utf-8") + + data = json.loads(_show(monkeypatch, capsys, "arms/bad.json", "--json").out) + + assert data["schema_valid"] is False + assert sorted(data["schema_errors"]) == [": 'answers' is a required property", + ": 'arms' is a required property"] + + +def test_a_crowded_directory_names_the_first_neighbours_and_counts_the_rest(monkeypatch, capsys, + root): + for i in range(MAX_SIBLINGS + 5): + (root / "arms" / f"z{i:02}.log").write_text("", encoding="utf-8") + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + text = _show(monkeypatch, capsys, "arms/onetbb.json").out + + assert len(data["siblings"]) == MAX_SIBLINGS and data["siblings_total"] == MAX_SIBLINGS + 8 + assert text.rstrip().endswith(", and 8 more") + + +def test_a_superseded_version_is_checked_against_its_own_schema(monkeypatch, capsys, root): + old = {"schema_version": "questions.v1", + **{k: v for k, v in _fixture("questions.v1").items() if k != "schema_version"}} + (root / "q.json").write_text(json.dumps(old, indent=2), encoding="utf-8") + + data = json.loads(_show(monkeypatch, capsys, "q.json", "--json").out) + + assert (data["schema"], data["schema_valid"], data["kind"]) == ("questions.v1", True, + "questions") + + +def test_the_count_is_kept_for_every_list_of_every_kind(tmp_path): + assert set(CONTENTS) == set(SCHEMA_VERSIONS.values()) + for version, expected in CONTENTS.items(): + save_artifact(kinds.VERSION_KINDS[version], _fixture(version), tmp_path / f"{version}.json") + assert show(tmp_path, f"{version}.json").contents == expected, version + + +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFOs are POSIX") +def test_a_fifo_is_named_as_a_neighbour_and_refused_as_an_address(monkeypatch, capsys, root): + os.mkfifo(root / "arms" / "pipe.json") # opening it would block + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + assert {"name": "pipe.json", "schema_version": None} in data["siblings"] + assert _fails(monkeypatch, capsys, "arms/pipe.json") == "Error: arms/pipe.json is not a regular file\n" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_neighbour_that_cannot_be_stated_is_named_not_a_crash(monkeypatch, capsys, root): + (root / "arms" / "loop").symlink_to(root / "arms" / "loop") + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + + assert {"name": "loop", "schema_version": None} in data["siblings"] + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root lists every directory") +def test_a_directory_that_cannot_be_listed_still_shows_its_artifact(monkeypatch, capsys, root): + (root / "arms").chmod(0o311) # files can be opened by name, the directory cannot be listed + try: + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + text = _show(monkeypatch, capsys, "arms/onetbb.json").out + finally: + (root / "arms").chmod(0o755) + + assert (data["siblings"], data["siblings_error"]) == ([], "cannot list: Permission denied") + assert text.splitlines()[-1] == "next to it: cannot list: Permission denied" + + +def test_control_characters_in_fields_are_escaped(monkeypatch, capsys, root): + save_artifact("arms", {**_fixture("arms.v1"), "library_name": "evil\x1b[2J\nnext"}, + root / "evil.json") + + lines = _show(monkeypatch, capsys, "evil.json").out.splitlines() + + assert "subject: evil\\x1b[2J\\nnext" in lines and len(lines) == 10 From aaf8e15ba48fbb0460423d327bd58db597371b22 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 06:46:06 -0700 Subject: [PATCH 04/18] docs(runs): the runs root, runs list and runs show docs/runs.md: what the runs root is and how DOKIMI_RUNS_DIR moves it, an example of each command, which file counts as an artifact and why the rest are counted but not read, where subject, model and time come from, what is not listed yet (no run_id grouping until #277; repo-runs/, answers/ and eval/ written without --run-id, and output flags pointing outside the root), the --json fields and the exit codes. The page goes in the mkdocs nav and the docs index, and quickstart's output layout points to it. The static benchmark's max_docs rises to 36 for the new page, as its spec asks. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/benchmarks/spec.v1.yaml | 2 +- dokimi/docs/README.md | 3 + dokimi/docs/quickstart.md | 3 +- dokimi/docs/runs.md | 113 +++++++++++++++++++++++++++++++++ dokimi/mkdocs.yml | 1 + 5 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 dokimi/docs/runs.md diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index cde41b078..a7181b917 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -80,7 +80,7 @@ future: # edit here, so raise max_docs in the same commit that adds the doc. golden_manifest: min_docs: 10 - max_docs: 35 + max_docs: 36 include: - docs/**/*.md exclude: diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index c336eecb0..c8e122378 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -294,6 +294,9 @@ code. - [quickstart.md](quickstart.md) โ€” end-to-end LLM evaluation pipeline: personas โ†’ questions โ†’ answers โ†’ judge โ†’ report. Includes troubleshooting and a cost estimate. +- [runs.md](runs.md) โ€” what the runs have written: `cli.py runs list` lists + every artifact under `results/` (or `DOKIMI_RUNS_DIR`) and counts what it + skipped, `cli.py runs show` checks one against its schema. No LLM call. - [user-flow.md](user-flow.md) โ€” the canonical end-to-end path: pick a target, preflight, run, then pin the question set so runs stay comparable. - [benchmarking-and-comparison.md](benchmarking-and-comparison.md) โ€” how to diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index f95bc0317..c7da5d3ce 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -218,7 +218,8 @@ at such a checkout. Everything the pipeline produces โ€” answers, eval scores, benchmark snapshots, reports โ€” stays under the git-ignored `answers/`, `eval/`, `baselines/`, `reports/` and `results/` directories. See -[`data/README.md`](../data/README.md). +[`data/README.md`](../data/README.md). `python cli.py runs list` lists the +artifacts under `results/` ([runs.md](runs.md)). ## Cost estimate diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md new file mode 100644 index 000000000..ed6f72f58 --- /dev/null +++ b/dokimi/docs/runs.md @@ -0,0 +1,113 @@ +--- +depends_on: + - dokimi/dokimi/runs/** + - dokimi/dokimi/commands/runs.py +--- + + +# The runs root: `runs list` and `runs show` + +Most commands that run an evaluation write their artifacts under `results/` by +default (a few write elsewhere, listed below). `runs list` shows what is there +and `runs show` opens one artifact. Both only read files: no model call, no +network access, nothing written. + +The runs root is `results/` in the working directory. Set `DOKIMI_RUNS_DIR` to +read another directory. + +## `runs list` + +```text +$ python cli.py runs list +SCHEMA SUBJECT MODEL TIME (UTC) PATH +arms.v1 onetbb gpt-6.1-sol 2026-10-29 14:02 arms/onetbb.json +plugin_delta.v1 onetbb gpt-6.1-sol 2026-10-28 10:11* arms/plugin_delta.json +eval.v1 - claude-sonnet-5-5 2026-10-28 10:05 onetbb_r7/eval/onetbb.json +answers.v1 onetbb gpt-6.1-sol 2026-10-28 09:40 onetbb_r7/answers/onetbb.json +novelty.v1 skill:skills/onetbb-tuning claude-sonnet-5-5 2026-10-27 18:30 novelty/skill_skills_onetbb-tuning/novelty.json + +5 artifacts under results/ (* the file's mtime: the artifact records no time) +skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_version (pairwise.v0: 1), 1 not .json +``` + +- **Each row is one artifact file.** A file is an artifact when it opens with the + `schema_version` of a registered kind: `save_artifact` has written it as the + first key since [#27](https://github.com/intel/dokimi/pull/27). Superseded + versions such as `questions.v1` count too. Files that open with anything + else are skipped without being read in full, so a large file of another kind + costs nothing. +- **Subject, model and time** come from the artifact's own fields, and which + fields depends on the kind: `product` of a questions set, `library_name` of + an arms run, `artifact.ref` of a novelty run, `repo.id` of a reach run, + `judge_model` and `evaluated_at` of an eval. An artifact that records no + subject shows `-` (an eval, a task run, a questions set from `questions + generate`, which records the model but not the product); nothing is guessed + from the path. Rows are newest first. When an artifact records no time, its + file's mtime is used for the order and shown with `*`. +- **Nothing is dropped silently.** The summary counts the files skipped, by + reason: JSON that does not open with a `schema_version` (other output, and + artifacts saved before #27, which did not put it first), JSON of an unknown + `schema_version` (named), `*.tmp` and `*.rejected` files left by + `save_artifact`, other files, entries that are not regular files (a FIFO), + and symlinked directories, which are not walked. A file or directory that + cannot be read, or a file that does not parse, is listed as unreadable, with + the first one's reason. +- `--kind KIND` (repeatable, for example `--kind arms --kind eval`) and + `--subject NAME` (an exact, case-sensitive match) filter the rows. +- `--json` prints `root`, `artifacts`, `total` (before filtering), `skipped`, + `unknown_versions` and `unreadable`. Each artifact has `schema_version`, + `kind`, `run_id`, `subject`, `model`, `generated_at` (the time the artifact + records, `evaluated_at` for an eval; `null` when none), `path` (relative to + the root) and `size_bytes`. + +**What it does not show yet.** No artifact records a `run_id` yet +([#277](https://github.com/intel/dokimi/issues/277)), so rows are not grouped +into runs and `run_id` is `null`. Files outside the root are not listed: + +- `repo run` writes its batches to `repo-runs/`, and its files carry no + `schema_version`; +- `answers generate` and `eval score` write to `answers/` and `eval/` unless + `--run-id` is given, and `eval panel-score` writes to `eval/`; +- `questions generate` and `questions refine` write to `questions/`, + `repo reach` to `reach-out/`, `tools preference` to `tool_preference.json` + in the working directory; +- an output flag can point anywhere. + +## `runs show` + +```text +$ python cli.py runs show arms/onetbb.json +path: arms/onetbb.json +kind: arms (arms.v1) +schema: valid +subject: onetbb +model: gpt-6.1-sol +time: 2026-10-29T14:02:11Z +run_id: - (no artifact records one yet) +size: 3,503 bytes +contents: arms: 4, answers: 0 +next to it: onetbb.md, plugin_delta.json (plugin_delta.v1) +``` + +The path is the one `runs list` prints, relative to the root. An absolute +path, or one from the working directory (`results/arms/onetbb.json`), works +too if it lies inside the root. The artifact is checked against the schema of +the version it declares. An artifact that fails the check prints +`schema: invalid, N errors` and each error, and the command still exits 0: +`runs show` reads, it does not gate. `contents` gives the length of the lists +the kind holds, and `next to it` the first 20 entries of the artifact's +directory. `--json` prints the fields of a `runs list` row plus `root`, +`schema`, `schema_valid`, `schema_errors`, `contents`, `siblings`, +`siblings_total` and `siblings_error`. + +## Exit codes + +| Case | Exit | +| --- | --- | +| Listed or shown, including an artifact that fails its schema | 0 | +| `results/` does not exist (`runs list` prints "no runs") | 0 | +| `DOKIMI_RUNS_DIR` does not exist, or the root is not a directory | 2 | +| `runs show`: no root, a path outside the root, a missing file, a directory, or a file that is not a readable artifact of a registered kind | 2 | diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index f22f10047..88b5b236a 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -110,6 +110,7 @@ nav: - reviewing.md - external-content.md - quickstart.md + - runs.md - user-flow.md - benchmarking-and-comparison.md - adding-doc-source.md From 6ec9ed55f815788b8e517d8c349190896637c2dc Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 07:40:30 -0700 Subject: [PATCH 05/18] fix(runs): print unknown versions escaped; an mtime past 9999 does not crash head_version keeps any character but an ASCII control, so an unknown schema_version could carry a Unicode line separator (U+2028) into the runs list summary and the runs show error. Both now go through the one escaping helper, plain(), moved to dokimi/runs/scan.py. tmpfs keeps an mtime past the year 9999, where datetime.fromtimestamp raises: the text of runs list and runs show crashed on an untimed artifact with such an mtime (--json did not). The time column and the show line now print it as @. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/commands/runs.py | 41 +++++++++++++++++++++------------- dokimi/dokimi/runs/scan.py | 7 ++++++ dokimi/dokimi/runs/show.py | 3 ++- dokimi/tests/test_runs_cli.py | 23 +++++++++++++++++++ dokimi/tests/test_runs_show.py | 26 ++++++++++++++++++++- 5 files changed, 82 insertions(+), 18 deletions(-) diff --git a/dokimi/dokimi/commands/runs.py b/dokimi/dokimi/commands/runs.py index f508e9b8a..9ae3405cf 100644 --- a/dokimi/dokimi/commands/runs.py +++ b/dokimi/dokimi/commands/runs.py @@ -10,18 +10,19 @@ from datetime import datetime, timezone from dokimi.artifacts import SCHEMA_VERSIONS -from dokimi.runs.scan import SKIP_REASONS, Artifact, ScanResult, runs_root, scan, select +from dokimi.runs.scan import ( + SKIP_REASONS, + Artifact, + ScanResult, + plain, + runs_root, + scan, + select, +) from dokimi.runs.show import Shown, show from dokimi.utils import InputError -def _plain(text: str) -> str: - """``text`` with control characters escaped: a file name or field cannot split a row or - reach the terminal as an escape sequence.""" - return "".join(c if c.isprintable() else c.encode("unicode_escape").decode("ascii") - for c in text) - - def _untimed(artifact: Artifact) -> bool: return artifact.recorded_at is None and not artifact.generated_at @@ -33,7 +34,15 @@ def _when(artifact: Artifact) -> str: return stamp.strftime("%Y-%m-%d %H:%M") if artifact.generated_at: # not ISO 8601, or out of range: shown as written return artifact.generated_at - return datetime.fromtimestamp(artifact.mtime, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + "*" + return _mtime(artifact) + "*" + + +def _mtime(artifact: Artifact) -> str: + """The file's mtime in UTC to the minute; ``@`` when it is past what a date holds.""" + try: + return datetime.fromtimestamp(artifact.mtime, tz=timezone.utc).strftime("%Y-%m-%d %H:%M") + except (ValueError, OverflowError, OSError): # tmpfs keeps mtimes past the year 9999 + return f"@{artifact.mtime:.0f}" def _count(n: int, noun: str) -> str: @@ -42,7 +51,7 @@ def _count(n: int, noun: str) -> str: def _table(artifacts: list[Artifact]) -> list[str]: rows: list[tuple[str, ...]] = [("SCHEMA", "SUBJECT", "MODEL", "TIME (UTC)", "PATH")] - rows += [tuple(_plain(cell) for cell in (a.schema_version, a.subject or "-", a.model or "-", + rows += [tuple(plain(cell) for cell in (a.schema_version, a.subject or "-", a.model or "-", _when(a), a.path.as_posix())) for a in artifacts] widths = [max(len(row[i]) for row in rows) for i in range(4)] @@ -57,7 +66,7 @@ def _summary(result: ScanResult, shown: list[Artifact], root: str) -> list[str]: if any(_untimed(a) for a in shown): lines[0] += " (* the file's mtime: the artifact records no time)" labels = {**SKIP_REASONS, "unknown_schema_version": SKIP_REASONS["unknown_schema_version"] - + " (" + ", ".join(f"{v}: {n}" for v, n in sorted(result.unknown_versions.items())) + + " (" + ", ".join(f"{plain(v)}: {n}" for v, n in sorted(result.unknown_versions.items())) + ")"} skipped = [f"{result.skipped[reason]} {label}" for reason, label in labels.items() if result.skipped[reason]] @@ -66,7 +75,7 @@ def _summary(result: ScanResult, shown: list[Artifact], root: str) -> list[str]: if result.unreadable: first = result.unreadable[0] lines.append(f"unreadable: {len(result.unreadable)} " - f"(first: {_plain(first.path.as_posix())}: {_plain(first.reason)})") + f"(first: {plain(first.path.as_posix())}: {plain(first.reason)})") return lines @@ -84,7 +93,7 @@ def cmd_runs_list(args: argparse.Namespace) -> None: "unreadable": [u.to_json() for u in result.unreadable], }, indent=2)) return - shown_root = _plain(root.path.as_posix().rstrip("/") + "/") + shown_root = plain(root.path.as_posix().rstrip("/") + "/") if not exists: print(f"no runs under {shown_root} (it does not exist; DOKIMI_RUNS_DIR names another root)") return @@ -106,7 +115,7 @@ def _show_json(shown: Shown, root: str) -> dict: def _show_text(shown: Shown) -> list[str]: a = shown.artifact errors = _count(len(shown.errors), "error") - when = a.generated_at or "- (" + _when(a).rstrip("*") + " UTC, the file's mtime)" + when = a.generated_at or f"- ({_mtime(a)} UTC, the file's mtime)" beside = [s.name + (f" ({s.schema_version})" if s.schema_version else "") for s in shown.siblings] if shown.siblings_total > len(shown.siblings): @@ -123,8 +132,8 @@ def _show_text(shown: Shown) -> list[str]: ("contents", ", ".join(f"{key}: {n}" for key, n in shown.contents.items()) or "-"), ("next to it", shown.siblings_error or ", ".join(beside) or "-"), ] - lines = [f"{label + ':':<12}{_plain(value)}" for label, value in rows] - lines[3:3] = [f" - {_plain(e)}" for e in shown.errors] + lines = [f"{label + ':':<12}{plain(value)}" for label, value in rows] + lines[3:3] = [f" - {plain(e)}" for e in shown.errors] return lines diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py index 1f3e480a0..eb261af54 100644 --- a/dokimi/dokimi/runs/scan.py +++ b/dokimi/dokimi/runs/scan.py @@ -140,6 +140,13 @@ class ScanResult: unreadable: list[Unreadable] = field(default_factory=list) +def plain(text: str) -> str: + """``text`` with control and separator characters escaped (``\\n``, ``\\x1b``, ``\\u2028``), + so a file name or a field cannot split a printed line or reach the terminal as an escape.""" + return "".join(c if c.isprintable() else c.encode("unicode_escape").decode("ascii") + for c in text) + + def head_version(path: Path) -> str | None: """The ``schema_version`` a file opens with, read from its first bytes; ``None`` if none.""" with open(path, "rb") as fh: diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index e1d0e5631..b53e6273b 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -16,6 +16,7 @@ Artifact, head_version, load_document, + plain, read_artifact, read_error_reason, ) @@ -80,7 +81,7 @@ def _version(path: Path, address: str) -> str: raise InputError(f"{address} is not an artifact: it does not open with a schema_version") if version not in VERSION_KINDS: raise InputError(f"{address} is not an artifact of a registered kind: " - f"schema_version {version}") + f"schema_version {plain(version)}") return version diff --git a/dokimi/tests/test_runs_cli.py b/dokimi/tests/test_runs_cli.py index 7f32d6b4f..e5ba87a83 100644 --- a/dokimi/tests/test_runs_cli.py +++ b/dokimi/tests/test_runs_cli.py @@ -173,3 +173,26 @@ def test_control_characters_in_names_and_fields_are_escaped(monkeypatch, capsys, assert len(rows) == 2 assert rows[1].split()[1] == "evil\\x1b[2J" and rows[1].endswith("two\\nlines.json") + + +def test_an_unknown_version_with_a_line_separator_is_printed_escaped(monkeypatch, capsys, root): + (root / "x.json").write_text(json.dumps({"schema_version": "v\u2028forged"}, ensure_ascii=False), + encoding="utf-8") + + out = _list(monkeypatch, capsys).out + + assert "\u2028" not in out + assert "JSON of an unknown schema_version (v\\u2028forged: 1)" in out + + +def test_an_mtime_past_the_year_9999_is_shown_as_seconds(monkeypatch, capsys, root): + from dokimi.commands import runs as runs_cmd + from dokimi.runs.scan import Artifact, ScanResult + + far = Artifact(path=Path("p.json"), schema_version="plugin_delta.v1", subjects=(), models=(), + generated_at=None, size_bytes=1, mtime=253402300800.0) # tmpfs keeps such mtimes + monkeypatch.setattr(runs_cmd, "scan", lambda path: ScanResult(root=path, artifacts=[far])) + + rows = _list(monkeypatch, capsys).out.splitlines() + + assert rows[1].split()[:4] == ["plugin_delta.v1", "-", "-", "@253402300800*"] diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 9bc68d51a..65df6af19 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -6,6 +6,7 @@ from __future__ import annotations +import dataclasses import json import os from datetime import datetime, timezone @@ -16,7 +17,7 @@ from dokimi import cli from dokimi.artifacts import SCHEMA_DIR, SCHEMA_VERSIONS, save_artifact from dokimi.runs import kinds -from dokimi.runs.show import MAX_SIBLINGS, show +from dokimi.runs.show import MAX_SIBLINGS, Shown, show #: What ``contents`` counts in each kind's fixture (the length of its top-level lists). CONTENTS = { @@ -264,3 +265,26 @@ def test_control_characters_in_fields_are_escaped(monkeypatch, capsys, root): lines = _show(monkeypatch, capsys, "evil.json").out.splitlines() assert "subject: evil\\x1b[2J\\nnext" in lines and len(lines) == 10 + + +def test_an_unknown_version_is_named_escaped_in_the_error(monkeypatch, capsys, root): + (root / "x.json").write_text(json.dumps({"schema_version": "v\u2028forged"}, ensure_ascii=False), + encoding="utf-8") + + err = _fails(monkeypatch, capsys, "x.json") + + assert err == ("Error: x.json is not an artifact of a registered kind: " + "schema_version v\\u2028forged\n") + + +def test_an_mtime_past_the_year_9999_is_shown_as_seconds(monkeypatch, capsys, root): + from dokimi.commands import runs as runs_cmd + + real = show(root, "arms/plugin_delta.json") + far = Shown(artifact=dataclasses.replace(real.artifact, mtime=253402300800.0), + schema=real.schema, errors=[], contents=real.contents) + monkeypatch.setattr(runs_cmd, "show", lambda root, address: far) + + out = _show(monkeypatch, capsys, "arms/plugin_delta.json").out + + assert "time: - (@253402300800 UTC, the file's mtime)" in out.splitlines() From f14148f88df09910f2b0c4143f77a76420e7cd58 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 07:41:22 -0700 Subject: [PATCH 06/18] fix(runs): a root or an address that cannot be looked up exits 2 On Python 3.11-3.13 Path.exists(), is_dir() and is_file() raise PermissionError for a path under a directory that cannot be searched. RunsRoot.exists() (a DOKIMI_RUNS_DIR below such a directory) and runs show's resolve() (an address in such a directory) let it out, and the command crashed with a traceback and exit 1. Both now raise InputError with the reason, so the command prints it and exits 2. Python 3.14 answers False there, and the message is the existing "does not exist" / "no file"; the tests accept either. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/runs/scan.py | 12 ++++++++---- dokimi/dokimi/runs/show.py | 21 ++++++++++++--------- dokimi/tests/test_runs_cli.py | 20 ++++++++++++++++++++ dokimi/tests/test_runs_show.py | 14 ++++++++++++++ 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py index eb261af54..2769ce44a 100644 --- a/dokimi/dokimi/runs/scan.py +++ b/dokimi/dokimi/runs/scan.py @@ -44,12 +44,16 @@ def exists(self) -> bool: """True when the root is a directory; False when the default root is not there yet. Raises :class:`~dokimi.utils.InputError` when ``DOKIMI_RUNS_DIR`` names a missing path, - or when the root is something other than a directory. + when the root is something other than a directory, or when it cannot be looked up. """ - if self.path.is_dir(): - return True named = f"{ENV_VAR}={self.path}" if self.from_env else str(self.path) - if self.path.exists(): + try: # Python 3.11-3.13 raise PermissionError here for a parent that cannot be searched + if self.path.is_dir(): + return True + found = self.path.exists() + except OSError as exc: + raise InputError(f"runs root {named}: {read_error_reason(exc)}") from exc + if found: raise InputError(f"runs root {named} is not a directory") if self.from_env: raise InputError(f"runs root {named} does not exist") diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index b53e6273b..598e516ec 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -60,15 +60,18 @@ def resolve(root: Path, address: str) -> Path: """The file ``address`` names: a path relative to the root (as ``runs list`` prints it), or an absolute or working-directory path inside the root. Raises ``InputError`` otherwise.""" candidates = [root / address, Path(address)] - path = next((p for p in candidates if p.exists()), candidates[0]) - if not _inside(root, path): - raise InputError(f"{address} is not under the runs root {root.as_posix()}/") - if not path.exists(): - raise InputError(f"no file {address} under the runs root {root.as_posix()}/") - if path.is_dir(): - raise InputError(f"{address} is a directory; runs show takes one artifact file") - if not path.is_file(): - raise InputError(f"{address} is not a regular file") + try: # Python 3.11-3.13 raise PermissionError here for a directory that cannot be searched + path = next((p for p in candidates if p.exists()), candidates[0]) + if not _inside(root, path): + raise InputError(f"{address} is not under the runs root {root.as_posix()}/") + if not path.exists(): + raise InputError(f"no file {address} under the runs root {root.as_posix()}/") + if path.is_dir(): + raise InputError(f"{address} is a directory; runs show takes one artifact file") + if not path.is_file(): + raise InputError(f"{address} is not a regular file") + except OSError as exc: + raise InputError(f"{address}: {read_error_reason(exc)}") from exc return Path(os.path.abspath(path)) diff --git a/dokimi/tests/test_runs_cli.py b/dokimi/tests/test_runs_cli.py index e5ba87a83..399a40e20 100644 --- a/dokimi/tests/test_runs_cli.py +++ b/dokimi/tests/test_runs_cli.py @@ -196,3 +196,23 @@ def test_an_mtime_past_the_year_9999_is_shown_as_seconds(monkeypatch, capsys, ro rows = _list(monkeypatch, capsys).out.splitlines() assert rows[1].split()[:4] == ["plugin_delta.v1", "-", "-", "@253402300800*"] + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root searches every directory") +def test_a_root_under_a_directory_that_cannot_be_searched_exits_2(tmp_path, monkeypatch, capsys): + named = tmp_path / "locked" / "root" + named.mkdir(parents=True) + (tmp_path / "locked").chmod(0o600) + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(named)) + try: + with pytest.raises(SystemExit) as exc: + _list(monkeypatch, capsys) + finally: + (tmp_path / "locked").chmod(0o755) + + assert exc.value.code == 2 + # Python 3.11-3.13 raise PermissionError on the lookup; 3.14 reports the path as missing. + assert capsys.readouterr().err in ( + f"Error: runs root DOKIMI_RUNS_DIR={named}: cannot read: Permission denied\n", + f"Error: runs root DOKIMI_RUNS_DIR={named} does not exist\n") diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 65df6af19..fcbd60e2f 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -288,3 +288,17 @@ def test_an_mtime_past_the_year_9999_is_shown_as_seconds(monkeypatch, capsys, ro out = _show(monkeypatch, capsys, "arms/plugin_delta.json").out assert "time: - (@253402300800 UTC, the file's mtime)" in out.splitlines() + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root searches every directory") +def test_an_address_in_a_directory_that_cannot_be_searched_exits_2(monkeypatch, capsys, root): + (root / "arms").chmod(0o600) + try: + err = _fails(monkeypatch, capsys, "arms/onetbb.json") + finally: + (root / "arms").chmod(0o755) + + # Python 3.11-3.13 raise PermissionError on the lookup; 3.14 reports the file as missing. + assert err in ("Error: arms/onetbb.json: cannot read: Permission denied\n", + "Error: no file arms/onetbb.json under the runs root results/\n") From 35f5885314d014ef9e8ea478d3678de9e1e3fc9c Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 07:43:43 -0700 Subject: [PATCH 07/18] fix(runs): nothing outside the runs root is read runs show checked that the address lay inside the root as spelled, so results/linked -> /elsewhere let runs show linked/eval.json read a file outside the root (and list its neighbours), and runs list read a file symlink that pointed outside. #278 lists only what is under the root. The rule is now the same for both commands: a file is read only when its real path lies inside the real root. runs list counts a file symlink that leads out as skipped (linked_outside, "symlinks to files outside the root (not read)"); runs show exits 2 with "... leads outside the runs root"; a neighbour that links out is named but its head is not read. A symlink inside the root, and a root that is itself a symlink, work as before, and runs show keeps the name the address used. Symlinked directories are still not walked by runs list. Refs #278 Signed-off-by: Vladislav Perevezentsev --- ...030_120000_vlad-perevezentsev_runs-list.md | 3 +- dokimi/docs/runs.md | 8 ++-- dokimi/dokimi/runs/scan.py | 21 +++++++-- dokimi/dokimi/runs/show.py | 39 ++++++++-------- dokimi/tests/test_runs_scan.py | 26 +++++++++++ dokimi/tests/test_runs_show.py | 45 +++++++++++++++++++ 6 files changed, 117 insertions(+), 25 deletions(-) diff --git a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md index 6fdaa2ec4..cdfe54953 100644 --- a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md +++ b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md @@ -18,7 +18,8 @@ SPDX-License-Identifier: Apache-2.0 not exist or is not a directory exits 2. The command only reads. - **`dokimi runs show PATH` shows one artifact** โ€” `PATH` as `runs list` prints it (relative to the root), or an absolute or working-directory path inside the - root. It prints the artifact's kind, its fields, the length of the lists it + root once symlinks are resolved (neither command reads a file outside the + root). It prints the artifact's kind, its fields, the length of the lists it holds (answers, evaluations, cells, ...) and the files next to it, and checks it against its schema: an artifact that fails the check is shown with "schema: invalid, N errors" and every error, and the command still exits 0. diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md index ed6f72f58..08f57a0cb 100644 --- a/dokimi/docs/runs.md +++ b/dokimi/docs/runs.md @@ -52,7 +52,8 @@ skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_vers artifacts saved before #27, which did not put it first), JSON of an unknown `schema_version` (named), `*.tmp` and `*.rejected` files left by `save_artifact`, other files, entries that are not regular files (a FIFO), - and symlinked directories, which are not walked. A file or directory that + symlinked directories, which are not walked, and symlinks to files outside + the root, which are not read. A file or directory that cannot be read, or a file that does not parse, is listed as unreadable, with the first one's reason. - `--kind KIND` (repeatable, for example `--kind arms --kind eval`) and @@ -94,7 +95,8 @@ next to it: onetbb.md, plugin_delta.json (plugin_delta.v1) The path is the one `runs list` prints, relative to the root. An absolute path, or one from the working directory (`results/arms/onetbb.json`), works -too if it lies inside the root. The artifact is checked against the schema of +too if it lies inside the root once symlinks are resolved: neither command +reads anything outside the root. The artifact is checked against the schema of the version it declares. An artifact that fails the check prints `schema: invalid, N errors` and each error, and the command still exits 0: `runs show` reads, it does not gate. `contents` gives the length of the lists @@ -110,4 +112,4 @@ directory. `--json` prints the fields of a `runs list` row plus `root`, | Listed or shown, including an artifact that fails its schema | 0 | | `results/` does not exist (`runs list` prints "no runs") | 0 | | `DOKIMI_RUNS_DIR` does not exist, or the root is not a directory | 2 | -| `runs show`: no root, a path outside the root, a missing file, a directory, or a file that is not a readable artifact of a registered kind | 2 | +| `runs show`: no root, a path outside the root (or a symlink that leads out of it), a missing file, a directory, or a file that is not a readable artifact of a registered kind | 2 | diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py index 2769ce44a..450440d3c 100644 --- a/dokimi/dokimi/runs/scan.py +++ b/dokimi/dokimi/runs/scan.py @@ -130,9 +130,19 @@ def to_json(self) -> dict: "not_json": "not .json", "not_a_file": "not a regular file (a FIFO, socket or device)", "linked_directory": "symlinked directories (not walked)", + "linked_outside": "symlinks to files outside the root (not read)", } +def inside(root: Path | str, path: Path | str) -> bool: + """True when ``path`` is ``root`` or under it, as spelled (resolve both first for the target).""" + base, target = (os.path.normcase(os.path.abspath(p)) for p in (root, path)) + try: + return os.path.commonpath([base, target]) == base + except ValueError: # another drive on Windows + return False + + @dataclass class ScanResult: """What :func:`scan` found: the artifacts, newest first, and every file it skipped.""" @@ -206,13 +216,16 @@ def read_error_reason(exc: BaseException) -> str: return str(exc) -def _scan_file(result: ScanResult, path: Path) -> None: +def _scan_file(result: ScanResult, path: Path, real_root: str) -> None: rel = path.relative_to(result.root) skip = _skip_reason(path.name) if skip: result.skipped[skip] += 1 return try: + if path.is_symlink() and not inside(real_root, os.path.realpath(path)): + result.skipped["linked_outside"] += 1 + return regular = stat.S_ISREG(path.stat().st_mode) # opening a FIFO would block version = head_version(path) if regular else None except OSError as exc: @@ -249,8 +262,10 @@ def when(artifact: Artifact) -> float: def scan(root: Path) -> ScanResult: - """Every artifact under ``root`` (a directory), newest first; symlinked dirs are not walked.""" + """Every artifact under ``root`` (a directory), newest first. Symlinked directories are not + walked, and a file symlink is read only when it resolves inside the root.""" result = ScanResult(root=root) + real_root = os.path.realpath(root) def unlisted(exc: OSError) -> None: where = Path(exc.filename).relative_to(root) if exc.filename else Path(".") @@ -261,7 +276,7 @@ def unlisted(exc: OSError) -> None: result.skipped["linked_directory"] += sum(_is_link(base / d) for d in dirnames) dirnames.sort() for name in sorted(filenames): - _scan_file(result, base / name) + _scan_file(result, base / name, real_root) result.artifacts = newest_first(result.artifacts) result.skipped = +result.skipped # drop the zero counts return result diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index 598e516ec..4c45941b0 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -15,6 +15,7 @@ from dokimi.runs.scan import ( Artifact, head_version, + inside, load_document, plain, read_artifact, @@ -48,22 +49,20 @@ class Shown: siblings_error: str | None = None -def _inside(root: Path, path: Path) -> bool: - base, target = (os.path.normcase(os.path.abspath(p)) for p in (root, path)) - try: - return os.path.commonpath([base, target]) == base - except ValueError: # another drive on Windows - return False - +def resolve(root: Path, address: str) -> tuple[Path, Path]: + """The file ``address`` names, and the root its path is relative to (both absolute). -def resolve(root: Path, address: str) -> Path: - """The file ``address`` names: a path relative to the root (as ``runs list`` prints it), or - an absolute or working-directory path inside the root. Raises ``InputError`` otherwise.""" + ``address`` is a path relative to the root (as ``runs list`` prints it), or an absolute or + working-directory path; once symlinks are resolved it must lie inside the resolved root, so + nothing outside the root is read. Raises ``InputError`` otherwise. + """ candidates = [root / address, Path(address)] + real_root = os.path.realpath(root) try: # Python 3.11-3.13 raise PermissionError here for a directory that cannot be searched path = next((p for p in candidates if p.exists()), candidates[0]) - if not _inside(root, path): - raise InputError(f"{address} is not under the runs root {root.as_posix()}/") + if not inside(real_root, os.path.realpath(path)): + where = "leads outside" if inside(root, path) else "is not under" # via a symlink + raise InputError(f"{address} {where} the runs root {root.as_posix()}/") if not path.exists(): raise InputError(f"no file {address} under the runs root {root.as_posix()}/") if path.is_dir(): @@ -72,7 +71,9 @@ def resolve(root: Path, address: str) -> Path: raise InputError(f"{address} is not a regular file") except OSError as exc: raise InputError(f"{address}: {read_error_reason(exc)}") from exc - return Path(os.path.abspath(path)) + if inside(root, path): # as spelled: keep the name runs list prints + return Path(os.path.abspath(path)), Path(os.path.abspath(root)) + return Path(os.path.realpath(path)), Path(real_root) def _version(path: Path, address: str) -> str: @@ -88,7 +89,7 @@ def _version(path: Path, address: str) -> str: return version -def _siblings(shown: Shown, path: Path) -> None: +def _siblings(shown: Shown, path: Path, real_root: str) -> None: try: with os.scandir(path.parent) as listing: entries = sorted((e for e in listing if e.name != path.name), key=lambda e: e.name) @@ -101,7 +102,9 @@ def _siblings(shown: Shown, path: Path) -> None: if entry.is_dir(): shown.siblings.append(Sibling(entry.name + "/")) continue - readable = entry.name.lower().endswith(".json") and entry.is_file() # not a FIFO + readable = (entry.name.lower().endswith(".json") and entry.is_file() # not a FIFO + and (not entry.is_symlink() + or inside(real_root, os.path.realpath(entry.path)))) version = head_version(Path(entry.path)) if readable else None except OSError: # a link loop, or an entry it may not stat: named, not read version = None @@ -114,16 +117,16 @@ def show(root: Path, address: str) -> Shown: Raises ``InputError`` when ``address`` is outside the root, missing, a directory, not an artifact of a registered kind, or not readable as one. """ - path = resolve(root, address) + path, base = resolve(root, address) version = _version(path, address) kind = VERSION_KINDS[version] try: data = load_document(path, version) - artifact = read_artifact(Path(os.path.abspath(root)), path, version, data) + artifact = read_artifact(base, path, version, data) except (OSError, ValueError, RecursionError) as exc: raise InputError(f"{address}: {read_error_reason(exc)}") from exc schema = schema_version_for(kind, data) shown = Shown(artifact=artifact, schema=schema, errors=schema_errors(schema, data), contents=contents(kind, data)) - _siblings(shown, path) + _siblings(shown, path, os.path.realpath(root)) return shown diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 928a3d642..bfefdcd79 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -293,3 +293,29 @@ def test_a_directory_that_cannot_be_walked_is_unreadable_not_a_crash(tmp_path, m (bad,) = result.unreadable expected = "locked" if mode == 0o000 else "locked/sub" assert (bad.path.as_posix(), bad.reason) == (expected, "cannot list: Permission denied") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_file_symlink_is_read_only_when_it_resolves_inside_the_root(tmp_path): + root = tmp_path / "results" + _save(root, "eval.v1", "arms/real.json") + _save(tmp_path, "eval.v1", "outside/eval.json") + (root / "latest.json").symlink_to(root / "arms" / "real.json") + (root / "external.json").symlink_to(tmp_path / "outside" / "eval.json") + + result = scan(root) + + assert sorted(a.path.as_posix() for a in result.artifacts) == ["arms/real.json", "latest.json"] + assert result.skipped == {"linked_outside": 1} + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_root_that_is_itself_a_symlink_is_read(tmp_path): + _save(tmp_path, "eval.v1", "store/e.json") + (tmp_path / "store" / "latest.json").symlink_to(tmp_path / "store" / "e.json") + (tmp_path / "results").symlink_to(tmp_path / "store", target_is_directory=True) + + result = scan(tmp_path / "results") + + assert sorted(a.path.as_posix() for a in result.artifacts) == ["e.json", "latest.json"] + assert not result.skipped diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index fcbd60e2f..6671b5696 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -302,3 +302,48 @@ def test_an_address_in_a_directory_that_cannot_be_searched_exits_2(monkeypatch, # Python 3.11-3.13 raise PermissionError on the lookup; 3.14 reports the file as missing. assert err in ("Error: arms/onetbb.json: cannot read: Permission denied\n", "Error: no file arms/onetbb.json under the runs root results/\n") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +@pytest.mark.parametrize("address", ["linked/eval.json", "external.json"]) +def test_a_symlink_that_leads_outside_the_root_is_not_read(monkeypatch, capsys, root, address): + save_artifact("eval", _fixture("eval.v1"), root.parent / "ext" / "eval.json") + (root / "linked").symlink_to(root.parent / "ext", target_is_directory=True) + (root / "external.json").symlink_to(root.parent / "ext" / "eval.json") + + err = _fails(monkeypatch, capsys, address) + + assert err == f"Error: {address} leads outside the runs root results/\n" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_symlink_inside_the_root_is_shown_by_its_own_name(monkeypatch, capsys, root): + (root / "latest.json").symlink_to(root / "arms" / "onetbb.json") + + data = json.loads(_show(monkeypatch, capsys, "latest.json", "--json").out) + + assert (data["path"], data["kind"]) == ("latest.json", "arms") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_the_real_path_of_a_symlinked_root_is_inside_it(monkeypatch, capsys, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("DOKIMI_RUNS_DIR", raising=False) + save_artifact("eval", _fixture("eval.v1"), tmp_path / "store" / "e.json") + (tmp_path / "results").symlink_to(tmp_path / "store", target_is_directory=True) + + data = json.loads(_show(monkeypatch, capsys, str(tmp_path / "store" / "e.json"), "--json").out) + + assert data["path"] == "e.json" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_a_neighbour_that_links_outside_the_root_is_named_not_read(monkeypatch, capsys, root): + save_artifact("eval", _fixture("eval.v1"), root.parent / "ext" / "eval.json") + (root / "arms" / "ext.json").symlink_to(root.parent / "ext" / "eval.json") + (root / "arms" / "in.json").symlink_to(root / "arms" / "plugin_delta.json") + + data = json.loads(_show(monkeypatch, capsys, "arms/onetbb.json", "--json").out) + + assert {"name": "ext.json", "schema_version": None} in data["siblings"] + assert {"name": "in.json", "schema_version": "plugin_delta.v1"} in data["siblings"] From bba4f969cfe07cc410db709e9db66f06623119ce Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 07:53:54 -0700 Subject: [PATCH 08/18] test(runs): a time that is not ISO 8601, an artifact that cannot be opened Two documented behaviours had no test (Codecov on #307): runs list prints a recorded time it cannot parse as written, and runs show exits 2 on an artifact it may not open. The open error now takes its wording from read_error_reason, as every other read error does; the message is unchanged. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/runs/show.py | 2 +- dokimi/tests/test_runs_cli.py | 8 ++++++++ dokimi/tests/test_runs_show.py | 12 ++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index 4c45941b0..e1c2a71a1 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -80,7 +80,7 @@ def _version(path: Path, address: str) -> str: try: version = head_version(path) except OSError as exc: - raise InputError(f"{address}: cannot read: {exc.strerror or exc}") from exc + raise InputError(f"{address}: {read_error_reason(exc)}") from exc if version is None: raise InputError(f"{address} is not an artifact: it does not open with a schema_version") if version not in VERSION_KINDS: diff --git a/dokimi/tests/test_runs_cli.py b/dokimi/tests/test_runs_cli.py index 399a40e20..64ada5ceb 100644 --- a/dokimi/tests/test_runs_cli.py +++ b/dokimi/tests/test_runs_cli.py @@ -216,3 +216,11 @@ def test_a_root_under_a_directory_that_cannot_be_searched_exits_2(tmp_path, monk assert capsys.readouterr().err in ( f"Error: runs root DOKIMI_RUNS_DIR={named}: cannot read: Permission denied\n", f"Error: runs root DOKIMI_RUNS_DIR={named} does not exist\n") + + +def test_a_time_that_is_not_iso_8601_is_shown_as_written(monkeypatch, capsys, root): + save_artifact("eval", {**_fixture("eval.v1"), "evaluated_at": "yesterday"}, root / "e.json") + + rows = _list(monkeypatch, capsys).out.splitlines() + + assert rows[1].split()[:4] == ["eval.v1", "-", "demo-judge", "yesterday"] diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 6671b5696..9e826783c 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -347,3 +347,15 @@ def test_a_neighbour_that_links_outside_the_root_is_named_not_read(monkeypatch, assert {"name": "ext.json", "schema_version": None} in data["siblings"] assert {"name": "in.json", "schema_version": "plugin_delta.v1"} in data["siblings"] + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="POSIX permissions; root reads every file") +def test_an_artifact_that_cannot_be_opened_exits_2(monkeypatch, capsys, root): + (root / "arms" / "onetbb.json").chmod(0o000) + try: + err = _fails(monkeypatch, capsys, "arms/onetbb.json") + finally: + (root / "arms" / "onetbb.json").chmod(0o644) + + assert err == "Error: arms/onetbb.json: cannot read: Permission denied\n" From bdfefd49e485e9aa58c6d545a4bd18c851a0b77d Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 07:57:23 -0700 Subject: [PATCH 09/18] docs(runs): only a file of a registered kind is parsed, so only it can fail to docs/runs.md said a file that does not parse is listed as unreadable. Only a file that opens with a registered schema_version is parsed; a broken file of no registered kind is counted as skipped. Say so, and that the text summary names the first unreadable file while --json lists all. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/docs/runs.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md index 08f57a0cb..2c2a7015d 100644 --- a/dokimi/docs/runs.md +++ b/dokimi/docs/runs.md @@ -53,9 +53,10 @@ skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_vers `schema_version` (named), `*.tmp` and `*.rejected` files left by `save_artifact`, other files, entries that are not regular files (a FIFO), symlinked directories, which are not walked, and symlinks to files outside - the root, which are not read. A file or directory that - cannot be read, or a file that does not parse, is listed as unreadable, with - the first one's reason. + the root, which are not read. A file or directory that cannot be read is + listed as unreadable, as is a file that opens with a registered + `schema_version` but does not parse (a file of no registered kind is never + parsed). The text summary names the first one's reason; `--json` lists all. - `--kind KIND` (repeatable, for example `--kind arms --kind eval`) and `--subject NAME` (an exact, case-sensitive match) filter the rows. - `--json` prints `root`, `artifacts`, `total` (before filtering), `skipped`, From 98caf53deb320dc39dfc0aa169d3bbbffa35d6b2 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 07:57:23 -0700 Subject: [PATCH 10/18] fix(runs): name the address and the root escaped in error messages cli.main prints an InputError as it is, so a runs show address or a DOKIMI_RUNS_DIR with a newline or an escape sequence reached stderr raw. Every message of resolve(), _version(), show(), RunsRoot.exists() and runs show's missing-root error now names them through plain(); the paths used to read files are unchanged. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/commands/runs.py | 2 +- dokimi/dokimi/runs/scan.py | 2 +- dokimi/dokimi/runs/show.py | 20 +++++++++++--------- dokimi/tests/test_runs_cli.py | 11 +++++++++++ dokimi/tests/test_runs_show.py | 7 +++++++ 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/dokimi/dokimi/commands/runs.py b/dokimi/dokimi/commands/runs.py index 9ae3405cf..97ab0927e 100644 --- a/dokimi/dokimi/commands/runs.py +++ b/dokimi/dokimi/commands/runs.py @@ -142,7 +142,7 @@ def cmd_runs_show(args: argparse.Namespace) -> None: check is printed, and the exit code stays 0.""" root = runs_root() if not root.exists(): - raise InputError(f"no runs root: {root.path.as_posix()}/ does not exist " + raise InputError(f"no runs root: {plain(root.path.as_posix())}/ does not exist " f"(DOKIMI_RUNS_DIR names another root)") shown = show(root.path, args.path) if args.json: diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py index 450440d3c..e2b471ea0 100644 --- a/dokimi/dokimi/runs/scan.py +++ b/dokimi/dokimi/runs/scan.py @@ -46,7 +46,7 @@ def exists(self) -> bool: Raises :class:`~dokimi.utils.InputError` when ``DOKIMI_RUNS_DIR`` names a missing path, when the root is something other than a directory, or when it cannot be looked up. """ - named = f"{ENV_VAR}={self.path}" if self.from_env else str(self.path) + named = plain(f"{ENV_VAR}={self.path}" if self.from_env else str(self.path)) try: # Python 3.11-3.13 raise PermissionError here for a parent that cannot be searched if self.path.is_dir(): return True diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index e1c2a71a1..f1985bbd2 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -58,33 +58,35 @@ def resolve(root: Path, address: str) -> tuple[Path, Path]: """ candidates = [root / address, Path(address)] real_root = os.path.realpath(root) + label, shown_root = plain(address), plain(root.as_posix()) # for messages only try: # Python 3.11-3.13 raise PermissionError here for a directory that cannot be searched path = next((p for p in candidates if p.exists()), candidates[0]) if not inside(real_root, os.path.realpath(path)): where = "leads outside" if inside(root, path) else "is not under" # via a symlink - raise InputError(f"{address} {where} the runs root {root.as_posix()}/") + raise InputError(f"{label} {where} the runs root {shown_root}/") if not path.exists(): - raise InputError(f"no file {address} under the runs root {root.as_posix()}/") + raise InputError(f"no file {label} under the runs root {shown_root}/") if path.is_dir(): - raise InputError(f"{address} is a directory; runs show takes one artifact file") + raise InputError(f"{label} is a directory; runs show takes one artifact file") if not path.is_file(): - raise InputError(f"{address} is not a regular file") + raise InputError(f"{label} is not a regular file") except OSError as exc: - raise InputError(f"{address}: {read_error_reason(exc)}") from exc + raise InputError(f"{label}: {read_error_reason(exc)}") from exc if inside(root, path): # as spelled: keep the name runs list prints return Path(os.path.abspath(path)), Path(os.path.abspath(root)) return Path(os.path.realpath(path)), Path(real_root) def _version(path: Path, address: str) -> str: + label = plain(address) try: version = head_version(path) except OSError as exc: - raise InputError(f"{address}: {read_error_reason(exc)}") from exc + raise InputError(f"{label}: {read_error_reason(exc)}") from exc if version is None: - raise InputError(f"{address} is not an artifact: it does not open with a schema_version") + raise InputError(f"{label} is not an artifact: it does not open with a schema_version") if version not in VERSION_KINDS: - raise InputError(f"{address} is not an artifact of a registered kind: " + raise InputError(f"{label} is not an artifact of a registered kind: " f"schema_version {plain(version)}") return version @@ -124,7 +126,7 @@ def show(root: Path, address: str) -> Shown: data = load_document(path, version) artifact = read_artifact(base, path, version, data) except (OSError, ValueError, RecursionError) as exc: - raise InputError(f"{address}: {read_error_reason(exc)}") from exc + raise InputError(f"{plain(address)}: {read_error_reason(exc)}") from exc schema = schema_version_for(kind, data) shown = Shown(artifact=artifact, schema=schema, errors=schema_errors(schema, data), contents=contents(kind, data)) diff --git a/dokimi/tests/test_runs_cli.py b/dokimi/tests/test_runs_cli.py index 64ada5ceb..b2cf2678d 100644 --- a/dokimi/tests/test_runs_cli.py +++ b/dokimi/tests/test_runs_cli.py @@ -224,3 +224,14 @@ def test_a_time_that_is_not_iso_8601_is_shown_as_written(monkeypatch, capsys, ro rows = _list(monkeypatch, capsys).out.splitlines() assert rows[1].split()[:4] == ["eval.v1", "-", "demo-judge", "yesterday"] + + +@pytest.mark.skipif(os.name == "nt", reason="Windows paths cannot hold control characters") +def test_a_bad_root_is_named_escaped(tmp_path, monkeypatch, capsys): + monkeypatch.setenv("DOKIMI_RUNS_DIR", str(tmp_path / "no\nwhere")) + + with pytest.raises(SystemExit): + _list(monkeypatch, capsys) + + err = capsys.readouterr().err + assert err == f"Error: runs root DOKIMI_RUNS_DIR={tmp_path}/no\\nwhere does not exist\n" diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 9e826783c..85b374c8c 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -359,3 +359,10 @@ def test_an_artifact_that_cannot_be_opened_exits_2(monkeypatch, capsys, root): (root / "arms" / "onetbb.json").chmod(0o644) assert err == "Error: arms/onetbb.json: cannot read: Permission denied\n" + + +@pytest.mark.skipif(os.name == "nt", reason="Windows paths cannot hold control characters") +def test_a_rejected_address_is_named_escaped(monkeypatch, capsys, root): + err = _fails(monkeypatch, capsys, "missing\nforged\x1b[2J.json") + + assert err == "Error: no file missing\\nforged\\x1b[2J.json under the runs root results/\n" From 13a07381ea0a27b04666012356c6b79edf9924fa Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 08:05:54 -0700 Subject: [PATCH 11/18] fix(runs): runs list does not walk into a Windows junction A junction is not a symlink, and os.walk(followlinks=False) still descends into one on Python 3.11-3.13, so runs list read and listed the files of a junction that points outside the root (runs show already refused them by real path), and a junction back into the root would be walked twice. A directory that is a symlink or a junction (st_reparse_tag IO_REPARSE_TAG_MOUNT_POINT, which 3.11 reads too) is now pruned before the walk descends and counted as "symlinked directories and junctions (not walked)". The test runs on Windows only. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/docs/runs.md | 5 +++-- dokimi/dokimi/runs/scan.py | 18 +++++++++++++----- dokimi/tests/test_runs_scan.py | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md index 2c2a7015d..debfcad14 100644 --- a/dokimi/docs/runs.md +++ b/dokimi/docs/runs.md @@ -52,8 +52,9 @@ skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_vers artifacts saved before #27, which did not put it first), JSON of an unknown `schema_version` (named), `*.tmp` and `*.rejected` files left by `save_artifact`, other files, entries that are not regular files (a FIFO), - symlinked directories, which are not walked, and symlinks to files outside - the root, which are not read. A file or directory that cannot be read is + symlinked directories and Windows junctions, which are not walked, and + symlinks to files outside the root, which are not read. A file or directory + that cannot be read is listed as unreadable, as is a file that opens with a registered `schema_version` but does not parse (a file of no registered kind is never parsed). The text summary names the first one's reason; `--json` lists all. diff --git a/dokimi/dokimi/runs/scan.py b/dokimi/dokimi/runs/scan.py index e2b471ea0..2426babdf 100644 --- a/dokimi/dokimi/runs/scan.py +++ b/dokimi/dokimi/runs/scan.py @@ -129,7 +129,7 @@ def to_json(self) -> dict: "rejected": "*.rejected (payloads save_artifact refused)", "not_json": "not .json", "not_a_file": "not a regular file (a FIFO, socket or device)", - "linked_directory": "symlinked directories (not walked)", + "linked_directory": "symlinked directories and junctions (not walked)", "linked_outside": "symlinks to files outside the root (not read)", } @@ -245,9 +245,16 @@ def _scan_file(result: ScanResult, path: Path, real_root: str) -> None: result.unreadable.append(Unreadable(rel, read_error_reason(exc), version)) +#: ``st_reparse_tag`` of a Windows directory junction (``stat.IO_REPARSE_TAG_MOUNT_POINT``). +_JUNCTION_TAG = 0xA0000003 + + def _is_link(path: Path) -> bool: + """A symlink, or a Windows junction (os.walk descends into junctions on Python 3.11-3.13).""" try: - return path.is_symlink() + if path.is_symlink(): + return True + return getattr(os.lstat(path), "st_reparse_tag", 0) == _JUNCTION_TAG except OSError: # a directory listed but not searchable: os.walk reports it when it enters return False @@ -272,9 +279,10 @@ def unlisted(exc: OSError) -> None: result.unreadable.append(Unreadable(where, f"cannot list: {exc.strerror or exc}")) for directory, dirnames, filenames in os.walk(root, onerror=unlisted): - base = Path(directory) # os.walk does not descend into a linked directory: count it - result.skipped["linked_directory"] += sum(_is_link(base / d) for d in dirnames) - dirnames.sort() + base = Path(directory) + linked = {d for d in dirnames if _is_link(base / d)} + result.skipped["linked_directory"] += len(linked) + dirnames[:] = sorted(d for d in dirnames if d not in linked) for name in sorted(filenames): _scan_file(result, base / name, real_root) result.artifacts = newest_first(result.artifacts) diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index bfefdcd79..bc87bcaff 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -319,3 +319,18 @@ def test_a_root_that_is_itself_a_symlink_is_read(tmp_path): assert sorted(a.path.as_posix() for a in result.artifacts) == ["e.json", "latest.json"] assert not result.skipped + + +@pytest.mark.skipif(os.name != "nt", reason="junctions are a Windows directory link") +def test_a_junction_is_counted_and_not_walked(tmp_path): + import _winapi + + root = tmp_path / "results" + _save(root, "eval.v1", "in.json") + _save(tmp_path, "eval.v1", "outside/eval.json") + _winapi.CreateJunction(str(tmp_path / "outside"), str(root / "external")) + + result = scan(root) + + assert [a.path.as_posix() for a in result.artifacts] == ["in.json"] + assert result.skipped == {"linked_directory": 1} From 33f8422c3f31623468b5a8aa8db815fcac527499 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 08:05:54 -0700 Subject: [PATCH 12/18] fix(runs): runs show reads the file the filesystem opens for link/.. resolve() checked the address as the filesystem resolves it, then made it absolute with os.path.abspath, which folds "link/.." before the symlink is followed: with results/link -> results/nested/deep, runs show link/../eval.json checked results/nested/eval.json and read results/eval.json. The path is now made absolute with Path.absolute(), which keeps "..", and an address with ".." is shown and read by its real path. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/runs/show.py | 5 +++-- dokimi/tests/test_runs_show.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index f1985bbd2..982e770b6 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -72,8 +72,9 @@ def resolve(root: Path, address: str) -> tuple[Path, Path]: raise InputError(f"{label} is not a regular file") except OSError as exc: raise InputError(f"{label}: {read_error_reason(exc)}") from exc - if inside(root, path): # as spelled: keep the name runs list prints - return Path(os.path.abspath(path)), Path(os.path.abspath(root)) + spelled = path.absolute() # not abspath: `link/..` must stay what the filesystem opens + if ".." not in spelled.parts and inside(root, spelled): # keep the name runs list prints + return spelled, root.absolute() return Path(os.path.realpath(path)), Path(real_root) diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 85b374c8c..6e5257265 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -366,3 +366,15 @@ def test_a_rejected_address_is_named_escaped(monkeypatch, capsys, root): err = _fails(monkeypatch, capsys, "missing\nforged\x1b[2J.json") assert err == "Error: no file missing\\nforged\\x1b[2J.json under the runs root results/\n" + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_dot_dot_after_a_symlink_is_the_file_the_filesystem_opens(monkeypatch, capsys, root): + save_artifact("eval", _fixture("eval.v1"), root / "nested" / "eval.json") + save_artifact("plugin_delta", _fixture("plugin_delta.v1"), root / "eval.json") + (root / "nested" / "deep").mkdir() + (root / "link").symlink_to(root / "nested" / "deep", target_is_directory=True) + + data = json.loads(_show(monkeypatch, capsys, "link/../eval.json", "--json").out) + + assert (data["kind"], data["path"]) == ("eval", "nested/eval.json") From bdc14acc80120a0c59ec91af28426d4f8564a1d8 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 08:21:06 -0700 Subject: [PATCH 13/18] test(runs): raise the RecursionError instead of finding it by nesting The test nested 100 000 arrays to make json raise RecursionError. On Python 3.14 the C parser is bounded by the C stack, not the recursion limit, and parses that document, so the test failed on CI's 3.14 leg while the scanner was right. The parse now raises RecursionError through a patched load_document, and the test checks that the file is listed as unreadable with that reason, on every interpreter. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/tests/test_runs_scan.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index bc87bcaff..13fc5d2d3 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -6,6 +6,7 @@ from __future__ import annotations +import importlib import json import os import time @@ -20,6 +21,8 @@ from dokimi.utils import InputError FIXTURES = Path(__file__).resolve().parent / "fixtures" / "artifacts" +# The module, not the scan() that dokimi.runs re-exports under the same name. +scan_module = importlib.import_module("dokimi.runs.scan") #: version -> (keys added to its fixture, expected subject, model, time). EXPECTED: dict[str, tuple[dict, str | None, str | None, str | None]] = { @@ -238,13 +241,19 @@ def test_a_schema_version_past_the_first_bytes_is_not_a_leading_one(tmp_path): assert scan(tmp_path).skipped == {"no_leading_schema_version": 1} -def test_a_document_nested_too_deep_to_parse_is_unreadable(tmp_path): - deep = '{"schema_version": "eval.v1", "x": ' + "[" * 100_000 + "]" * 100_000 + "}" - (tmp_path / "deep.json").write_text(deep, encoding="utf-8") +def test_a_document_nested_too_deep_to_parse_is_unreadable(tmp_path, monkeypatch): + # How deep json can go depends on the interpreter (3.14 bounds it by C stack), so the + # RecursionError is raised for the parse instead of found by nesting. + def too_deep(path, version): + raise RecursionError("maximum recursion depth exceeded while decoding a JSON array") + + _save(tmp_path, "eval.v1", "deep.json") + monkeypatch.setattr(scan_module, "load_document", too_deep) (bad,) = scan(tmp_path).unreadable - assert bad.schema_version == "eval.v1" and "recursion" in bad.reason + assert (bad.path.as_posix(), bad.schema_version) == ("deep.json", "eval.v1") + assert bad.reason == "maximum recursion depth exceeded while decoding a JSON array" def test_a_time_out_of_range_falls_back_to_the_mtime(tmp_path): From 942ecdf32730d3038f6f812b8599018869816c04 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 08:21:06 -0700 Subject: [PATCH 14/18] docs(runs): the schema_version must open the first 256 bytes docs/runs.md and the changelog fragment said a file is an artifact when it opens with a registered schema_version. head_version reads only the first 256 bytes, so a file whose first key comes later (after 1000 spaces, say) is skipped. Say that; save_artifact's output always fits. Refs #278 Signed-off-by: Vladislav Perevezentsev --- .../20261030_120000_vlad-perevezentsev_runs-list.md | 4 ++-- dokimi/docs/runs.md | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md index cdfe54953..e5d054fed 100644 --- a/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md +++ b/dokimi/changelog.d/20261030_120000_vlad-perevezentsev_runs-list.md @@ -11,8 +11,8 @@ SPDX-License-Identifier: Apache-2.0 `--kind` (repeatable) and `--subject` (an exact match) filter the rows, and `--json` prints them with the fields `schema_version`, `kind`, `run_id` (always `null` until artifacts carry one), `subject`, `model`, `generated_at`, `path` - and `size_bytes`. A file is an artifact when it opens with the - `schema_version` of a registered kind; every other file is counted by why it + and `size_bytes`. A file is an artifact when its first 256 bytes open with + the `schema_version` of a registered kind, as `save_artifact` writes it; every other file is counted by why it was skipped, and a file that cannot be read is listed with its reason. A missing `results/` prints "no runs" and exits 0; a `DOKIMI_RUNS_DIR` that does not exist or is not a directory exits 2. The command only reads. diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md index debfcad14..140f941dd 100644 --- a/dokimi/docs/runs.md +++ b/dokimi/docs/runs.md @@ -33,9 +33,10 @@ novelty.v1 skill:skills/onetbb-tuning claude-sonnet-5-5 2026-10-27 18:30 skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_version (pairwise.v0: 1), 1 not .json ``` -- **Each row is one artifact file.** A file is an artifact when it opens with the - `schema_version` of a registered kind: `save_artifact` has written it as the - first key since [#27](https://github.com/intel/dokimi/pull/27). Superseded +- **Each row is one artifact file.** A file is an artifact when its first 256 + bytes open with the `schema_version` of a registered kind: `save_artifact` + has written it as the first key since + [#27](https://github.com/intel/dokimi/pull/27), well inside that. Superseded versions such as `questions.v1` count too. Files that open with anything else are skipped without being read in full, so a large file of another kind costs nothing. From 2f31691cd1d6a9be8be609cee6da70cf9e74780e Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 08:40:02 -0700 Subject: [PATCH 15/18] fix(runs): runs show lists neighbours from inside the root only resolve() accepts an address whose real path is inside the root, and keeps the spelled path for display. With results/linked -> /elsewhere and /elsewhere/return.json -> results/real.json, runs show linked/return.json was accepted, and its neighbours were listed from /elsewhere: outside file names and schema_versions in the output. When the spelled directory does not resolve inside the root, the neighbours now come from the artifact's own (real) directory. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/runs/show.py | 2 ++ dokimi/tests/test_runs_show.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index 982e770b6..40e0d81d7 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -93,6 +93,8 @@ def _version(path: Path, address: str) -> str: def _siblings(shown: Shown, path: Path, real_root: str) -> None: + if not inside(real_root, os.path.realpath(path.parent)): # reached through a link out and back + path = Path(os.path.realpath(path)) # list the artifact's own directory, inside the root try: with os.scandir(path.parent) as listing: entries = sorted((e for e in listing if e.name != path.name), key=lambda e: e.name) diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 6e5257265..253143b65 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -378,3 +378,16 @@ def test_dot_dot_after_a_symlink_is_the_file_the_filesystem_opens(monkeypatch, c data = json.loads(_show(monkeypatch, capsys, "link/../eval.json", "--json").out) assert (data["kind"], data["path"]) == ("eval", "nested/eval.json") + + +@pytest.mark.skipif(os.name == "nt", reason="symlinks need privileges on Windows") +def test_neighbours_are_never_listed_from_outside_the_root(monkeypatch, capsys, root): + save_artifact("eval", _fixture("eval.v1"), root.parent / "ext" / "secret.json") + (root / "linked").symlink_to(root.parent / "ext", target_is_directory=True) + (root.parent / "ext" / "return.json").symlink_to(root / "arms" / "onetbb.json") + + data = json.loads(_show(monkeypatch, capsys, "linked/return.json", "--json").out) + + assert data["kind"] == "arms" + assert [s["name"] for s in data["siblings"]] == ["onetbb-matrix/", "onetbb.md", + "plugin_delta.json"] From 34e76e1ea06d7ba383b3cce4069c72868ef8759e Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Fri, 30 Oct 2026 09:38:58 -0700 Subject: [PATCH 16/18] fix(runs): runs show takes an absolute address under a root named with .. resolve() kept the spelled path when the address had no "..", and returned it with root.absolute(). For a root such as DOKIMI_RUNS_DIR=../results that base still holds "..", and read_artifact's lexical relative_to failed: an absolute address of a valid artifact exited 2. A ".." in the root now takes the real-path branch too, so the address is shown relative to the real root. Refs #278 Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/runs/show.py | 6 +++--- dokimi/tests/test_runs_show.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/runs/show.py b/dokimi/dokimi/runs/show.py index 40e0d81d7..b407c51cc 100644 --- a/dokimi/dokimi/runs/show.py +++ b/dokimi/dokimi/runs/show.py @@ -72,9 +72,9 @@ def resolve(root: Path, address: str) -> tuple[Path, Path]: raise InputError(f"{label} is not a regular file") except OSError as exc: raise InputError(f"{label}: {read_error_reason(exc)}") from exc - spelled = path.absolute() # not abspath: `link/..` must stay what the filesystem opens - if ".." not in spelled.parts and inside(root, spelled): # keep the name runs list prints - return spelled, root.absolute() + spelled, base = path.absolute(), root.absolute() # not abspath: `link/..` must stay as opened + if ".." not in spelled.parts + base.parts and inside(root, spelled): # the name list prints + return spelled, base return Path(os.path.realpath(path)), Path(real_root) diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 253143b65..04f13f849 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -391,3 +391,17 @@ def test_neighbours_are_never_listed_from_outside_the_root(monkeypatch, capsys, assert data["kind"] == "arms" assert [s["name"] for s in data["siblings"]] == ["onetbb-matrix/", "onetbb.md", "plugin_delta.json"] + + +def test_an_absolute_address_under_a_root_named_with_dot_dot(tmp_path, monkeypatch, capsys): + save_artifact("eval", _fixture("eval.v1"), tmp_path / "results" / "e.json") + (tmp_path / "work").mkdir() + monkeypatch.chdir(tmp_path / "work") + monkeypatch.setenv("DOKIMI_RUNS_DIR", os.path.join("..", "results")) + + by_path = json.loads(_show(monkeypatch, capsys, str(tmp_path / "results" / "e.json"), + "--json").out) + by_name = json.loads(_show(monkeypatch, capsys, "e.json", "--json").out) + + assert (by_path["path"], by_path["kind"]) == (by_name["path"], by_name["kind"]) == ("e.json", + "eval") From 391a9f6a86048355598c11e1e89b8e6cd2e339dc Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 30 Oct 2026 21:00:16 -0700 Subject: [PATCH 17/18] ci(windows): run the runs scan and show tests in windows-smoke test_a_junction_is_counted_and_not_walked skips off Windows, so no CI job ran the only test of the runs-root containment on Windows. Signed-off-by: Alexander Rybkin --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f13b08c9..dc4abe517 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -487,5 +487,6 @@ jobs: cd "$RUNNER_TEMP" && python "$smoke" - name: Install test deps run: python -m pip install --require-hashes -r requirements.lock.txt + # test_runs_scan.py holds the only junction test of the runs-root containment; it runs on Windows only. - name: Run a few fast tests - run: pytest -q tests/test_cli_help.py tests/test_console_encoding.py + run: pytest -q tests/test_cli_help.py tests/test_console_encoding.py tests/test_runs_scan.py tests/test_runs_show.py From 7b73110bad5c9d843ac85228ccd28a6dcb8c9a3c Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Fri, 30 Oct 2026 21:05:56 -0700 Subject: [PATCH 18/18] fix(benchmark): raise max_docs to 37 for docs/runs.md Main reached 36 docs with #255, and this branch adds docs/runs.md, so the merged tree has 37. Signed-off-by: Alexander Rybkin --- dokimi/benchmarks/spec.v1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index fbaf738fb..b36efa758 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 36 + max_docs: 37 include: - docs/**/*.md exclude: