From d18e6705c9e395f1c1572a2b9af5dbc3c0515296 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:27:08 -0700 Subject: [PATCH 01/43] feat(repos): dokimi:typecheck grader and repo scan --typecheck dokimi:typecheck passes when the task's listed workspace files have no mypy error, under the image's pinned mypy version. repo scan --typecheck checks each Python fence of the guidance against --python-env: a failing fence is a stale-API candidate, an unparsable fence or a scan without an interpreter is unresolved. The report gains an optional typecheck block. Signed-off-by: Nikolay Petrov --- ...20261031_120000_napetrov_repo-typecheck.md | 8 + dokimi/docs/reference/cli-interface.json | 35 +++ dokimi/docs/repo-evaluation-contract.md | 45 ++++ dokimi/dokimi/commands/repo.py | 84 +++++-- dokimi/dokimi/repos/graders/__init__.py | 7 +- dokimi/dokimi/repos/graders/main.py | 11 +- dokimi/dokimi/repos/graders/review.py | 4 +- dokimi/dokimi/repos/graders/typecheck.py | 104 +++++++++ dokimi/dokimi/repos/l0/__init__.py | 17 +- dokimi/dokimi/repos/l0/typecheck.py | 162 ++++++++++++++ dokimi/dokimi/repos/manifest.py | 4 +- dokimi/dokimi/schemas/repo_readiness.v1.json | 1 + dokimi/pyproject.toml | 3 + dokimi/scripts/growth_baseline.json | 1 - dokimi/tests/data/cli_flags_allowlist.txt | 2 + dokimi/tests/test_repo_l0_typecheck.py | 206 ++++++++++++++++++ dokimi/tests/test_repo_typecheck_grader.py | 135 ++++++++++++ 17 files changed, 792 insertions(+), 37 deletions(-) create mode 100644 dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md create mode 100644 dokimi/dokimi/repos/graders/typecheck.py create mode 100644 dokimi/dokimi/repos/l0/typecheck.py create mode 100644 dokimi/tests/test_repo_l0_typecheck.py create mode 100644 dokimi/tests/test_repo_typecheck_grader.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md new file mode 100644 index 000000000..e7ef56bf0 --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md @@ -0,0 +1,8 @@ +### Added + +- **`dokimi:typecheck` built-in grader and `repo scan --typecheck` (Python fences).** The grader type-checks the + task's listed files with the image's pinned mypy and passes when none has an error (`tool`, `version`, `paths` + in `task.json`; a mismatched version is a grader failure). `repo scan --typecheck [mypy|basedpyright]` checks + each Python fence of the guidance against `--python-env`'s site-packages: a fence with an error is a `fail` + stale-API candidate, a fence that does not parse or has no interpreter is `unresolved`. The report gains a + `typecheck` block, `null` without the flag. See `docs/repo-evaluation-contract.md` §4.5 and §8.2. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..b8c921e40 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -11677,6 +11677,22 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--python-env", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -11692,6 +11708,25 @@ "position": null, "required": false, "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'basedpyright'", + "'mypy'" + ], + "const": "mypy", + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--typecheck", + "kind": "option", + "nargs": "?", + "position": null, + "required": false, + "type": null } ], "path": "dokimi repo scan", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c2b873922..ba49e2b65 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -626,6 +626,17 @@ failed the task, and on a task with `score` that is a failed run with score |---|---|---|---|---| | `dokimi:review_seeds` | `manifest.json` (`manifest`); oracle answer `oracle.json` (`oracle_answer`), a list of findings | `null`; for a manifest with no seeds (a clean-diff control), `true` when no finding is `blocker` or `major` | `n_findings`, `n_severe`, `n_decoys`, `unmatched`, `recall` (`null` with no seeds) | `seeds_hit`, `decoys_flagged`, `recall_by_class` (`"hit/total"`) | | `dokimi:review_replay` | `gold.json` (`gold`); `task.json` `window`, default 5 | `null` (score-only) | `n_findings`, `n_severe`, `n_threads`, `recall` (= `recall_all`), `recall_addressed`, `precision_proxy` | `threads_hit`, `matches`, `recall_by_kind`, `decoys_flagged` (always `[]`) | +| `dokimi:typecheck` | none: the task's own fields `tool` (`mypy`), `version` (the image's mypy, exactly), `paths` (workspace-relative files) | `true` when no error is reported on a listed file and every listed file exists, else `false` | `n_errors`, `n_missing`, `n_files` | `errors` (first 50, `file:line: message [code]`), `missing`, `tool`, `tool_version` | + +`dokimi:typecheck` checks the listed files only. Its pass is "no error on a +listed file" (an error in a module a listed file imports belongs to that module, +not to the listed file), and a listed file that is missing fails. +It runs mypy in-process in the grading container, so the image's `python3` +must import the pinned mypy: a `version` other than the installed one is a +grader failure, not a fail. It is not "no new errors against the base": the +grader is stdlib-only and cannot run `git` to read the base, and it does not +start basedpyright (the L0 check, §8.2, runs both tools on the host). A repository that +needs that comparison writes it in its own `grade` entry point. `manifest.json` is `{"seeds": [{id, class, file, lines, kw}], "decoys": [{id, file, lines, kw}]}`; `file` `"*"` matches any file, and `kw` is a @@ -1335,6 +1346,40 @@ pass, CI secrets and the agent's network. `risk` is the worst of the launch observations. There is no overall score. `--fail-on-error` exits 1 when any finding is an error. +### 8.2 Python fences (`repo scan --typecheck`) + +`--typecheck [mypy|basedpyright]` (bare: mypy) type-checks every fence tagged +`python`, `py` or `python3` in the guidance files. Each fence that parses is one +`python_fence` claim, so the report and `--fail-on-fail` treat it like any other +claim: + +- `ok`: the tool reports no error in the fence. +- `fail`: a stale-API candidate. The claim's line is the error's line in the + guidance file, and its `why` names the first error (`mypy: line N: message`). +- `unresolved`: the fence is not valid Python (a placeholder such as `f()`), + or no `--python-env` was given, so nothing was checked. The reason says which. + +`--python-env PYTHON` names the repository's interpreter, a virtualenv with its +pinned dependencies installed (and the repository itself, if its fences import +it). The fences are checked against that site-packages with imports followed +silently (`--follow-imports=silent`), so only errors in the fence count. A +module the environment does not have is a `fail`: a fence that imports an +uninstalled package is a fence the pinned environment cannot run. mypy is the +host's pinned copy; basedpyright must be on `PATH` and is recorded with the +version it reports, not pinned by dokimi. The report's `typecheck` block names +the tool, its version, the interpreter (`null`: unknown) and the counts +(`fences`, `checked`, `fail`, `unknown`, `syntax`); it is `null` without the +flag. `--python-env` needs `--typecheck`. The tools run on the host through +`dokimi.process`; one run covers every fence of the repository. + +```bash +python cli.py repo scan --manifest --typecheck --python-env /bin/python +``` + +A guidance note under test can be scored the same way as any other content: +feed the note as a `patch:` arm (see §3) and grade the arm's workspace with +the `dokimi:typecheck` grader above. + ## 9. Running `repo tasks check` and `repo run` need docker. Both export `eval_dir` from the diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 88f64ad09..20352c123 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -8,7 +8,9 @@ import argparse import json import math +import os import re +import shutil import sys from pathlib import Path from typing import Any @@ -122,15 +124,39 @@ def _read_static(path: str) -> Any: sys.exit(EXIT_USAGE) +def _typecheck_or_exit(args: argparse.Namespace): + """The ``--typecheck`` check, or None; ``--python-env`` needs it and must name an interpreter.""" + from dokimi.repos.l0 import Check + + if args.typecheck is None: + if args.python_env is not None: + print("✗ --python-env needs --typecheck", file=sys.stderr) + sys.exit(EXIT_USAGE) + return None + if args.python_env is not None and not os.access(args.python_env, os.X_OK): + print(f"✗ --python-env {args.python_env}: not an executable interpreter", file=sys.stderr) + sys.exit(EXIT_USAGE) + if args.typecheck != "mypy" and shutil.which(args.typecheck) is None: + print(f"✗ --typecheck {args.typecheck}: not on PATH", file=sys.stderr) + sys.exit(EXIT_USAGE) + return Check(args.typecheck, args.python_env) + + def cmd_repo_scan(args: argparse.Namespace) -> None: """L0: check every mechanically checkable claim in the guidance against the same tree.""" from dokimi.artifacts import save_artifact, stamp from dokimi.repos.l0 import scan_repo + from dokimi.repos.l0.typecheck import TypecheckError _, tree = _source_tree(args) manifest = _manifest_or_exit(tree, args, guidance_default=True, also_usage=(OSError,)) static = _read_static(args.static) if args.static else None - payload = scan_repo(tree, manifest, static) + check = _typecheck_or_exit(args) + try: + payload = scan_repo(tree, manifest, static, typecheck=check) + except TypecheckError as exc: + print(f"✗ --typecheck {check.tool}: {exc}", file=sys.stderr) + sys.exit(EXIT_USAGE) if args.out: try: save_artifact("repo_readiness", payload, Path(args.out)) @@ -144,30 +170,41 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: if args.json: print(json.dumps(stamp("repo_readiness", dict(payload)), indent=2)) else: - c = payload["counts"] - detected = " (detected)" if payload["adapters_detected"] else "" - print(f"{manifest.id}: {len(payload['files'])} guidance files in {payload['repo']['tree']}; " - f"adapters: {', '.join(payload['adapters']) or 'none'}{detected}") - print(f" claims: {c['total']} (ok {c['ok']}, unresolved {c['unresolved']}, fail {c['fail']})") - for warning in payload["warnings"]: - print(f" warning: {warning}") - if payload["incomplete"]: - print(f" incomplete: {'; '.join(payload['incomplete_reasons'])} (repo scan --fail-on-fail exits 1)") - if static_info is not None and not static_errors: - r = payload["by_source"]["repository"] - print(f" repository static: {r['total']} claims, {r['fail']} fail; " - f"{static_info['superseded']} framework claims replaced") - for claim in payload["claims"]: - if claim["status"].split("_", 1)[0] == "fail": - print(f" FAIL {claim['file']}:{claim['line']} [{claim.get('kind')}] " - f"{str(claim.get('claim'))[:90]} -- {claim.get('why')}") - if args.out: - print(f"wrote {args.out}") + _print_scan_text(manifest, payload, args.out) _scan_gate(payload, args.fail_on_fail) # runs first: an invalid --static still gets the incomplete reason if static_errors: sys.exit(1) +def _print_scan_text(manifest, payload: dict, out: str | None) -> None: + """The text report of ``repo scan``: the counts, the warnings, the typecheck and static lines, the FAIL lines.""" + c = payload["counts"] + detected = " (detected)" if payload["adapters_detected"] else "" + print(f"{manifest.id}: {len(payload['files'])} guidance files in {payload['repo']['tree']}; " + f"adapters: {', '.join(payload['adapters']) or 'none'}{detected}") + print(f" claims: {c['total']} (ok {c['ok']}, unresolved {c['unresolved']}, fail {c['fail']})") + for warning in payload["warnings"]: + print(f" warning: {warning}") + if payload["incomplete"]: + print(f" incomplete: {'; '.join(payload['incomplete_reasons'])} (repo scan --fail-on-fail exits 1)") + tc = payload["typecheck"] + if tc is not None: + where = tc["python"] or "no --python-env: unknown" + print(f" typecheck {tc['tool']} {tc['tool_version'] or ''} ({where}): {tc['fences']} python fences, " + f"{tc['checked']} checked, {tc['fail']} fail, {tc['unknown']} unknown, {tc['syntax']} not Python") + static_info = payload["static"] + if static_info is not None and not static_info["errors"]: + r = payload["by_source"]["repository"] + print(f" repository static: {r['total']} claims, {r['fail']} fail; " + f"{static_info['superseded']} framework claims replaced") + for claim in payload["claims"]: + if claim["status"].split("_", 1)[0] == "fail": + print(f" FAIL {claim['file']}:{claim['line']} [{claim.get('kind')}] " + f"{str(claim.get('claim'))[:90]} -- {claim.get('why')}") + if out: + print(f"wrote {out}") + + def _scan_gate(payload: dict, fail_on_fail: bool) -> None: """--fail-on-fail: exit 1 on a failed claim or an incomplete scan, saying on stderr what the scan missed.""" if not fail_on_fail or not (payload["counts"]["fail"] or payload["incomplete"]): @@ -474,6 +511,13 @@ def register(sub, positive_int) -> None: "the report, its claims replacing the framework's on the same file, line and text. " "A file that cannot be read or is not JSON exits 2; a schema-invalid document is " "reported and makes the command exit 1") + scan_p.add_argument("--typecheck", nargs="?", const="mypy", default=None, choices=("mypy", "basedpyright"), + help="Type-check each Python fence of the guidance (§8.2): one claim per fence, fail when the " + "tool reports an error. mypy is the host's pinned copy; basedpyright must be on PATH. Without " + "--python-env the fences are unknown (unresolved), not checked. Bare --typecheck means mypy") + scan_p.add_argument("--python-env", default=None, dest="python_env", metavar="PYTHON", + help="The repository's interpreter (a venv with its pinned dependencies installed); the fences " + "are checked against its site-packages. Needs --typecheck") scan_p.add_argument("--out", default=None, help="Write the repo_readiness.v1 report to this file") scan_p.add_argument("--json", action="store_true", help="Print the full report as JSON") scan_p.add_argument("--fail-on-fail", action="store_true", dest="fail_on_fail", diff --git a/dokimi/dokimi/repos/graders/__init__.py b/dokimi/dokimi/repos/graders/__init__.py index 373cc322f..5f0f800b6 100644 --- a/dokimi/dokimi/repos/graders/__init__.py +++ b/dokimi/dokimi/repos/graders/__init__.py @@ -15,12 +15,15 @@ from pathlib import Path -from .review import GRADERS +from . import typecheck +from .review import GRADERS as REVIEW_GRADERS PREFIX = "dokimi:" -NAMES = tuple(GRADERS) +NAMES = (*REVIEW_GRADERS, *typecheck.GRADERS) # the data file each built-in grader reads next to task.json: the task key that renames it, and its default DATA_FILES = {"review_seeds": ("manifest", "manifest.json"), "review_replay": ("gold", "gold.json")} +# graders whose options are the task's own fields, checked by ``tasks check`` instead of a data file +TASK_PROBLEMS = {"typecheck": typecheck.task_problems} HOST_DIR = Path(__file__).resolve().parent ENTRY = "main.py" diff --git a/dokimi/dokimi/repos/graders/main.py b/dokimi/dokimi/repos/graders/main.py index 7d5536444..a8c1d2a34 100644 --- a/dokimi/dokimi/repos/graders/main.py +++ b/dokimi/dokimi/repos/graders/main.py @@ -28,15 +28,16 @@ from pathlib import Path if __package__: - from . import review + from . import review, typecheck else: # a script under -I: its own directory is not on sys.path # By name, not `import review`: dependency hygiene (deptry) reads a bare import as a third-party package. sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) review = importlib.import_module("review") + typecheck = importlib.import_module("typecheck") -GRADERS = dict(review.GRADERS) -ORACLES = dict(review.ORACLES) -VERSIONS = {name: review.VERSION for name in review.GRADERS} +GRADERS = {**review.GRADERS, **typecheck.GRADERS} +ORACLES = {**review.ORACLES, **typecheck.ORACLES} +VERSIONS = {**{name: review.VERSION for name in review.GRADERS}, **{name: typecheck.VERSION for name in typecheck.GRADERS}} RUN_DIR = "/eval/run" @@ -76,7 +77,7 @@ def grade(name, task_dir, run_dir): try: answer = run_dir / "answer.txt" text = answer.read_text(encoding="utf-8") if answer.exists() else "" - g = GRADERS[name](t, task_dir, text) + g = GRADERS[name](t, task_dir, text, run_dir) out["metrics"], out["details"] = split_result(g) for k in gate_names(t): # a gate metric the grader did not reach is unmeasured, not failed out["metrics"].setdefault(k, None) diff --git a/dokimi/dokimi/repos/graders/review.py b/dokimi/dokimi/repos/graders/review.py index 7eeeac56f..e69767ee7 100644 --- a/dokimi/dokimi/repos/graders/review.py +++ b/dokimi/dokimi/repos/graders/review.py @@ -181,7 +181,7 @@ def _per_group(entries, key, hit): return {k: f"{a}/{b}" for k, (a, b) in counts.items()} -def grade_seeds(task, task_dir, answer): +def grade_seeds(task, task_dir, answer, run_dir=None): """``review_seeds``: the result dict (``pass`` and flat metrics, lists and objects for details).""" man = _read_json(task_dir, task.get("manifest", "manifest.json")) seeds, decoys = man.get("seeds", []), man.get("decoys", []) @@ -250,7 +250,7 @@ def _match(g, i, f, distance): "message": _cut(f.get("message")), "distance": distance} -def grade_replay(task, task_dir, answer): +def grade_replay(task, task_dir, answer, run_dir=None): """``review_replay``: score-only, ``recall`` over the human threads.""" gold = _read_json(task_dir, task.get("gold", "gold.json")) findings = findings_of(answer) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py new file mode 100644 index 000000000..f51170fb1 --- /dev/null +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -0,0 +1,104 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``dokimi:typecheck``: the listed files type-check clean under the image's pinned mypy (contract §4.5). + +task.json: ``"tool": "mypy"``, ``"version"`` (exactly the image's mypy version, else a grader +failure) and ``"paths"`` (workspace-relative files). ``pass`` is true when no error is reported +on a listed file and every listed file exists. Errors in other modules are not counted. + +It runs mypy in-process: the grader cannot start a child (the import layers allow spawns only in +``dokimi.process``, which the container cannot import), so basedpyright is not offered here. The +host-side L0 check (``repo scan --typecheck``) runs both tools. Standard library only at import +time; mypy is imported by :func:`grade` and must be importable by the image's ``python3``. +""" + +from __future__ import annotations + +import os +import posixpath +import re +import shutil +import tempfile +from pathlib import Path +from typing import Any + +# Bump when a result for the same inputs changes (a metric or ``pass``). +VERSION = "1" +TOOL = "mypy" +MAX_ERRORS = 50 # reported in details; the count is in the metrics +_ERROR = re.compile(r"^(?P[^\n:]+):(?P\d+):(?:\d+:)? error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") + + +def _safe_path(path: object) -> bool: + """A workspace-relative path: no leading slash, no ``..`` segment, not empty.""" + return isinstance(path, str) and bool(path) and not path.startswith("/") and ".." not in path.split("/") + + +def task_problems(raw: dict[str, Any]) -> list[str]: + """Problems with a ``dokimi:typecheck`` task's own fields (checked by ``tasks check``).""" + problems = [] + if raw.get("tool", TOOL) != TOOL: + problems.append(f"tool must be {TOOL!r}: basedpyright starts a process, which the grader does not " + f"(the L0 check runs it), not {raw.get('tool')!r}") + if not isinstance(raw.get("version"), str) or not raw.get("version"): + problems.append('version must be the pinned mypy version, e.g. "2.4.0"') + paths = raw.get("paths") + if not isinstance(paths, list) or not paths or not all(_safe_path(p) for p in paths): + problems.append("paths must be a non-empty list of workspace-relative file paths") + return problems + + +def _errors(stdout: str, listed: set[str]) -> list[dict[str, Any]]: + """The error lines of mypy's output on a listed file, in output order.""" + out = [] + for line in stdout.splitlines(): + m = _ERROR.match(line) + if m and posixpath.normpath(m.group("file")) in listed: + out.append({"file": posixpath.normpath(m.group("file")), "line": int(m.group("line")), + "code": m.group("code") or "", "message": m.group("message")}) + return out + + +def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | None = None) -> dict[str, Any]: + """Type-check the listed files in the workspace of ``run_dir``; the grader's result dict.""" + del task_dir, answer # the answer is not read: the workspace is the subject + problems = task_problems(task) + if problems: + raise ValueError("; ".join(problems)) + import mypy.version + from mypy import api + + if mypy.version.__version__ != task["version"]: + raise ValueError(f"the image's mypy is {mypy.version.__version__}, the task pins {task['version']}") + workspace = Path(run_dir or "/eval/run") / "workspace" + paths = [posixpath.normpath(p) for p in task["paths"]] + missing = [p for p in paths if not (workspace / p).is_file()] + present = [p for p in paths if p not in missing] + errors: list[dict[str, Any]] = [] + if present: + cache = tempfile.mkdtemp(prefix="dokimi-mypy-") + cwd = os.getcwd() + try: + os.chdir(workspace) + stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", *present]) + finally: + os.chdir(cwd) + shutil.rmtree(cache, ignore_errors=True) + if status not in (0, 1): # 2: mypy itself failed (a bad config, a crash) + raise RuntimeError(f"mypy exited {status}: {stderr[-300:]}") + errors = _errors(stdout, set(present)) + shown = [f"{e['file']}:{e['line']}: {e['message']}" + (f" [{e['code']}]" if e["code"] else "") for e in errors] + return {"pass": not errors and not missing, "n_errors": len(errors), "n_missing": len(missing), + "n_files": len(paths), "errors": shown[:MAX_ERRORS], "missing": missing, + "tool": TOOL, "tool_version": mypy.version.__version__} + + +def oracle(task: dict[str, Any], task_dir: Path) -> str: + """No reference answer: the grade reads the workspace only.""" + del task, task_dir + return "" + + +GRADERS = {"typecheck": grade} +ORACLES = {"typecheck": oracle} diff --git a/dokimi/dokimi/repos/l0/__init__.py b/dokimi/dokimi/repos/l0/__init__.py index 796eb510d..607d15184 100644 --- a/dokimi/dokimi/repos/l0/__init__.py +++ b/dokimi/dokimi/repos/l0/__init__.py @@ -19,8 +19,9 @@ from ..static import STATUS_CLASSES, check_static, status_class from .adapters import ADAPTERS, detect from .core import Claim, Scan +from .typecheck import Check, check_fences -__all__ = ["ADAPTERS", "Claim", "Scan", "guidance_files", "scan_repo"] +__all__ = ["ADAPTERS", "Check", "Claim", "Scan", "guidance_files", "scan_repo"] def guidance_files(tree: RepoTree, manifest: RepoManifest) -> list[str]: @@ -52,12 +53,14 @@ def _is_build_input(path: str) -> bool: or name in ("BUILD", "BUILD.bazel", "WORKSPACE", "WORKSPACE.bazel", "MODULE.bazel")) -def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | None = None) -> dict[str, Any]: +def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | None = None, + typecheck: Check | None = None) -> dict[str, Any]: """The ``repo_readiness.v1`` payload (without ``schema_version``) for ``tree``. ``static`` is the document the repository's ``static`` entry point wrote. Its claims replace the framework's claim on the same file, line and text, - because the repository knows its own rules better. + because the repository knows its own rules better. ``typecheck`` adds the + ``python_fence`` claims (§8.2); None leaves the Python fences unchecked. """ files = guidance_files(tree, manifest) scan = Scan(tree, frozenset(manifest.build_systems), repo_url=manifest.source_url, guidance=tuple(files)) @@ -68,9 +71,10 @@ def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | N skipped: dict[str, str] = {} scan.preload(files, skipped) aliases = _aliases(tree, files) - for f in files: - if f not in skipped and f not in aliases: - scan.check_file(f) + checked = [f for f in files if f not in skipped and f not in aliases] + for f in checked: + scan.check_file(f) + typecheck_block = None if typecheck is None else check_fences(scan, checked, typecheck) framework = [dict(c.to_dict(), source="framework") for c in scan.claims] static_info: dict[str, Any] | None = None @@ -108,6 +112,7 @@ def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | N "by_kind": _grouped(claims, "kind"), "by_file": _grouped(claims, "file"), "static": static_info, + "typecheck": typecheck_block, "warnings": warnings, "incomplete": bool(reasons), "incomplete_reasons": reasons, diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py new file mode 100644 index 000000000..21ca4e310 --- /dev/null +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -0,0 +1,162 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Python fences of the guidance, type-checked against the repository's environment (contract §8.2). + +Each fence that parses becomes a ``python_fence`` claim: ``ok`` when the tool reports no error, +``fail`` (a stale-API candidate) when it does, and ``unresolved`` when it does not parse or no +repository interpreter was given, so nothing was checked. One tool run covers every fence; the +interpreter's site-packages are the pinned dependencies, and imports are not followed past the fence. +""" + +from __future__ import annotations + +import ast +import json +import os +import sys +import tempfile +import textwrap +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from dokimi import process + +from .core import Scan +from .markdown import PYTHON_LANGS, Fence, split + +TOOLS = ("mypy", "basedpyright") +TIMEOUT = 600 # seconds for one run over every fence of a repository +MAX_WHY = 200 + + +class TypecheckError(RuntimeError): + """The tool could not run or exited with a failure of its own (not a type error).""" + + +@dataclass(frozen=True) +class Check: + tool: str # ``mypy`` (the host's pinned copy) or ``basedpyright`` (on PATH) + python: str | None = None # the repository's interpreter; None means unknown, nothing is checked + + +def python_fences(scan: Scan, files: list[str]) -> list[tuple[str, Fence, str]]: + """Every fence tagged as Python in ``files``: (file, fence, dedented code).""" + out = [] + for f in files: + for fence in split(scan.text(f)).fences: + if fence.lang in PYTHON_LANGS: + out.append((f, fence, textwrap.dedent("\n".join(text for _, text in fence.lines)))) + return out + + +def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: + """Add a ``python_fence`` claim per fence to ``scan``; the report's ``typecheck`` block.""" + if check.tool not in TOOLS: + raise TypecheckError(f"unknown tool {check.tool!r}; one of {', '.join(TOOLS)}") + found = python_fences(scan, files) + block: dict[str, Any] = {"tool": check.tool, "tool_version": None, "python": check.python, + "fences": len(found), "checked": 0, "fail": 0, "unknown": 0, "syntax": 0} + runnable = [] + for f, fence, code in found: + try: + ast.parse(code) + except (SyntaxError, ValueError) as exc: # a placeholder such as `f()` is not Python + block["syntax"] += 1 + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + f"not valid Python ({getattr(exc, 'msg', exc)}): not type-checked") + continue + runnable.append((f, fence, code)) + if check.python is None: + for f, fence, code in runnable: + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + "unknown: no repository environment (--python-env), not type-checked") + block["unknown"] = len(runnable) + return block + block["tool_version"] = tool_version(check) + found_errors = _run(check, [code for _, _, code in runnable]) + for i, (f, fence, code) in enumerate(runnable): + errors = found_errors.get(i, []) + if not errors: + scan.add(f, fence.line, "python_fence", _label(code), "ok", f"{check.tool}: no error") + continue + block["fail"] += 1 + line, message = errors[0] + more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" + scan.add(f, fence.line + line, "python_fence", _label(code), "fail", + f"{check.tool}: line {line}: {message}{more}"[:MAX_WHY]) + block["checked"] = len(runnable) + return block + + +def tool_version(check: Check) -> str: + """The version the tool reports, e.g. ``2.4.0``.""" + argv = [sys.executable, "-m", "mypy", "--version"] if check.tool == "mypy" else [check.tool, "--version"] + done = _run_tool(argv, os.getcwd()) + parts = done.stdout.split() + return parts[1] if len(parts) > 1 else done.stdout.strip() + + +def _label(code: str) -> str: + """The fence's first non-blank line, for the claim's text.""" + first = next((ln.strip() for ln in code.splitlines() if ln.strip()), "") + return first[:80] + + +def _run(check: Check, codes: list[str]) -> dict[int, list[tuple[int, str]]]: + """Errors per fence index: (line in the fence, message), in tool order.""" + with tempfile.TemporaryDirectory(prefix="dokimi-typecheck-") as tmp: + names = [] + for i, code in enumerate(codes): + Path(tmp, f"fence_{i}.py").write_text(code + "\n", encoding="utf-8") + names.append(f"fence_{i}.py") + if check.tool == "mypy": + argv = [sys.executable, "-m", "mypy", "--python-executable", str(check.python), + "--follow-imports=silent", "--no-incremental", "--cache-dir", os.path.join(tmp, "cache"), + "--output", "json", *names] + else: + argv = [check.tool, "--pythonpath", str(check.python), "--outputjson", *names] + done = _run_tool(argv, tmp) + if done.timed_out or done.returncode not in (0, 1): # 1: the tool found errors; anything else is its failure + raise TypecheckError(f"{check.tool} exited {done.returncode}: {done.stderr[-300:]}") + return _errors(check.tool, done.stdout) + + +def _run_tool(argv: list[str], cwd: str): + try: + return process.run(argv, timeout=TIMEOUT, env=process.Env(allow=None), cwd=cwd) + except OSError as exc: + raise TypecheckError(f"cannot run {argv[0]}: {exc}") from exc + + +def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: + """Parse the tool's JSON output into errors keyed by fence index.""" + out: dict[int, list[tuple[int, str]]] = {} + if tool == "mypy": # one JSON object per line + diagnostics = [] + for text in stdout.splitlines(): + try: + d = json.loads(text) + except ValueError: + continue + if isinstance(d, dict) and d.get("severity") == "error": + diagnostics.append((str(d.get("file")), int(d.get("line") or 0), str(d.get("message")))) + else: # basedpyright: one document; its ranges are zero-based + doc = json.loads(stdout or "{}") + diagnostics = [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, + str(d.get("message")) + (f" [{d['rule']}]" if d.get("rule") else "")) + for d in doc.get("generalDiagnostics", []) if d.get("severity") == "error"] + for name, line, message in diagnostics: + index = _fence_index(name) + if index is not None: + out.setdefault(index, []).append((line, message)) + return out + + +def _fence_index(name: str) -> int | None: + """``3`` for ``fence_3.py``; None for any other file (a note about an import, say).""" + stem = Path(name).name + if stem.startswith("fence_") and stem.endswith(".py") and stem[6:-3].isdigit(): + return int(stem[6:-3]) + return None diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 57aa5ce6a..b138951f9 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -31,7 +31,7 @@ from dokimi.artifacts import schema_errors from dokimi.measure import SCORE_BOUND, as_float, is_finite -from .graders import DATA_FILES, NAMES as BUILTIN_NAMES, PREFIX as BUILTIN_PREFIX +from .graders import DATA_FILES, NAMES as BUILTIN_NAMES, PREFIX as BUILTIN_PREFIX, TASK_PROBLEMS from .host_spec import HostSpecError, parse_host from .process_metrics import Counter, parse_counter from .source import RepoTree @@ -596,6 +596,8 @@ def _builtin_problems(tree: RepoTree, task_dir: str, raw: dict[str, Any], name: if name not in BUILTIN_NAMES: return [f"unknown built-in grader {BUILTIN_PREFIX + name!r}; known: " + ", ".join(BUILTIN_PREFIX + n for n in BUILTIN_NAMES)] + if name in TASK_PROBLEMS: + return TASK_PROBLEMS[name](raw) key, default = DATA_FILES[name] path = raw.get(key, default) if not isinstance(path, str): diff --git a/dokimi/dokimi/schemas/repo_readiness.v1.json b/dokimi/dokimi/schemas/repo_readiness.v1.json index 6340840af..8b9bca3a3 100644 --- a/dokimi/dokimi/schemas/repo_readiness.v1.json +++ b/dokimi/dokimi/schemas/repo_readiness.v1.json @@ -65,6 +65,7 @@ } ] }, + "typecheck": {"oneOf": [{"type": "null"}, {"type": "object", "description": "The Python-fence check (§8.2): tool and its version, the interpreter (null: unknown), fence counts."}], "description": "null when the check was not requested."}, "warnings": {"type": "array", "items": {"type": "string"}, "description": "Guidance the manifest names but the tree lacks: a glob with no match, a missing also_scan file, no guidance at all."}, "incomplete": {"type": "boolean", "description": "True when an also_scan file is missing, a guidance file or Bazel build input cannot be read, or no guidance file was scanned; `repo scan --fail-on-fail` then exits 1."}, "incomplete_reasons": {"type": "array", "items": {"type": "string"}, "description": "The warnings that make the scan incomplete, as they appear in `warnings` and in its order; empty when the scan is complete. Absent from reports written before it was added."}, diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index da8072efd..1f668e054 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -142,6 +142,9 @@ optional_dependencies_dev_groups = ["dev"] DEP001 = ["audit_mocks"] # Declared only to cap the version ragas resolves (see the comment in the ragas extra). DEP002 = ["langchain-community"] +# repos/graders/typecheck.py imports mypy inside the grading image, where it is installed +# for the grader; the wheel does not ship it. The host never imports it (the L0 check runs it as a tool). +DEP004 = ["mypy"] [tool.ruff] # The width `ruff format` wraps to, if someone runs it: neither CI nor diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index d751c4643..5fa393004 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -22,7 +22,6 @@ "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 30, "rationale": "pre-existing when the gate was introduced"}, "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13, "rationale": "W8 adds the --repeats guard after main's no_queries handling raised the file; the command is split with the rest of novelty.py in a separate PR"}, - "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 12}, "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "went over the cap on main in #120 (built-in review graders) after #93 seeded this baseline; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, "dokimi/commands/tools.py::cmd_tools_preference": {"complexity": 11}, diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..2f2c03bb2 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -148,6 +148,7 @@ --product --prompt-file # repo tasks replay (#265) adds it: no registry concept yet; a F1b migration names it --provider +--python-env # repo scan --typecheck: the interpreter whose site-packages the fences resolve against; no registry concept yet --queries --queries-by-skill # novelty routing (#172) adds it: no registry concept yet; a F1b migration names it --queries-from-headings @@ -205,6 +206,7 @@ --trials # catalog trigger-probe (#217) adds it: a banned synonym of reps; renaming it changes the CLI, so it waits for F1b --trigger-set --turn-reminder +--typecheck # repo scan's Python-fence check (mypy or basedpyright): no registry concept yet --typed-max-len --typed-threads --validate diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py new file mode 100644 index 000000000..3b7c78ee1 --- /dev/null +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -0,0 +1,206 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""L0 Python-fence check (contract §8.2): fences type-checked against an interpreter, or unknown without one.""" + +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos import parse_manifest +from dokimi.repos.l0 import Check, scan_repo +from dokimi.repos.source import WorkTree +from dokimi.repos.l0.typecheck import _fence_index + +MANIFEST = { + "schema": "repo_manifest.v1", + "id": "toy", + "source": {"url": "https://github.com/acme/toy"}, + "eval_dir": "dev/evals", + "guidance": {"globs": ["**/AGENTS.md"]}, + "entry_points": {"grade": "true"}, + "tasks": "dev/evals/tasks", + "build_systems": ["make"], +} + +FENCE = "```" +AGENTS = f"""# Toy + +{FENCE}python +import json + +print(json.dumps({{"a": 1}})) +{FENCE} + +{FENCE}python +import json + +print(json.dumps(1).nosuch()) +{FENCE} + +{FENCE}python +f() +{FENCE} +""" + + +def _tree(tmp_path: Path, agents: str = AGENTS) -> WorkTree: + root = tmp_path / "r" + root.mkdir() + (root / "AGENTS.md").write_text(agents, encoding="utf-8") + return WorkTree(root) + + +def _fence_claims(payload: dict) -> list[dict]: + return [c for c in payload["claims"] if c["kind"] == "python_fence"] + + +def test_without_the_check_the_python_fences_are_not_claimed(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST)) + assert payload["typecheck"] is None + assert _fence_claims(payload) == [] + + +def test_mypy_against_an_interpreter_passes_the_good_fence_and_fails_the_stale_one(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + claims = {c["line"]: c for c in _fence_claims(payload)} + assert claims[3]["status"] == "ok" # the good fence opens on line 3 + stale = claims[12] # the stale fence opens on line 9; its error is on code line 3, so the claim is line 12 + assert stale["status"] == "fail" + assert "nosuch" in stale["why"] + assert payload["typecheck"]["fail"] == 1 + assert payload["typecheck"]["checked"] == 2 + + +def test_a_fence_that_is_not_python_is_unresolved_not_failed(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + placeholder = {c["line"]: c for c in _fence_claims(payload)}[15] # opens on line 15 + assert placeholder["status"] == "unresolved" + assert "not valid Python" in placeholder["why"] + assert payload["typecheck"]["syntax"] == 1 + + +def test_without_an_interpreter_every_checkable_fence_is_unknown(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", None)) + fences = _fence_claims(payload) + assert {c["status"] for c in fences} == {"unresolved"} + assert all("unknown" in c["why"] for c in fences if "not valid" not in c["why"]) + assert payload["typecheck"] == {"tool": "mypy", "tool_version": None, "python": None, "fences": 3, + "checked": 0, "fail": 0, "unknown": 2, "syntax": 1} + + +def test_the_fence_line_of_a_failure_is_the_guidance_line(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + failed = [c for c in _fence_claims(payload) if c["status"] == "fail"] + assert [c["line"] for c in failed] == [12] + + +def test_the_report_names_the_tool_and_its_version(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + tc = payload["typecheck"] + assert tc["tool"] == "mypy" + assert tc["tool_version"] and tc["tool_version"][0].isdigit() + + +@pytest.mark.skipif(shutil.which("basedpyright") is None, reason="basedpyright is not installed") +def test_basedpyright_fails_the_same_stale_fence(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), + typecheck=Check("basedpyright", sys.executable)) + failed = [c for c in _fence_claims(payload) if c["status"] == "fail"] + assert [c["line"] for c in failed] == [12] + + +def test_fence_file_names_map_back_to_their_index_only(): + assert _fence_index("fence_3.py") == 3 + assert _fence_index("elsewhere/x/fence_12.py") == 12 + assert _fence_index("fence_x.py") is None + assert _fence_index("pkg/other.py") is None + + +def test_the_scan_command_refuses_python_env_without_typecheck(tmp_path, capsys): + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--python-env", sys.executable]) + from dokimi.commands.repo import _typecheck_or_exit + + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "needs --typecheck" in capsys.readouterr().err + + +def test_the_scan_command_refuses_a_python_env_that_is_not_executable(tmp_path, capsys): + missing = tmp_path / "no-such-python" + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", str(missing)]) + from dokimi.commands.repo import _typecheck_or_exit + + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "not an executable interpreter" in capsys.readouterr().err + + +def test_bare_typecheck_flag_means_mypy(tmp_path): + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck"]) + assert args.typecheck == "mypy" + assert args.python_env is None + + +def test_the_payload_is_json_serialisable(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + json.dumps(payload) + + +def _scan_args(tmp_path: Path, *extra: str): + """Parsed ``repo scan`` arguments for a plain tree whose AGENTS.md is the fixture.""" + tree = tmp_path / "tree" + tree.mkdir() + (tree / "AGENTS.md").write_text(AGENTS, encoding="utf-8") + manifest_file = tmp_path / "m.json" + manifest_file.write_text(json.dumps(MANIFEST), encoding="utf-8") + return build_parser().parse_args(["repo", "scan", str(tree), "--manifest-file", str(manifest_file), *extra]) + + +def test_the_scan_text_report_names_the_tool_and_the_interpreter(tmp_path, capsys): + args = _scan_args(tmp_path, "--typecheck", "--python-env", sys.executable) + args.func(args) + out = capsys.readouterr().out + assert f"({sys.executable}): 3 python fences, 2 checked, 1 fail, 0 unknown, 1 not Python" in out + assert "FAIL AGENTS.md:12 [python_fence]" in out + + +def test_the_scan_text_report_says_unknown_without_an_interpreter(tmp_path, capsys): + args = _scan_args(tmp_path, "--typecheck") + args.func(args) + assert "(no --python-env: unknown)" in capsys.readouterr().out + + +def test_a_tool_failure_is_a_usage_error_not_a_traceback(tmp_path, capsys, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + def broken(check, codes): + raise l0_typecheck.TypecheckError("mypy exited 2: broken") + + monkeypatch.setattr(l0_typecheck, "_run", broken) + args = _scan_args(tmp_path, "--typecheck", "--python-env", sys.executable) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "--typecheck mypy: mypy exited 2: broken" in capsys.readouterr().err + + +def test_basedpyright_missing_from_path_is_a_usage_error(tmp_path, capsys, monkeypatch): + empty = tmp_path / "empty-path" + empty.mkdir() + monkeypatch.setenv("PATH", str(empty)) + args = _scan_args(tmp_path, "--typecheck", "basedpyright") + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "--typecheck basedpyright: not on PATH" in capsys.readouterr().err diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py new file mode 100644 index 000000000..39cf5adbb --- /dev/null +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -0,0 +1,135 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The built-in grader ``dokimi:typecheck`` (contract §4.5): listed files type-check clean under the pinned mypy.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import mypy.version +import pytest + +from dokimi.repos.graders import main, typecheck + +PINNED = mypy.version.__version__ + + +def _task(**over) -> dict: + task = {"id": "typed", "grader": "dokimi:typecheck", "version": PINNED, "paths": ["pkg/api.py"]} + task.update(over) + return task + + +def _run(tmp_path: Path, files: dict[str, str], task: dict) -> Path: + """A run directory whose workspace holds ``files``; returns the run directory.""" + run = tmp_path / "run" + for name, text in files.items(): + path = run / "workspace" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + (run / "task.json").write_text(json.dumps(task), encoding="utf-8") + return run + + +CLEAN = "def add(a: int, b: int) -> int:\n return a + b\n" +WRONG = "def add(a: int, b: int) -> int:\n return f'{a}{b}'\n" + + +def test_listed_file_that_type_checks_passes(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_errors"] == 0 + assert result["tool_version"] == PINNED + + +def test_listed_file_with_a_type_error_fails_and_names_the_error(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + assert result["errors"][0].startswith("pkg/api.py:2:") + assert "[return-value]" in result["errors"][0] + + +def test_an_error_in_an_unlisted_module_is_not_counted(tmp_path): + other = "def broken() -> int:\n return 'no'\n" + api = "from pkg.other import broken\n\ndef use() -> int:\n return broken()\n" + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/other.py": other, "pkg/api.py": api}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_errors"] == 0 + + +def test_a_listed_file_that_is_missing_fails_without_running_the_tool(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": ""}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["missing"] == ["pkg/api.py"] + assert result["n_errors"] == 0 + + +def test_a_pinned_version_other_than_the_image_is_a_grader_error(tmp_path): + run = _run(tmp_path, {"pkg/api.py": CLEAN}, _task(version="0.0.1")) + with pytest.raises(ValueError, match="pins 0.0.1"): + typecheck.grade(_task(version="0.0.1"), tmp_path, "", run) + + +def test_the_entry_point_turns_a_grader_error_into_pass_none_with_errors(tmp_path): + run = _run(tmp_path, {"pkg/api.py": CLEAN}, _task(version="0.0.1")) + out = main.grade("typecheck", tmp_path, run) + assert out["pass"] is None + assert out["errors"] and "pins 0.0.1" in out["errors"][0] + assert json.loads((run / "grade.json").read_text(encoding="utf-8"))["pass"] is None + + +def test_the_entry_point_writes_the_metrics_that_the_gate_reads(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + out = main.grade("typecheck", tmp_path, run) + assert out["pass"] is False + assert out["metrics"]["n_errors"] == 1 + assert out["metrics"]["n_missing"] == 0 + assert out["details"]["grader_version"] == typecheck.VERSION + + +@pytest.mark.parametrize("over", [ + {"tool": "basedpyright"}, + {"version": ""}, + {"version": None}, + {"paths": []}, + {"paths": ["/etc/passwd"]}, + {"paths": ["../outside.py"]}, + {"paths": "pkg/api.py"}, +]) +def test_task_problems_refuse_bad_fields(over): + task = _task() + task.update(over) + assert typecheck.task_problems(task) + + +def test_task_problems_accept_a_well_formed_task(): + assert typecheck.task_problems(_task()) == [] + + +def test_the_manifest_checks_the_task_fields_at_parse_time(): + from dokimi.repos.manifest import _builtin_problems + + raw = _task(version="") + assert _builtin_problems(None, "tasks/typed", raw, "typecheck") + assert _builtin_problems(None, "tasks/typed", _task(), "typecheck") == [] + + +def test_the_grader_script_loads_its_siblings_by_name(monkeypatch): + import runpy + import sys + + from dokimi.repos.graders import main as grader_main + + monkeypatch.setattr(sys, "path", list(sys.path)) + for name in ("review", "typecheck"): + monkeypatch.delitem(sys.modules, name, raising=False) + script = runpy.run_path(grader_main.__file__, run_name="grader_script") # a script: no package + assert script["GRADERS"]["typecheck"] is script["typecheck"].GRADERS["typecheck"] + assert script["VERSIONS"]["typecheck"] == typecheck.VERSION From a6a152343d715813516a5549c2564b167d2f4f60 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:45:55 -0700 Subject: [PATCH 02/43] fix(repos): typecheck fails closed and reads the subject's config never Self-review of the typecheck grader and repo scan --typecheck: a missing mypy (exit 1, no report) no longer passes every fence; truncated tool output is an error; a fence-less scan runs no tool; a relative --python-env is made absolute; the grader runs mypy from a neutral directory so a workspace mypy.ini cannot suppress a listed file; the error parser accepts paths with a colon and line-less errors. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo.py | 3 +- dokimi/dokimi/repos/graders/typecheck.py | 23 +++++++----- dokimi/dokimi/repos/l0/typecheck.py | 8 ++++- dokimi/tests/test_repo_l0_typecheck.py | 42 ++++++++++++++++++++++ dokimi/tests/test_repo_typecheck_grader.py | 18 ++++++++++ 5 files changed, 84 insertions(+), 10 deletions(-) diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 20352c123..b78c0ec88 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -139,7 +139,8 @@ def _typecheck_or_exit(args: argparse.Namespace): if args.typecheck != "mypy" and shutil.which(args.typecheck) is None: print(f"✗ --typecheck {args.typecheck}: not on PATH", file=sys.stderr) sys.exit(EXIT_USAGE) - return Check(args.typecheck, args.python_env) + python = os.path.abspath(args.python_env) if args.python_env is not None else None # the tool runs elsewhere + return Check(args.typecheck, python) def cmd_repo_scan(args: argparse.Namespace) -> None: diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index f51170fb1..5f13216e4 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -27,7 +27,7 @@ VERSION = "1" TOOL = "mypy" MAX_ERRORS = 50 # reported in details; the count is in the metrics -_ERROR = re.compile(r"^(?P[^\n:]+):(?P\d+):(?:\d+:)? error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") +_ERROR = re.compile(r"^(?P.+?)(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") def _safe_path(path: object) -> bool: @@ -49,13 +49,17 @@ def task_problems(raw: dict[str, Any]) -> list[str]: return problems -def _errors(stdout: str, listed: set[str]) -> list[dict[str, Any]]: - """The error lines of mypy's output on a listed file, in output order.""" +def _errors(stdout: str, listed: set[str], root: str) -> list[dict[str, Any]]: + """The error lines of mypy's output on a listed file, in output order; ``root`` is stripped from the names.""" out = [] for line in stdout.splitlines(): m = _ERROR.match(line) - if m and posixpath.normpath(m.group("file")) in listed: - out.append({"file": posixpath.normpath(m.group("file")), "line": int(m.group("line")), + if not m: + continue + name = m.group("file") + name = posixpath.normpath(name[len(root):] if name.startswith(root) else name) + if name in listed: + out.append({"file": name, "line": int(m.group("line") or 0), "code": m.group("code") or "", "message": m.group("message")}) return out @@ -78,16 +82,19 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non errors: list[dict[str, Any]] = [] if present: cache = tempfile.mkdtemp(prefix="dokimi-mypy-") + neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cwd-") # no config here: the subject's mypy.ini is not read cwd = os.getcwd() try: - os.chdir(workspace) - stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", *present]) + os.chdir(neutral) + stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", + *[str(workspace / p) for p in present]]) finally: os.chdir(cwd) shutil.rmtree(cache, ignore_errors=True) + shutil.rmtree(neutral, ignore_errors=True) if status not in (0, 1): # 2: mypy itself failed (a bad config, a crash) raise RuntimeError(f"mypy exited {status}: {stderr[-300:]}") - errors = _errors(stdout, set(present)) + errors = _errors(stdout, set(present), str(workspace) + os.sep) shown = [f"{e['file']}:{e['line']}: {e['message']}" + (f" [{e['code']}]" if e["code"] else "") for e in errors] return {"pass": not errors and not missing, "n_errors": len(errors), "n_missing": len(missing), "n_files": len(paths), "errors": shown[:MAX_ERRORS], "missing": missing, diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index 21ca4e310..e2592faeb 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -75,7 +75,7 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: block["unknown"] = len(runnable) return block block["tool_version"] = tool_version(check) - found_errors = _run(check, [code for _, _, code in runnable]) + found_errors = _run(check, [code for _, _, code in runnable]) if runnable else {} # no file: a usage error for i, (f, fence, code) in enumerate(runnable): errors = found_errors.get(i, []) if not errors: @@ -94,6 +94,8 @@ def tool_version(check: Check) -> str: """The version the tool reports, e.g. ``2.4.0``.""" argv = [sys.executable, "-m", "mypy", "--version"] if check.tool == "mypy" else [check.tool, "--version"] done = _run_tool(argv, os.getcwd()) + if done.timed_out or done.returncode != 0: # a missing module exits 1 with the reason on stderr + raise TypecheckError(f"{check.tool} --version exited {done.returncode}: {done.stderr[-300:]}") parts = done.stdout.split() return parts[1] if len(parts) > 1 else done.stdout.strip() @@ -120,6 +122,10 @@ def _run(check: Check, codes: list[str]) -> dict[int, list[tuple[int, str]]]: done = _run_tool(argv, tmp) if done.timed_out or done.returncode not in (0, 1): # 1: the tool found errors; anything else is its failure raise TypecheckError(f"{check.tool} exited {done.returncode}: {done.stderr[-300:]}") + if done.truncated: # the middle of the output was dropped, so an error there would be missed + raise TypecheckError(f"{check.tool} output was truncated; not every error is known") + if done.returncode == 1 and not done.stdout.strip(): # exit 1 with no report: the tool did not run + raise TypecheckError(f"{check.tool} exited 1 with no report: {done.stderr[-300:]}") return _errors(check.tool, done.stdout) diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 3b7c78ee1..3b7601dbd 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -204,3 +204,45 @@ def test_basedpyright_missing_from_path_is_a_usage_error(tmp_path, capsys, monke args.func(args) assert exc.value.code == 2 assert "--typecheck basedpyright: not on PATH" in capsys.readouterr().err + + +def test_an_interpreter_with_nothing_to_check_runs_no_tool(tmp_path): + payload = scan_repo(_tree(tmp_path, "# Toy\n\nNo code here.\n"), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + assert payload["typecheck"]["checked"] == 0 + assert payload["typecheck"]["fail"] == 0 + + +def _completed(**over): + from types import SimpleNamespace + + done = {"returncode": 0, "stdout": "", "stderr": "", "timed_out": False, "truncated": False} + done.update(over) + return SimpleNamespace(**done) + + +def test_a_tool_that_exits_one_with_no_report_is_an_error_not_a_pass(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd: _completed( + returncode=1, stderr="No module named mypy")) + with pytest.raises(l0_typecheck.TypecheckError, match="no report"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_truncated_tool_output_is_an_error_not_a_partial_result(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd: _completed(truncated=True)) + with pytest.raises(l0_typecheck.TypecheckError, match="truncated"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_a_relative_python_env_is_made_absolute_before_the_tool_runs(tmp_path, monkeypatch): + from dokimi.commands.repo import _typecheck_or_exit + + link = tmp_path / "py" + link.symlink_to(sys.executable) + monkeypatch.chdir(tmp_path) + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", "py"]) + assert Path(_typecheck_or_exit(args).python).is_absolute() diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 39cf5adbb..dad09a503 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -133,3 +133,21 @@ def test_the_grader_script_loads_its_siblings_by_name(monkeypatch): script = runpy.run_path(grader_main.__file__, run_name="grader_script") # a script: no package assert script["GRADERS"]["typecheck"] is script["typecheck"].GRADERS["typecheck"] assert script["VERSIONS"]["typecheck"] == typecheck.VERSION + + +def test_the_subjects_mypy_ini_cannot_suppress_a_listed_file(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG, + "mypy.ini": "[mypy-pkg.api]\nignore_errors = True\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + +def test_a_listed_path_with_a_colon_is_matched(): + out = typecheck._errors("a:b.py:3: error: boom [misc]", {"a:b.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("a:b.py", 3)] + + +def test_a_line_less_error_on_a_listed_file_is_counted(): + out = typecheck._errors("pkg/a.py: error: Duplicate module named 'a'", {"pkg/a.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("pkg/a.py", 0)] From d96d7db82c55f83fbd69c1c8b9e2503baf0accf2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 19:03:34 -0700 Subject: [PATCH 03/43] fix(repo): a fence too deep to parse is unresolved; a line-less mypy error is the fence line ast.parse can raise RecursionError or MemoryError on a deeply nested fence; that aborted repo scan with a traceback. Such a fence is now unresolved like any other unparsable one. mypy writes line -1 for an error with no line; that was added to the fence line, pointing one line above the fence. It now maps to the fence line. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/l0/typecheck.py | 11 +++++++---- dokimi/tests/test_repo_l0_typecheck.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index e2592faeb..ba6108314 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -62,10 +62,11 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: for f, fence, code in found: try: ast.parse(code) - except (SyntaxError, ValueError) as exc: # a placeholder such as `f()` is not Python + except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: # `f()` is not Python block["syntax"] += 1 + why = exc.msg if isinstance(exc, SyntaxError) else type(exc).__name__ # too deep: no parse tree scan.add(f, fence.line, "python_fence", _label(code), "unresolved", - f"not valid Python ({getattr(exc, 'msg', exc)}): not type-checked") + f"not valid Python ({why}): not type-checked") continue runnable.append((f, fence, code)) if check.python is None: @@ -84,8 +85,9 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: block["fail"] += 1 line, message = errors[0] more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" + where = f"line {line}: " if line else "" # 0: the tool gave no line, so the claim is the fence itself scan.add(f, fence.line + line, "python_fence", _label(code), "fail", - f"{check.tool}: line {line}: {message}{more}"[:MAX_WHY]) + f"{check.tool}: {where}{message}{more}"[:MAX_WHY]) block["checked"] = len(runnable) return block @@ -147,7 +149,8 @@ def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: except ValueError: continue if isinstance(d, dict) and d.get("severity") == "error": - diagnostics.append((str(d.get("file")), int(d.get("line") or 0), str(d.get("message")))) + line = int(d.get("line") or 0) # mypy writes -1 for an error with no line + diagnostics.append((str(d.get("file")), max(line, 0), str(d.get("message")))) else: # basedpyright: one document; its ranges are zero-based doc = json.loads(stdout or "{}") diagnostics = [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 3b7601dbd..209f54013 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -246,3 +246,20 @@ def test_a_relative_python_env_is_made_absolute_before_the_tool_runs(tmp_path, m monkeypatch.chdir(tmp_path) args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", "py"]) assert Path(_typecheck_or_exit(args).python).is_absolute() + + +def test_a_fence_too_deep_for_the_parser_is_unresolved_not_a_crash(tmp_path): + deep = "x = " + "+".join(["1"] * 20000) # ast.parse raises RecursionError on this + agents = f"# Toy\n\n{FENCE}python\n{deep}\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", None)) + [claim] = _fence_claims(payload) + assert claim["status"] == "unresolved" + assert "RecursionError" in claim["why"] + assert payload["typecheck"]["syntax"] == 1 + + +def test_a_line_less_mypy_error_is_the_fence_line_not_the_line_above(tmp_path): + from dokimi.repos.l0.typecheck import _errors + + report = json.dumps({"file": "fence_0.py", "line": -1, "severity": "error", "message": "no line"}) + assert _errors("mypy", report) == {0: [(0, "no line")]} From 057ad88ab84371ea203b00966d1fdec9eca2b314 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 19:08:23 -0700 Subject: [PATCH 04/43] docs(repo): the contract lists a too-deep fence among the unresolved cases Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index ba49e2b65..e9cb6e56f 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1357,7 +1357,8 @@ claim: - `fail`: a stale-API candidate. The claim's line is the error's line in the guidance file, and its `why` names the first error (`mypy: line N: message`). - `unresolved`: the fence is not valid Python (a placeholder such as `f()`), - or no `--python-env` was given, so nothing was checked. The reason says which. + is too deeply nested for the parser, or no `--python-env` was given, so nothing + was checked. The reason says which. `--python-env PYTHON` names the repository's interpreter, a virtualenv with its pinned dependencies installed (and the repository itself, if its fences import From 05255f9aea00f0fdcf6d5735074a4a972c945dea Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:18:19 -0700 Subject: [PATCH 05/43] Address review: the typecheck grader's paths, blocking errors, oracle mypy runs with the workspace absolute, under an explicit neutral config whose mypy_path is the workspace root, and without changing directory, so a relative run directory and a namespace package both resolve. A blocking error (exit 2 with a diagnostic) is a result, not a tool failure; when mypy stops before the listed files, that error fails the grade. A task's reference/ directory is copied over the workspace by oracle, so a reference repair is what the self-check grades. The typecheck tests need mypy on the test interpreter: it is added to requirements-test.txt (the Tests (py3.11) and (py3.14) legs failed). Thread: PRRT_kwDOVBg7g86yWj1M Thread: PRRT_kwDOVBg7g86yWj1Q Thread: PRRT_kwDOVBg7g86yWj1T Thread: PRRT_kwDOVBg7g86yWj1V Thread: PRRT_kwDOVBg7g86yWj9q Thread: PRRT_kwDOVBg7g86yWj90 Thread: PRRT_kwDOVBg7g86yWj-A Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/main.py | 2 +- dokimi/dokimi/repos/graders/review.py | 4 +- dokimi/dokimi/repos/graders/typecheck.py | 41 +++++++++----- dokimi/requirements-test.txt | 2 + dokimi/tests/test_repo_typecheck_grader.py | 64 ++++++++++++++++++++++ 5 files changed, 96 insertions(+), 17 deletions(-) diff --git a/dokimi/dokimi/repos/graders/main.py b/dokimi/dokimi/repos/graders/main.py index a8c1d2a34..e8d6d23fe 100644 --- a/dokimi/dokimi/repos/graders/main.py +++ b/dokimi/dokimi/repos/graders/main.py @@ -93,7 +93,7 @@ def grade(name, task_dir, run_dir): def oracle(name, task_dir, run_dir): """Write the reference answer to ``run_dir/answer.txt``; returns it.""" - text = ORACLES[name](_read_task(run_dir), task_dir) + text = ORACLES[name](_read_task(run_dir), task_dir, run_dir) if text: (run_dir / "answer.txt").write_text(text, encoding="utf-8") return text diff --git a/dokimi/dokimi/repos/graders/review.py b/dokimi/dokimi/repos/graders/review.py index e69767ee7..d3d2d93ef 100644 --- a/dokimi/dokimi/repos/graders/review.py +++ b/dokimi/dokimi/repos/graders/review.py @@ -211,7 +211,7 @@ def grade_seeds(task, task_dir, answer, run_dir=None): return out -def oracle_seeds(task, task_dir): +def oracle_seeds(task, task_dir, run_dir=None): """The reference answer: ``oracle.json`` (a list of findings) in a fence.""" with open(str(task_dir / task.get("oracle_answer", "oracle.json")), encoding="utf-8") as fh: # ``` can only sit in a JSON string: escape it as oracle_replay does, or it closes the fence @@ -276,7 +276,7 @@ def grade_replay(task, task_dir, answer, run_dir=None): "precision_proxy": _ratio(near, len(findings)), "decoys_flagged": []} -def oracle_replay(task, task_dir): +def oracle_replay(task, task_dir, run_dir=None): """The reference answer: one finding per gold thread, at its anchor.""" gold = _read_json(task_dir, task.get("gold", "gold.json")) ans = [{"file": g["file"], "line": g["line"], "severity": "minor", "rule": g.get("kind", ""), diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 5f13216e4..629445eec 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -26,6 +26,7 @@ # Bump when a result for the same inputs changes (a metric or ``pass``). VERSION = "1" TOOL = "mypy" +REFERENCE_DIR = "reference" # task dir: the files of a reference repair, copied over the workspace by oracle MAX_ERRORS = 50 # reported in details; the count is in the metrics _ERROR = re.compile(r"^(?P.+?)(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") @@ -49,8 +50,9 @@ def task_problems(raw: dict[str, Any]) -> list[str]: return problems -def _errors(stdout: str, listed: set[str], root: str) -> list[dict[str, Any]]: - """The error lines of mypy's output on a listed file, in output order; ``root`` is stripped from the names.""" +def _errors(stdout: str, listed: set[str] | None, root: str) -> list[dict[str, Any]]: + """The error lines of mypy's output on a listed file (any file when ``listed`` is None), in output + order; ``root`` is stripped from the names.""" out = [] for line in stdout.splitlines(): m = _ERROR.match(line) @@ -58,7 +60,7 @@ def _errors(stdout: str, listed: set[str], root: str) -> list[dict[str, Any]]: continue name = m.group("file") name = posixpath.normpath(name[len(root):] if name.startswith(root) else name) - if name in listed: + if listed is None or name in listed: out.append({"file": name, "line": int(m.group("line") or 0), "code": m.group("code") or "", "message": m.group("message")}) return out @@ -75,35 +77,46 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non if mypy.version.__version__ != task["version"]: raise ValueError(f"the image's mypy is {mypy.version.__version__}, the task pins {task['version']}") - workspace = Path(run_dir or "/eval/run") / "workspace" + workspace = (Path(run_dir or "/eval/run") / "workspace").resolve() # absolute: mypy gets these paths paths = [posixpath.normpath(p) for p in task["paths"]] missing = [p for p in paths if not (workspace / p).is_file()] present = [p for p in paths if p not in missing] errors: list[dict[str, Any]] = [] if present: cache = tempfile.mkdtemp(prefix="dokimi-mypy-") - neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cwd-") # no config here: the subject's mypy.ini is not read - cwd = os.getcwd() + neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cfg-") try: - os.chdir(neutral) - stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", + # An explicit neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are + # not read. mypy_path keeps the workspace root on the import path, so a namespace package resolves. + config = Path(neutral, "mypy.ini") + config.write_text(f"[mypy]\nmypy_path = {workspace}\n", encoding="utf-8") + stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", f"--config-file={config}", *[str(workspace / p) for p in present]]) finally: - os.chdir(cwd) shutil.rmtree(cache, ignore_errors=True) shutil.rmtree(neutral, ignore_errors=True) - if status not in (0, 1): # 2: mypy itself failed (a bad config, a crash) + # 2 is also what mypy gives for a blocking source error (a syntax error, a duplicate module): those + # are diagnostics like any other. A 2 with no diagnostic line is mypy itself failing. + if status not in (0, 1) and not any(": error: " in line for line in stdout.splitlines()): raise RuntimeError(f"mypy exited {status}: {stderr[-300:]}") - errors = _errors(stdout, set(present), str(workspace) + os.sep) + root = str(workspace) + os.sep + errors = _errors(stdout, set(present), root) + if status == 2 and not errors: + # mypy stopped at a blocking error in a file it did not reach ("errors prevented further + # checking"): the listed files were not checked, so that error is the grade's + errors = _errors(stdout, None, root) shown = [f"{e['file']}:{e['line']}: {e['message']}" + (f" [{e['code']}]" if e["code"] else "") for e in errors] return {"pass": not errors and not missing, "n_errors": len(errors), "n_missing": len(missing), "n_files": len(paths), "errors": shown[:MAX_ERRORS], "missing": missing, "tool": TOOL, "tool_version": mypy.version.__version__} -def oracle(task: dict[str, Any], task_dir: Path) -> str: - """No reference answer: the grade reads the workspace only.""" - del task, task_dir +def oracle(task: dict[str, Any], task_dir: Path, run_dir: Path | None = None) -> str: + """No reference answer: the grade reads the workspace only. A task with a ``reference/`` directory + has its files copied over the workspace first, so the reference repair is what gets graded.""" + reference = task_dir / REFERENCE_DIR + if reference.is_dir() and run_dir is not None: + shutil.copytree(reference, Path(run_dir) / "workspace", dirs_exist_ok=True) return "" diff --git a/dokimi/requirements-test.txt b/dokimi/requirements-test.txt index ae449e15d..eb3dbd99b 100644 --- a/dokimi/requirements-test.txt +++ b/dokimi/requirements-test.txt @@ -11,6 +11,8 @@ pytest-socket>=0.7 pytest-timeout>=2.2 pytest-randomly>=3.15 pytest-xdist>=3.6 +# The typecheck tests run mypy (tests/test_repo_typecheck_grader.py, test_repo_l0_typecheck.py) on this interpreter. +mypy>=1.8 diff-cover>=10.0 httpx>=0.27.0 numpy>=1.24.0 diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index dad09a503..51ab5dff7 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -151,3 +151,67 @@ def test_a_listed_path_with_a_colon_is_matched(): def test_a_line_less_error_on_a_listed_file_is_counted(): out = typecheck._errors("pkg/a.py: error: Duplicate module named 'a'", {"pkg/a.py"}, "") assert [(e["file"], e["line"]) for e in out] == [("pkg/a.py", 0)] + + +NAMESPACE_API = "from pkg.other import broken\n\ndef use() -> int:\n return broken()\n" +NAMESPACE_OTHER = "def broken() -> int:\n return 1\n" + + +def test_a_relative_run_directory_is_graded(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + result = typecheck.grade(_task(), tmp_path, "", Path("run")) + assert result["pass"] is True + assert result["missing"] == [] + + +def test_a_namespace_package_resolves_from_the_workspace_root(tmp_path): + run = _run(tmp_path, {"pkg/api.py": NAMESPACE_API, "pkg/other.py": NAMESPACE_OTHER}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True, result["errors"] + + +def test_a_blocking_syntax_error_in_a_listed_file_fails_the_grade(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "def broken(:\n pass\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] >= 1 + assert result["errors"][0].startswith("pkg/api.py:1:") + + +def test_mypy_failing_without_a_diagnostic_is_a_grader_error(tmp_path, monkeypatch): + run = _run(tmp_path, {"pkg/api.py": CLEAN}, _task()) + import mypy.api + + monkeypatch.setattr(mypy.api, "run", lambda args: ("", "cannot start", 2)) + with pytest.raises(RuntimeError, match="mypy exited 2"): + typecheck.grade(_task(), tmp_path, "", run) + + +def test_the_reference_directory_repairs_the_workspace_before_grading(tmp_path): + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + assert typecheck.grade(_task(), tmp_path, "", run)["pass"] is False + assert typecheck.oracle(_task(), task_dir, run) == "" + assert typecheck.grade(_task(), tmp_path, "", run)["pass"] is True + + +def test_the_entry_point_oracle_repairs_the_workspace_too(tmp_path): + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + main.oracle("typecheck", task_dir, run) + assert main.grade("typecheck", task_dir, run)["pass"] is True + + +def test_a_blocking_error_in_an_imported_module_fails_the_listed_file(tmp_path): + # mypy stops at the syntax error and never reaches api.py, so the listed file is not checked + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/other.py": "def broken(:\n pass\n", + "pkg/api.py": "from pkg.other import broken\n\ndef use() -> int:\n return 'x'\n"}, + _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] >= 1 From eb8668517e9db4ca086ae5c4de08fd965d5f3194 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:18:19 -0700 Subject: [PATCH 06/43] Address review: the L0 fence check's target, config and probe Fences are parsed with the grammar of the repository interpreter's version, and mypy runs with that --python-version, an explicit empty config (no subject or host mypy settings), --check-untyped-defs, and -I, so a module in the checkout cannot stand in for mypy in the version probe or the run. Thread: PRRT_kwDOVBg7g86yWj1X Thread: PRRT_kwDOVBg7g86yWj1Y Thread: PRRT_kwDOVBg7g86yWj1Z Thread: PRRT_kwDOVBg7g86yWj94 Thread: PRRT_kwDOVBg7g86yWj97 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/l0/typecheck.py | 35 +++++++--- dokimi/tests/test_repo_l0_typecheck.py | 92 ++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 12 deletions(-) diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index ba6108314..d4d64c5a8 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -58,10 +58,12 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: found = python_fences(scan, files) block: dict[str, Any] = {"tool": check.tool, "tool_version": None, "python": check.python, "fences": len(found), "checked": 0, "fail": 0, "unknown": 0, "syntax": 0} + # The repository's grammar and version are the target: a fence is parsed as that Python parses it. + target = python_version(check) if check.python is not None and found else None runnable = [] for f, fence, code in found: try: - ast.parse(code) + ast.parse(code, feature_version=target) except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: # `f()` is not Python block["syntax"] += 1 why = exc.msg if isinstance(exc, SyntaxError) else type(exc).__name__ # too deep: no parse tree @@ -76,7 +78,7 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: block["unknown"] = len(runnable) return block block["tool_version"] = tool_version(check) - found_errors = _run(check, [code for _, _, code in runnable]) if runnable else {} # no file: a usage error + found_errors = _run(check, [code for _, _, code in runnable], target) if runnable else {} # no file: usage error for i, (f, fence, code) in enumerate(runnable): errors = found_errors.get(i, []) if not errors: @@ -93,30 +95,47 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: def tool_version(check: Check) -> str: - """The version the tool reports, e.g. ``2.4.0``.""" - argv = [sys.executable, "-m", "mypy", "--version"] if check.tool == "mypy" else [check.tool, "--version"] - done = _run_tool(argv, os.getcwd()) + """The version the tool reports, e.g. ``2.4.0``. ``-I``: a module in the checkout cannot stand in for it.""" + argv = ([sys.executable, "-I", "-m", "mypy", "--version"] if check.tool == "mypy" + else [check.tool, "--version"]) + done = _run_tool(argv, tempfile.gettempdir()) if done.timed_out or done.returncode != 0: # a missing module exits 1 with the reason on stderr raise TypecheckError(f"{check.tool} --version exited {done.returncode}: {done.stderr[-300:]}") parts = done.stdout.split() return parts[1] if len(parts) > 1 else done.stdout.strip() +def python_version(check: Check) -> tuple[int, int]: + """The repository interpreter's (major, minor) version, read by the interpreter itself (``-I``).""" + done = _run_tool([str(check.python), "-I", "-c", "import sys; print(*sys.version_info[:2])"], + tempfile.gettempdir()) + if done.timed_out or done.returncode != 0: + raise TypecheckError(f"{check.python} did not report its version: {done.stderr[-300:]}") + major, minor = done.stdout.split()[:2] + return int(major), int(minor) + + def _label(code: str) -> str: """The fence's first non-blank line, for the claim's text.""" first = next((ln.strip() for ln in code.splitlines() if ln.strip()), "") return first[:80] -def _run(check: Check, codes: list[str]) -> dict[int, list[tuple[int, str]]]: - """Errors per fence index: (line in the fence, message), in tool order.""" +def _run(check: Check, codes: list[str], target: tuple[int, int] | None = None) -> dict[int, list[tuple[int, str]]]: + """Errors per fence index: (line in the fence, message), in tool order. ``target``: the repository's version.""" with tempfile.TemporaryDirectory(prefix="dokimi-typecheck-") as tmp: names = [] for i, code in enumerate(codes): Path(tmp, f"fence_{i}.py").write_text(code + "\n", encoding="utf-8") names.append(f"fence_{i}.py") if check.tool == "mypy": - argv = [sys.executable, "-m", "mypy", "--python-executable", str(check.python), + # The config is explicit and empty: no subject or host mypy settings apply. -I: nothing in the + # run directory or PYTHON* variables can stand in for mypy or for the standard library. + config = os.path.join(tmp, "mypy.ini") + Path(config).write_text("[mypy]\n", encoding="utf-8") + version = [] if target is None else ["--python-version", f"{target[0]}.{target[1]}"] + argv = [sys.executable, "-I", "-m", "mypy", "--python-executable", str(check.python), *version, + "--config-file", config, "--check-untyped-defs", "--follow-imports=silent", "--no-incremental", "--cache-dir", os.path.join(tmp, "cache"), "--output", "json", *names] else: diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 209f54013..9bc2ed879 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -147,9 +147,34 @@ def test_the_scan_command_refuses_a_python_env_that_is_not_executable(tmp_path, def test_bare_typecheck_flag_means_mypy(tmp_path): + from dokimi.commands.repo import _typecheck_or_exit + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck"]) - assert args.typecheck == "mypy" + assert args.typecheck is True assert args.python_env is None + assert _typecheck_or_exit(args).tool == "mypy" + + +def test_the_typecheck_flag_takes_no_value_so_a_source_after_it_is_not_its_checker(tmp_path): + # `repo scan --typecheck `: the source must not be read as the checker name + args = build_parser().parse_args(["repo", "scan", "--typecheck", str(tmp_path)]) + assert str(tmp_path) in vars(args).values() + assert args.typecheck is True + + +def test_the_checker_is_chosen_by_typecheck_tool(tmp_path): + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--typecheck-tool", "basedpyright"]) + assert args.typecheck_tool == "basedpyright" + + +def test_typecheck_tool_without_typecheck_is_refused(tmp_path, capsys): + from dokimi.commands.repo import _typecheck_or_exit + + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck-tool", "mypy"]) + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "needs --typecheck" in capsys.readouterr().err def test_the_payload_is_json_serialisable(tmp_path): @@ -184,7 +209,7 @@ def test_the_scan_text_report_says_unknown_without_an_interpreter(tmp_path, caps def test_a_tool_failure_is_a_usage_error_not_a_traceback(tmp_path, capsys, monkeypatch): from dokimi.repos.l0 import typecheck as l0_typecheck - def broken(check, codes): + def broken(check, codes, target=None): raise l0_typecheck.TypecheckError("mypy exited 2: broken") monkeypatch.setattr(l0_typecheck, "_run", broken) @@ -199,7 +224,7 @@ def test_basedpyright_missing_from_path_is_a_usage_error(tmp_path, capsys, monke empty = tmp_path / "empty-path" empty.mkdir() monkeypatch.setenv("PATH", str(empty)) - args = _scan_args(tmp_path, "--typecheck", "basedpyright") + args = _scan_args(tmp_path, "--typecheck", "--typecheck-tool", "basedpyright") with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 2 @@ -249,7 +274,7 @@ def test_a_relative_python_env_is_made_absolute_before_the_tool_runs(tmp_path, m def test_a_fence_too_deep_for_the_parser_is_unresolved_not_a_crash(tmp_path): - deep = "x = " + "+".join(["1"] * 20000) # ast.parse raises RecursionError on this + deep = "x = " + "+".join(["1"] * 200000) # ast.parse raises RecursionError on this (CPython 3.14 among them) agents = f"# Toy\n\n{FENCE}python\n{deep}\n{FENCE}\n" payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", None)) [claim] = _fence_claims(payload) @@ -263,3 +288,62 @@ def test_a_line_less_mypy_error_is_the_fence_line_not_the_line_above(tmp_path): report = json.dumps({"file": "fence_0.py", "line": -1, "severity": "error", "message": "no line"}) assert _errors("mypy", report) == {0: [(0, "no line")]} + + +def test_mypy_checks_untyped_function_bodies(tmp_path): + # Guidance examples are often unannotated functions; mypy skips their bodies unless told not to. + agents = f"{FENCE}python\nimport json\n\ndef example():\n json.dumps(1).nosuch()\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + [claim] = _fence_claims(payload) + assert claim["status"] == "fail" + assert "nosuch" in claim["why"] + + +def test_a_subject_or_host_mypy_config_cannot_suppress_a_fence(tmp_path, monkeypatch): + home = tmp_path / "home" + (home / ".config" / "mypy").mkdir(parents=True) + (home / ".config" / "mypy" / "config").write_text("[mypy]\nignore_errors = True\n", encoding="utf-8") + (home / ".mypy.ini").write_text("[mypy]\nignore_errors = True\n", encoding="utf-8") + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("XDG_CONFIG_HOME", str(home / ".config")) + tree = _tree(tmp_path, f"{FENCE}python\nimport json\nprint(json.dumps(1).nosuch())\n{FENCE}\n") + (tree.root / "mypy.ini").write_text("[mypy]\nignore_errors = True\n", encoding="utf-8") + [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + + +def test_the_version_probe_is_not_shadowed_by_a_module_in_the_checkout(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + import mypy.version + + checkout = tmp_path / "checkout" + checkout.mkdir() + (checkout / "mypy.py").write_text("print('mypy 9.9.9')\n", encoding="utf-8") + monkeypatch.chdir(checkout) + assert l0_typecheck.tool_version(Check("mypy", sys.executable)) == mypy.version.__version__ + + +def test_mypy_is_told_the_repository_interpreters_version(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd): + seen.append(argv) + return _completed(returncode=0, stdout="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("mypy", sys.executable), ["x.py"], (3, 11)) + argv = seen[0] + assert argv[argv.index("--python-version") + 1] == "3.11" + + +def test_a_fence_that_is_newer_syntax_than_the_repository_is_checked_against_its_grammar(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 11)) + tree = _tree(tmp_path, f"{FENCE}python\ntype Alias = int\n{FENCE}\n") + payload = scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + assert payload["typecheck"]["syntax"] == 1 # `type` is not a 3.11 statement, whatever the host runs From cb07efb77cc24742fe5088fb71f968861cb96474 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:18:19 -0700 Subject: [PATCH 07/43] Address review: --typecheck takes no value; --typecheck-tool picks the tool A bare --typecheck with nargs="?" read the source after it as the checker name, so `repo scan --typecheck .` exited 2. --typecheck is now a flag and --typecheck-tool {mypy,basedpyright} selects the tool (default mypy); either needs --typecheck. The CLI snapshot, the flag allowlist, the contract and the changelog fragment follow. Thread: PRRT_kwDOVBg7g86yWj1K Signed-off-by: Nikolay Petrov --- ...20261031_120000_napetrov_repo-typecheck.md | 2 +- dokimi/docs/reference/cli-interface.json | 22 ++++++++++++++++--- dokimi/docs/repo-evaluation-contract.md | 16 ++++++++++---- dokimi/dokimi/commands/repo.py | 21 ++++++++++++------ dokimi/tests/data/cli_flags_allowlist.txt | 3 ++- 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md index e7ef56bf0..307d5bbc4 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md @@ -2,7 +2,7 @@ - **`dokimi:typecheck` built-in grader and `repo scan --typecheck` (Python fences).** The grader type-checks the task's listed files with the image's pinned mypy and passes when none has an error (`tool`, `version`, `paths` - in `task.json`; a mismatched version is a grader failure). `repo scan --typecheck [mypy|basedpyright]` checks + in `task.json`; a mismatched version is a grader failure). `repo scan --typecheck` (with `--typecheck-tool mypy|basedpyright`) checks each Python fence of the guidance against `--python-env`'s site-packages: a fence with an error is a `fail` stale-API candidate, a fence that does not parse or has no interpreter is `unresolved`. The report gains a `typecheck` block, `null` without the flag. See `docs/repo-evaluation-contract.md` §4.5 and §8.2. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index b8c921e40..95e1335ef 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -11709,21 +11709,37 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--typecheck", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": [ "'basedpyright'", "'mypy'" ], - "const": "mypy", + "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, - "key": "--typecheck", + "key": "--typecheck-tool", "kind": "option", - "nargs": "?", + "nargs": null, "position": null, "required": false, "type": null diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e9cb6e56f..c4962fc39 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -630,7 +630,11 @@ failed the task, and on a task with `score` that is a failed run with score `dokimi:typecheck` checks the listed files only. Its pass is "no error on a listed file" (an error in a module a listed file imports belongs to that module, -not to the listed file), and a listed file that is missing fails. +not to the listed file), and a listed file that is missing fails. The exception +is a blocking error (a syntax error, a duplicate module) that stops mypy before +it reaches the listed files: then that error is the grade's, and the grade fails. +The task's `oracle` copies its `reference/` directory, when it has one, over the +workspace, so a reference repair is what gets graded; its answer is empty. It runs mypy in-process in the grading container, so the image's `python3` must import the pinned mypy: a `version` other than the installed one is a grader failure, not a fail. It is not "no new errors against the base": the @@ -1348,8 +1352,9 @@ finding is an error. ### 8.2 Python fences (`repo scan --typecheck`) -`--typecheck [mypy|basedpyright]` (bare: mypy) type-checks every fence tagged -`python`, `py` or `python3` in the guidance files. Each fence that parses is one +`--typecheck` type-checks every fence tagged +`python`, `py` or `python3` in the guidance files, with `--typecheck-tool mypy|basedpyright` +(default mypy). Each fence that parses is one `python_fence` claim, so the report and `--fail-on-fail` treat it like any other claim: @@ -1370,7 +1375,10 @@ host's pinned copy; basedpyright must be on `PATH` and is recorded with the version it reports, not pinned by dokimi. The report's `typecheck` block names the tool, its version, the interpreter (`null`: unknown) and the counts (`fences`, `checked`, `fail`, `unknown`, `syntax`); it is `null` without the -flag. `--python-env` needs `--typecheck`. The tools run on the host through +flag. `--python-env` and `--typecheck-tool` need `--typecheck`. The fences are parsed with the +grammar of the repository interpreter's version, and mypy runs with that +`--python-version` and an explicit empty config, so the subject's and the host's +mypy settings do not apply. The tools run on the host through `dokimi.process`; one run covers every fence of the repository. ```bash diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index b78c0ec88..c669b0351 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -128,19 +128,23 @@ def _typecheck_or_exit(args: argparse.Namespace): """The ``--typecheck`` check, or None; ``--python-env`` needs it and must name an interpreter.""" from dokimi.repos.l0 import Check - if args.typecheck is None: + if not args.typecheck: if args.python_env is not None: print("✗ --python-env needs --typecheck", file=sys.stderr) sys.exit(EXIT_USAGE) + if args.typecheck_tool is not None: + print("✗ --typecheck-tool needs --typecheck", file=sys.stderr) + sys.exit(EXIT_USAGE) return None + tool = args.typecheck_tool or "mypy" if args.python_env is not None and not os.access(args.python_env, os.X_OK): print(f"✗ --python-env {args.python_env}: not an executable interpreter", file=sys.stderr) sys.exit(EXIT_USAGE) - if args.typecheck != "mypy" and shutil.which(args.typecheck) is None: - print(f"✗ --typecheck {args.typecheck}: not on PATH", file=sys.stderr) + if tool != "mypy" and shutil.which(tool) is None: + print(f"✗ --typecheck {tool}: not on PATH", file=sys.stderr) sys.exit(EXIT_USAGE) python = os.path.abspath(args.python_env) if args.python_env is not None else None # the tool runs elsewhere - return Check(args.typecheck, python) + return Check(tool, python) def cmd_repo_scan(args: argparse.Namespace) -> None: @@ -512,10 +516,13 @@ def register(sub, positive_int) -> None: "the report, its claims replacing the framework's on the same file, line and text. " "A file that cannot be read or is not JSON exits 2; a schema-invalid document is " "reported and makes the command exit 1") - scan_p.add_argument("--typecheck", nargs="?", const="mypy", default=None, choices=("mypy", "basedpyright"), + scan_p.add_argument("--typecheck", action="store_true", default=False, help="Type-check each Python fence of the guidance (§8.2): one claim per fence, fail when the " - "tool reports an error. mypy is the host's pinned copy; basedpyright must be on PATH. Without " - "--python-env the fences are unknown (unresolved), not checked. Bare --typecheck means mypy") + "tool reports an error. The tool is mypy (the host's pinned copy) unless --typecheck-tool names " + "basedpyright, which must be on PATH. Without --python-env the fences are unknown (unresolved), " + "not checked") + scan_p.add_argument("--typecheck-tool", default=None, dest="typecheck_tool", choices=("mypy", "basedpyright"), + help="The tool --typecheck runs (default mypy). Needs --typecheck") scan_p.add_argument("--python-env", default=None, dest="python_env", metavar="PYTHON", help="The repository's interpreter (a venv with its pinned dependencies installed); the fences " "are checked against its site-packages. Needs --typecheck") diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index 2f2c03bb2..5d39b143e 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -206,7 +206,8 @@ --trials # catalog trigger-probe (#217) adds it: a banned synonym of reps; renaming it changes the CLI, so it waits for F1b --trigger-set --turn-reminder ---typecheck # repo scan's Python-fence check (mypy or basedpyright): no registry concept yet +--typecheck # repo scan's Python-fence check: no registry concept yet +--typecheck-tool # repo scan's checker for --typecheck (mypy or basedpyright): no registry concept yet --typed-max-len --typed-threads --validate From 8e2e01b2628dc28355148eacaf9c5c122b0c3c37 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:39:40 -0700 Subject: [PATCH 08/43] build(test): pin mypy to the CI lock version (2.4.0) Signed-off-by: Nikolay Petrov --- dokimi/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/requirements-test.txt b/dokimi/requirements-test.txt index eb3dbd99b..219b74f19 100644 --- a/dokimi/requirements-test.txt +++ b/dokimi/requirements-test.txt @@ -12,7 +12,7 @@ pytest-timeout>=2.2 pytest-randomly>=3.15 pytest-xdist>=3.6 # The typecheck tests run mypy (tests/test_repo_typecheck_grader.py, test_repo_l0_typecheck.py) on this interpreter. -mypy>=1.8 +mypy==2.4.0 diff-cover>=10.0 httpx>=0.27.0 numpy>=1.24.0 From 715f7b3ebb8cb7d7c6911026332ecd1ff83dbd0c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:53:24 -0700 Subject: [PATCH 09/43] build(lock): add mypy and its pins to requirements.lock.txt requirements-test.txt is an input of requirements.lock.txt, and the pytest job installs with --require-hashes from that lock only. The mypy pin added in the previous commit did not reach it, so the typecheck tests failed to collect ("No module named 'mypy'") on the py3.11 and py3.14 Tests jobs. The five blocks (mypy, mypy-extensions, pathspec, librt, ast-serialize) are copied from requirements-ci.lock.txt, which pins the same versions and hashes. The lock was merged, not recompiled with uv, so no other pin moves. Signed-off-by: Nikolay Petrov --- dokimi/requirements.lock.txt | 260 +++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) diff --git a/dokimi/requirements.lock.txt b/dokimi/requirements.lock.txt index 24aa39e0b..69d18bab0 100644 --- a/dokimi/requirements.lock.txt +++ b/dokimi/requirements.lock.txt @@ -183,6 +183,65 @@ anyio==4.15.1 \ # via # httpx # openai +ast-serialize==0.13.0 \ + --hash=sha256:017ddd4f22e727ef93e66df2d53340a6ff809b7e34cc2218f67918ae6239aad0 \ + --hash=sha256:12441bc7e41e495db5634c98adc8f8886b619ce2f1e68effe3f792a2adca9f47 \ + --hash=sha256:192aed400b2b92ebe41da17e856b9b6e17dbcebe0011ce4c6370d4a8a0486233 \ + --hash=sha256:19b1e8f4c088ce91053df310b444fceeddb39f728ca7d04272c983646e36314b \ + --hash=sha256:220a993dfc8b173e7062f690f9e00f4ebefe56718171bf35dccd73a9f8cea100 \ + --hash=sha256:359fcebc49f855bd189cf568235dc84eabf521e00d03fce43135cad6484906bc \ + --hash=sha256:3a9469e4b93d87e4ee8f5c7e9462f973032793a24b9ae37ffee860220f17586a \ + --hash=sha256:3bb8dd779c0a25478fe1db1a8b06dd4dd5e66077d6d0afe354acadb6d9aee7d0 \ + --hash=sha256:3e20e9ca3952196b91798f77ef267c36c7b3470021f950aa361d9be003fb655f \ + --hash=sha256:3f5d4a7fcc916026010bae2a154e5be2c05040e67ef5c494c66a5cf315c41e30 \ + --hash=sha256:446de6067d853f61b4bde8741762c83d06b57ea81f96dd47977714d9f31837fa \ + --hash=sha256:452fdaf5ff0b791870bb332254e083107d7abe29ef43251411c265c5b138f9a2 \ + --hash=sha256:4b2ee61692de03009e6a8f372f24fbc2d4b768a2f51ecd426bb84acdfd6da3d1 \ + --hash=sha256:4d1e15da4b6afc6fe80b87704be452aa0639df93d019a516e9ac9540357fc9b2 \ + --hash=sha256:4dc7a24c734aded0557ef90bfabd2278b37502aacde84f49b53b4cb6a0711ea9 \ + --hash=sha256:4e0bc018a457052d4638b469f90674e6ec0e32d86ac4a7bfa1f7d1c71a961426 \ + --hash=sha256:4f55668338bcb871e83ee21ba865fb08b7b4b13af9312742f9878c39f9d84ee3 \ + --hash=sha256:5e88733df5ffff9062b5ff2779cf402e0aa1a61ca3f7f84b577a1af2b7e09676 \ + --hash=sha256:619050b18705310e19e254cdb7554289fe14da374cbfbb1362cd84635896fb7f \ + --hash=sha256:684e191dd41b08b0b92692a181380a70cd76e3b6469606a40aae1ca6dab39e7d \ + --hash=sha256:6a406251363eeb5c7b85a405eddd123e627a531bd150dc673a7f5dd087743b5c \ + --hash=sha256:6a49f2a01a6df3150e022087cec0bf0ad580fec8f38a17f124d07dbb115106d1 \ + --hash=sha256:6bab08a6f287cd620578084f9974cfaf3bef71959105f62af85fa70298b24851 \ + --hash=sha256:6c95c04f1781cbefe89d512ce30e051d10823827ae542d9f18d5c2e7ba0fad08 \ + --hash=sha256:6cbfa6dae34d5686056ef7c40ce1d3e7e1de48555e3a2fed985aef2d1f869d9a \ + --hash=sha256:771cf5ee8329ee8472dd7a3d8bea7dbc480032ed8ddb4d37d40b57b95ee19ef2 \ + --hash=sha256:77efef815ecae1195ac9889f616bd518d53b2173e87157043253b864af1c81c5 \ + --hash=sha256:7ce1b50c5a68233e890926405afc308a5f10f6f49ec3a094d3dfa8b6733e4496 \ + --hash=sha256:7d7376c611055f5ee44e5e13a2620dbc6846f47c583952c1704c8805154c2d2f \ + --hash=sha256:807875ed8c5de739c8c55a45944336fcb9b8601d77fcee384a743cd0497211d6 \ + --hash=sha256:841262622499585f0610a927434db526578553d8dae270b90d4c419a385e46b7 \ + --hash=sha256:84cd9efdd3cd780b1f5361049becd91e0bcb0f16c2c216f41ee82e728a98390b \ + --hash=sha256:8aff1682f9fa3e119a1cf8ef47504d38f7019b22b79e0f2b5c2135b9532d14db \ + --hash=sha256:8e7c6fec7fe03cb8f40c4af81d742aa0cf690cf0b7bded47508a8a392093f414 \ + --hash=sha256:8f672c8e6d3b9ef6e365a5543aee2012247d1d58d948ceddb75b33a6679609ca \ + --hash=sha256:98edcd24240fa217d903c8f221bc05575baf38a87a207264ee7c800d21eef5a4 \ + --hash=sha256:9eaa20714acf43ef0a0c82850a2ec8097f834648f527c38f1483e2fd9a52cd3e \ + --hash=sha256:a0bdcef01e643e0810d2dedfb64d924bcfe079a15dc20d1c067870cc01d5c5e6 \ + --hash=sha256:a1714ee591a8e19833c0530a89e5a9faa7f62a11fc88f62fc5b722c7425dcc1f \ + --hash=sha256:a918572608ceb20fba8c2b83560f3d20be90effa4614589477b46d81672f0c3d \ + --hash=sha256:b004c3bdab0beb45194cd66c0b8feea40d676e461d26296f9c791c8e3e1b7061 \ + --hash=sha256:bd89da715b857a27c33fad713ea0561912c56f773ebd0fed2760cedd99724dc6 \ + --hash=sha256:be6b1a4ee49866c77eb8a50e9cbc845370e6c15230a126d0a71aeab35f31c78c \ + --hash=sha256:c51c855d8b7d5403925599acd3c6fc91b321eba3bf46dcc9fe88fd2d6619dac7 \ + --hash=sha256:c77e5b62dfbfdfc1b025105f5038114ae988a0e616d48a845e0e57173f3f37c8 \ + --hash=sha256:cdd8fd066858b57ea2761b3d3989c90ea23913825bbb5453cc684c28bba3fb19 \ + --hash=sha256:d47c8f0eedf0a41681c10a7c497fef3691c4a6b4af4de6c93a4916bb29712554 \ + --hash=sha256:dffcffa543c8fcfb1ca941038eeae23e7f97ad8994e4d6f81fbd658cfa8cb440 \ + --hash=sha256:e241f68cf5060bff9b161b202b60d6d52161ff3777fe56eb6a9a6764fdb7fdd9 \ + --hash=sha256:e611937e6e77448496489627ae2558b6f6143449b1fb33f8a495665212eee58a \ + --hash=sha256:ee58f0db40f121ff0820242b286702700bc0ec58a53b6ac43ce4f43714d42e0d \ + --hash=sha256:efaab6400de8ee2d0e38695feccb0758acf11c1d0f8bc58a7090c566dd3ae88e \ + --hash=sha256:f0cc1f94fcd3b67005a32ee3e4c6b27cdc41659f697840d00fbb1e815ec27044 \ + --hash=sha256:f30f0e59be30c0c9540e8908b14874bc8d3c1d52a4562a4cfb426b003bd6c28b \ + --hash=sha256:f8da1a31e941adbea886a85fb25efc6f09353d58c665fdbc523a914e3d2e49fe \ + --hash=sha256:fadba24386498ed745c848b0d45e4939a506694bd2b474c57576e9640f3defe2 \ + --hash=sha256:fe2a3c8480e8e5eb41eaa958279b8c530b77b45065423f4ebd5223e118293055 + # via mypy attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 @@ -1263,6 +1322,145 @@ jsonschema-specifications==2025.9.1 \ --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d # via jsonschema +librt==0.16.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:001bfd59a7d45b17e3e75f2a8c6405280b35e7b84471792778e718c4f368950e \ + --hash=sha256:0058f9d68721094105917254c72ac0569117bb7b13b9769cf45d26d89f9d21cd \ + --hash=sha256:02118f56a9c36ddd07dfd9b919d9ecc117ba20a90987d56aa4c429fa34509188 \ + --hash=sha256:0253721561787b8df8443eb347b7a6461015354e5bdd37ee38a41fef220d2bb0 \ + --hash=sha256:02d89c813d5ff74b17df72d3a34819d132cd168e56b81bf755b809bd9e46b8c4 \ + --hash=sha256:0314058469f4d2fd279ce7c62ac274ac82c3918ef7db62ef0697c4c359370155 \ + --hash=sha256:0dbe4096a7ecc00fa835d24510ad8545a4efef738dac96e0e63516783ccde905 \ + --hash=sha256:0ead24d2562a49473dddd9efef8581f020007eb0054389c3ee3ffad38b1ca4c9 \ + --hash=sha256:13b4e8aba90b0b1c82474e9844aa9ffe7ad3faa484350e1da64cb8188d903134 \ + --hash=sha256:14ed6ebe3e4f85f326d7920011ad30ff49ed9334e62cf88caef9ba973d9e3a92 \ + --hash=sha256:17bac7f7a16b328fff77e440287693eb017abde913595b5827ebccbc21ecd8a6 \ + --hash=sha256:1b384b90ab79a7bc30b566895809a636e0666f21f3cf12b54823d025b7e83839 \ + --hash=sha256:1bc17e54e5305f8d40b7ca203671ff5a9e59c1d0f8ea0f625dcca53a3984de11 \ + --hash=sha256:1d28ae980ae2218f9c5b95d191e947296f918c9bf0b400d467a9430275bbe678 \ + --hash=sha256:1e511762a074005bb0aa569166779834e75e438370226930d0ce1866d4b6a33b \ + --hash=sha256:20fe0bf9053885e21c62b3e091fb5e73e1c54d1daf2e70eb388d70763bcd4220 \ + --hash=sha256:242e00b3d4fa37c3d3c1ca5f5c9adb7d909ddb1eac9c41f2787320d00caa0af2 \ + --hash=sha256:25a58a19ea8d83b68209f04912df765e9260635ef77646542ed4b4abe6bc7940 \ + --hash=sha256:273d00be33792a15189331df10f1f1331621b043881e66c6c4377f7f776e1291 \ + --hash=sha256:28e038895b998d7a0c7798922ce8a1dc157675df5cf1c9ef0aca809ed804b7a1 \ + --hash=sha256:2bec3818c7da7c96ceae0ef5915a3d16c52dd08f3ea913bf1fe8568c447c7978 \ + --hash=sha256:2c4aa329c17bd1aaea4f6e89335d8ccd494b3a5830b6654462273e50e11023f0 \ + --hash=sha256:300c3ffdc459f4a779a8411ecb188e3ac0b1ff3a3a7b099642555dedae06c69b \ + --hash=sha256:30b7beaf3f4487b7d8adef1f158b49067cb4d5a19fa7a3bf31a4e7a820e435c5 \ + --hash=sha256:314e703f0c19320dc8094e7a784b9cf29e1b67402515abf580069a6363c0b4f1 \ + --hash=sha256:33f41443a1f4e1f099331b3d8120e409fbff84b9760bc1cc9ea496f37ddaa5cc \ + --hash=sha256:349c0bcb87ebd07481b6ff781e25cdc699723dbe2212e57dabb27f7a13b7b87d \ + --hash=sha256:36e53948e99bbe3ffea257124cfcae1cfb01831555c9a9c903c9f9a72db7fd07 \ + --hash=sha256:375bfe6b572a8f6cfc398709356046173bf27e64c4c5edaf5f7062f051fb4bf9 \ + --hash=sha256:378dfaffb38e59c24a87cde5713cd865d51ff7383fa12947f3907f306ea1ca55 \ + --hash=sha256:3931f7a3db322e7f44e02a280e3949326ce9579ad388ee8d691dc7c76da9fb70 \ + --hash=sha256:39ca4f2f2fe05de8e63493da592d84311adabe5bef52b193851981da9816b302 \ + --hash=sha256:39ec1d5a14e37baf1450a6cabf03fe552340808bf1ad9d71824ab90117716459 \ + --hash=sha256:3ddeb3c9dedb461bb457c6c7d9aa7fbf35329da313d1a7543d00c8d0f3473c96 \ + --hash=sha256:3e0c39bdc85370422e8b637be76eb1fd07d30967551b03e62267dd156f553152 \ + --hash=sha256:3e483a8d69ede8067db70c0e83007423b6925de6fd53afed01d66160f2e9398c \ + --hash=sha256:3f0b8114c44b2ac06ff5dacd08e07e8e807ff4f46083f2a1602685122559be41 \ + --hash=sha256:3ff4b2367926b69c6215635902cccb04048e73094e9862900d27cb2c6bbff143 \ + --hash=sha256:4323193ac0cd025f85af531df8ba91bf24d1973b401697347a6282e8fd3fcf5e \ + --hash=sha256:468df902df016a06eb0e40b0747dc8d14e47d7a38b18b63b1fb167d85cb94d63 \ + --hash=sha256:473eebc7866bb0a0c8849a292b5e7157c1aba5d14d0f0f610c52158d6d964262 \ + --hash=sha256:47ada6ea32636492c61aa8ad27ae3b9404bfe7a97e3ba946d1984236cc741da0 \ + --hash=sha256:4b6183e2e2e0ee00aac2ec07c7f7d151c97e85666b304f574b71cff0f9fccc4e \ + --hash=sha256:4e29522c62e28595ff7e324c6834ade51127707f0e255b18d1c1cf03d39c1048 \ + --hash=sha256:4eb1313a19847089ee81e88742abedf285c60538816640b99742d8534b81d26a \ + --hash=sha256:52327da75a94012e7f932f913d20d3876bed3c102be00e6c3e8600ff7bdd58a7 \ + --hash=sha256:54d11f726aae9df5a6ffbbf0a03a52449bbac84a53ef03669cb41cdfd4ae41bf \ + --hash=sha256:5696d7f52e7b37217cb3a8f92c744fe835942602fdd4c1a8bc4741d3bfdce15e \ + --hash=sha256:5750a105b42a416f930edc59054927a406effb2550cd5bab92ad7a5842ed5d05 \ + --hash=sha256:5810ba811297fdf37a1531a57667cb8ace0842013ca8606bf9eb7c24cf4be154 \ + --hash=sha256:5981c011b306781ce561e18e14230a14524a3d8109b97553666c942c18f31a96 \ + --hash=sha256:5a269c46ae327d8e6f8c1f85f7516cb52c0fa48127565a1105a4f4a05ff2a0b4 \ + --hash=sha256:5b976054553670829985ed767feb78fb6bcede0175327c4844dd5c281c1be659 \ + --hash=sha256:5bcc2c4726ced915b00de0c9856a4eeabfb3fddb93e10e0b8f735b7709358b6d \ + --hash=sha256:5cd5b092441053364af968ea12084692cb9d4a22f3ce9524e377880bf028761e \ + --hash=sha256:5f49cff01bd608ef7d97104cb035c75455e79c2d70bf4a506cf773338ac1860d \ + --hash=sha256:6072e92dd876ff6ceeb6cf371e35e51f479349837391341f479b08df4564242b \ + --hash=sha256:64c79520414a3fdfc6aabd7593e6169afa14d5f8d9908d4b498db068868b08dd \ + --hash=sha256:67e718c7a43f8db325abbbf1404e2d535f12f8f7a1a82259568385cc5274b82a \ + --hash=sha256:69ba927445cfaaffb4081003ef5224c55a5c2ab67ef956f416ef744916e44121 \ + --hash=sha256:6a63610fa76524edfa605b5b259a603915c7a6e10e54f003e5503030506e81de \ + --hash=sha256:6c5da27e8056439f927ea896735da60e616c477a8293feaa3233d4e7781a6726 \ + --hash=sha256:6c8893eae2fd13c5488d94056f3e6e5cf3142bfb1c4acaf136cb33d760c5964b \ + --hash=sha256:6d4a64283ee61824b5790de882bc68e2d9d7a5143537cb7a966f7354f71646d4 \ + --hash=sha256:6fe436af2eaf630474f491af5d032cbe45f93fcff5c3b9fe4ab194a7255b20ff \ + --hash=sha256:71b93b42784e25b975079573c642a8fedb049a7bb31d70a51721b1666b3b2ced \ + --hash=sha256:7393c9a48dcce4817dbd4b0d8ff6237efe9b0a0609f5b0adaef315f8541726b5 \ + --hash=sha256:77c7a2b4fe2c1369e0d5aa1cade26740a7b14be32fbc9a5535d617d20065c39d \ + --hash=sha256:7a1d272724b581bb6bc769dfdafed6da2ecc9886ba2450311de55a4ac2e1e9cd \ + --hash=sha256:7cc365f006891afb006b52d5ee5ee74c09306ffa20e2f8705a32a4450af2f3ba \ + --hash=sha256:7e510b7770bee609617a3374a96548eb114cae048023e3f049ee449e7ff2db32 \ + --hash=sha256:80039ba9b6a7d5f1a0175a4cca6bbefead87bd854c80abad1cb30afe47a830db \ + --hash=sha256:83d4041a3d9b2fd053a8a4e1f22878b3e5833e2712956382d5c048d791454e91 \ + --hash=sha256:845a511b60ca43b9880dcc84a9784c891d6a2098c829130b320846c69c9c0c68 \ + --hash=sha256:877698bf6bca5721d8be345f2fe09778e40ecadea8b58c73075f2b1a53666bf2 \ + --hash=sha256:8caf96a4ef8fb27d0ac0d1ad8337d26a240acd4a02fe4345d0a8f264753e8f99 \ + --hash=sha256:8ceafb70f2a4f0826f11031942e59c0728fd98da112dc346d4352bde1e486866 \ + --hash=sha256:8f36c58e33b304b525c6c9c5076399c6ebf1109e17b9051a05a407b091b9215b \ + --hash=sha256:8ff5d26c529336be9bd7ae04483235d77778ee7d6444a95353102b542601ce81 \ + --hash=sha256:909d8e3c1faee44cb762b1c519ff8613dcc5ceae5c99987a00917b5a31fd1d6a \ + --hash=sha256:92caf82ebef5e12d21c72242b70d1e92536f1711cf2a727a4c276de4b4469087 \ + --hash=sha256:931a0bb0fcac88f263e269e46eb30ba8e21402cd3c62ca40cb97034c0693fab1 \ + --hash=sha256:943c6bbecbdf7fa575a4f2952fcfd848c88ef95507c3fca411e89d4ac3ff8143 \ + --hash=sha256:94aed6a8308818b91677957d1bd03188869cd7aeb23c5dba7912a6c0402f7602 \ + --hash=sha256:94be5cb7bca4df6201f4183e9e4fa2086c655283d20b38cd84500a69057575a7 \ + --hash=sha256:953107e2f68d0f3512c48f898b0dbf0ce5cc52bba0f318d847c985dc555ee4cc \ + --hash=sha256:96f576f2711f8519152ec76d0e599243555c1f07679fa73606ca8c8c868c0be6 \ + --hash=sha256:a33e0dae1f8592146a4764d54ce842b278732d21a84e17c3bbe6b1bc158a2248 \ + --hash=sha256:a4aaefb4ba6c07e1aeebb2795c8958148f1d6f9af3b555b53d23d766edb6d67a \ + --hash=sha256:a8afb6557920860b7a3a596eb804cf37e09e7cf8a803db2478c202acc72d8c2e \ + --hash=sha256:aa9357a1b4d4fc787bb718a59cb1112c28c8a976d6bfa268b71cc0a4ab8f3a94 \ + --hash=sha256:ac38d6d8d66bf3d744148dbbc0b8e193e195a51e364ed55e224631f5721891fc \ + --hash=sha256:ad37d5b9abd49c9a655dcda7ea52a8a752884062ef1ee71ae17c2f2a0f81fe6a \ + --hash=sha256:aea7b1f2b125dad5de85f049136651bff256c883c65e6b9209b2da0a1ac3cdef \ + --hash=sha256:afced3dfc17cd805ecf7a3d77996a71cf5f2c75aa66eb0c21a9930f4fc992f86 \ + --hash=sha256:b0e3e721c75d2e79a76d4422c79d7ba705fe1bbafec907037fe7a657a480a0e3 \ + --hash=sha256:b6d085d70bce51d43c5c7c36d63490770180d8779e71c49305c87b4213918de7 \ + --hash=sha256:b95d5d92ab83d39e760a52091bb1baba664f3a2351e39b1e16801e5747c2f0e9 \ + --hash=sha256:b9d6d4b14e92d876f8026b54c20c445f36425214c1081dc76f74e40db386b82b \ + --hash=sha256:bc02954b1295de798bbdb0b4e2d8a28c2117de8b5c73dcbeb27dc32572dfb971 \ + --hash=sha256:bd3150023d3dc2bc70f3784e59ffa1140d56ddba3d8125b3d6f9f85221279bfc \ + --hash=sha256:be56ba9c884143495b517f23fe794ae367d58cd89ea0fdd6d437e3c024a87f9f \ + --hash=sha256:c17194318e4c0c0348b36f36c2ec7534436fe0a4c15582403162a4f08c80797a \ + --hash=sha256:c3d1bb7841a816ace6449bb26d3f9560dbfa20e71c568d23f0f62bf1e68f50b1 \ + --hash=sha256:c43bd6e642d8a248c114327f98dd25ac5a7cb5aa168ef02f0559b91874df16b8 \ + --hash=sha256:c5db585d43449a5f54303d4b2774e45e1babd975cfe1630a3d708c0b80c3e560 \ + --hash=sha256:c5e6144e68b577f157519f2ba88ca20e3ed61c29b00e5cdfa76cd2d45acf059a \ + --hash=sha256:c6f1b27bf1632a7e016af9f145f82be95e1edd7721a646505c21059257cb5a04 \ + --hash=sha256:c71d1b76210a36729fedfc5115069b50a3d8619054745f8758fe5d6f19e86671 \ + --hash=sha256:c72c5295a84bd249526da9bdca38f2e176d15c31c13bb0063c5053f4ca023421 \ + --hash=sha256:ca8052401c55d7511dda6760719fda7618067e83535d7d0010096d216c34b667 \ + --hash=sha256:d1aabe3925cbb4a08d15b7b20ba4011b53019da0c4173a25155139b7b1baed65 \ + --hash=sha256:d3c94211ee0c4f8d649ec06b7c115c0ec4eadb873a0e3154ca15cef3f814b071 \ + --hash=sha256:d46ca272b251d033dd4527b0dec5f261a28a52bd5fa0f99c117b0a1f8588cc2d \ + --hash=sha256:d608f0bf3b8cbddd0067fe02cb8cab7d13e8eb9386b23a1843d4363044fd9e22 \ + --hash=sha256:d6a365f2ab45a984d0e00eee0dd17f599ceab8cadab6ea07b6111c8132fc0e42 \ + --hash=sha256:d92db7a0f6aee44f1baee94750457e8d2d1c6ccea41842de6268d34e8dc7eddd \ + --hash=sha256:df183721229ae51eef90108c115b43e98cb169b6155d34480338e5fc6616df00 \ + --hash=sha256:e05108e0849966f53a8d2d3112a7af881d0efaa479bc735bba91108f9f2350a7 \ + --hash=sha256:e1967e36ac4cae0c7e9615ad32e1a513cdacff79f9e8afb28bedc91caf48b4f3 \ + --hash=sha256:e42f8e098b9c5396fefa05fb1cc7e33b0e08fc51da106b5de4a45fd22aac6743 \ + --hash=sha256:e56aaf8c167548dc8e5d6f3bd0f48dcdd299a23c73be3f744aab79d99e9c7f5d \ + --hash=sha256:e9ce0bc440e7fd09b5f51f372f0f6640658f854b8fb05260f819cc669c93c42d \ + --hash=sha256:ebefd60b42e2a82b32d136bb5f7c94eadfcd29f772b547df6f3291d1ed855a1c \ + --hash=sha256:ef46c1a29ffb8c72e882e22618ec618778eacd0578fb22c6e7cf9c11d15f357b \ + --hash=sha256:efc49c462d4516b8a58b00b490078fa64689fd1fe66970cc190131d7afb8027e \ + --hash=sha256:f01f3805f2dae4781c0c34b440e31740d082950bdaf89a6f601ad589a28af57a \ + --hash=sha256:f06c689cb14afd9b612727553a5ec5a40febf113ca41c4413a2b0b334285884b \ + --hash=sha256:f1e8591bd8a5a628cd7f07954c6a1592359a878bf032957a8e9057a41d644311 \ + --hash=sha256:f4462528b6000afe8f16907b5c7c2553abf1df005ba5140e6eb394541c3624c3 \ + --hash=sha256:f7be7cf555bc30ec12622e9447299cc4a9b8ff307548b634794353db0c2065dc \ + --hash=sha256:f81b5b19ce748ef68d4746656b7929762eb2fe99269b266e4be07e2ee4de7144 \ + --hash=sha256:f9807485a908f00355820f18e91e045ffdcdc5adb68aaec40a1e2b88c5f7bba1 \ + --hash=sha256:fbe4fb8c5445f7496d7f7f6bb0807875d09d47e6771ffa175fb2df2895fb86ba \ + --hash=sha256:fe4372c52d4849096c6cc1cda2817d293ec51440c890474ed59ef38d46556f18 \ + --hash=sha256:fe52bf4641069e7978a14253b036cb9002def1926317e710f2e249f8a8c47742 \ + --hash=sha256:ff7baa55f8e7c69851419e50a666015d02a74198716fd45c0125a2112e0a389f + # via mypy litellm==1.106.0 \ --hash=sha256:5c84cc7d928cdb6ec57c5d87f221e3d27ca755958ebae389d5663f822b6f7a19 \ --hash=sha256:791625a05eb1481d14633d593c19dfaa4b91ad69ec9c3e0886a72855cd76c1d1 \ @@ -1629,6 +1827,64 @@ multiprocess==0.70.19 \ --hash=sha256:e6c0674d34b8adac22533f6786576b3de4e396aaeda9e0c15378af9b8ada2702 \ --hash=sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f # via datasets +mypy==2.4.0 \ + --hash=sha256:058165f564ccf559c68c70fec2091fca5891110480210c22594635e3f6683437 \ + --hash=sha256:0bb95cf34899e4619c61ab0a8667804e139e580b30d5df12af2102dfe44d0c97 \ + --hash=sha256:13fa24f439c0e48a290a3922fa14ccd22f2762bae99d2142931b3e40a9055080 \ + --hash=sha256:172e30b8fea631fe310f0c665477f52d9ea40bb4e99e0c81dc30118563b13710 \ + --hash=sha256:1dc0f64b0a92ae27a49d2175f0bacfa56e15bdc5b420cf92e0c1f79292219cbb \ + --hash=sha256:20e9a5cd875837520c43db98dea0b6d0c2197833d95c30127d8f570fb9b1f00b \ + --hash=sha256:2106b55105ba5ea9be4f53a24517fc5fa927ff1585edc9bc1a975abb72caef89 \ + --hash=sha256:236e0d68f6941992b0811128e652590f590db444ab29ad8f1324765b9298b946 \ + --hash=sha256:29243242cf72582b65f9582ad9e56e8cb281566ed3519f4cd70bb8b9f2977e90 \ + --hash=sha256:295ecf2e57542cd836ca537486951289678c8c7d1ee6ad74ebe29b2168a003cf \ + --hash=sha256:29eb0b9427a6b11b992e452f6cceb8af724f4dceb47e779d0b35e405e996ea5e \ + --hash=sha256:3011537be6cf1de4511c0255a324362a812b58184bbe61e15f59c8b31033bd74 \ + --hash=sha256:3adef556a19eb3b630bf86a79c29d7da3d61e541472d0d897ca01b171c0abf8f \ + --hash=sha256:3bd0e340f0ebe65c548210f53be3fd8192e83964760caf0c28bef368e68b0d37 \ + --hash=sha256:4209da39d85cf240f762af622d8180fcdfcb4727d021f44ade62d613a1a43324 \ + --hash=sha256:4a378fc15fb33e321f04652c166ce73eeb8833a97c3d218132844e938cd93220 \ + --hash=sha256:502b94b0b331f7dafe32fd6b151797ddbb4f32385b362e722c783a025e5954a3 \ + --hash=sha256:528c8744b8b5e3ecb8774f86af38d2376216816e9908317ad055f3c9c2d74799 \ + --hash=sha256:5786ef987b3767e51aaa53f20aec104c0252b42ecda7aef8e8b4cbae279b05c5 \ + --hash=sha256:5d20e6c7c35fcbf2a0ebdd0eaeacfbc243009dfd33ab7822d54e213912e6dbbd \ + --hash=sha256:615b03922d40e186fd1df73156473db0ba525c639bb4fd1cf28e1694879c9b23 \ + --hash=sha256:6306086b87cf7f8a29aa618d9fd9bffb56c59247166b9660fdb54d86d7714ecd \ + --hash=sha256:6be721bd4bd57576193653b75b4af3461c9d0bf7dd8b528f782e9be210dc75bb \ + --hash=sha256:720434d48542ecfe84d32d287b727569d3fc8f5769acd39051130e490a5c295c \ + --hash=sha256:77bdaebd452f43fcfc4cc3ba94352a3ea537cd01e3f2d0879f48673d2ec00d6e \ + --hash=sha256:7c4f8f8d1d1c0e2832d8ee7113dd08f6df6c7aad9e863fcbed9f25832be0b8c4 \ + --hash=sha256:7da85fbcff6dac1abcc636707bed38b45598131fb7a605d9719c70b5cc733af8 \ + --hash=sha256:7f38f57d344f8b6accb40e01c3d83cfc590498231724d16c07ffb7940f157818 \ + --hash=sha256:82d0f94c8587ccb472622ee7795280aaa38a06640d5f45b3f16909d6dd86a989 \ + --hash=sha256:86d616fe84c6eab8026f8c50ab5bcb90db780d2ccd233d971e34e92bede9b359 \ + --hash=sha256:9279488933040b638c0ab739084c0ca100efeea6db581bf5d7628d8e89de53fe \ + --hash=sha256:970b221ed5842213d98e3c480c08f795ace4b1f81fb21e1b126bd0476bce1c34 \ + --hash=sha256:9f03a7828cca2b0adcd6662aee8f2711ff8830e1027641fdea3ab0b787483566 \ + --hash=sha256:9f459f0b4f0596d9d51fe7716b404b35287b99e77da98a7af90a65dd5fd61141 \ + --hash=sha256:9fa247e02b505a45a2775f69df38d360d197e3790bc60f717595db9eda358b6e \ + --hash=sha256:a3f86fd1313dd69d013e265f1fdcd12ea7a9d606f9875b2a3db946cd334555f3 \ + --hash=sha256:a6e851b82c0661f69f1630fc16172c68787a6a9cf0991e7c6437d60976cdcd76 \ + --hash=sha256:a96b07a49b7b1d025ce59c1b3acbcf24bead9a83da4523c4a6bde1bb94e7a0e1 \ + --hash=sha256:afa89837d9be67e0cadfa33bca3bb7efdda98c3b07e74dc3b635ebfb1c8a926a \ + --hash=sha256:ba05652540bf12828e52abae807b024b09ca144ff4f75e2450a81d69c376425b \ + --hash=sha256:bc378bdad4e9f12b5bd96466083d1e71acf00594ec9c7b2bdb5e02816f77f303 \ + --hash=sha256:c9de622fd397495695d0598ddc789222bfcfec9d7c9ec3a1e385c855e3bc5e01 \ + --hash=sha256:cb734b2668c1f40d07ce093bbeb4407e9527c67901627b0e1679825be3f09975 \ + --hash=sha256:d01c5d26a352acc6d5cf3128225477e1e8465e8d3029d4c345807fbf7f3cf093 \ + --hash=sha256:e05ff2925d8b37ad26c80c1b9dc43ae5d455da2df1e23c24c095a6425917c57e \ + --hash=sha256:e1fde197ae65be856a034a91b70ed747a16562ca69577785f06c661548424bf1 \ + --hash=sha256:e3ebe2f72a2a1156065a9851570ffbf50c0a93cdccadef9c6e05c508a4fd10b1 \ + --hash=sha256:e76172710bd4e5eeae061abfd68347e5264632e02778be61784671ae3a2132f5 \ + --hash=sha256:f83353e47ab520bf6fd4df8f5897d9fe081211f2fbc4b7d37736a3e3c166cbcf \ + --hash=sha256:f9b028548b3af480e2b1ed8df14ccaac86f99c9f600d1580770ab7ba3dcd40f0 \ + --hash=sha256:fb443e81057896132d3642d6be219e6efd158691ac7883e3ba8fcb469865f05d \ + --hash=sha256:ffda5244fd1ad71a1e54405e35f50d09b80c3978efa120f58bd1252ae32c62d2 + # via -r requirements-test.txt +mypy-extensions==1.1.0 \ + --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ + --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 + # via mypy numpy==2.4.6 ; python_full_version < '3.12' \ --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ @@ -1855,6 +2111,10 @@ pandas==3.0.6 \ --hash=sha256:fb625f426b375bcc96e3a04c5d5d266cd7be6ae5d6866e0e703382ab5164068c \ --hash=sha256:ff51a4459ed036e93d1eb1bb5e6e7b28685d3cb6b7c12b91c05b31024e234729 # via datasets +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + # via mypy pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 From 90e50373eeed8769b425d81a5fe9e1c44dc81d8b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:55:31 -0700 Subject: [PATCH 10/43] fix(typecheck): tool runs get an allow-list environment; silent --python-env is a usage error A tool run inherited the whole host environment, so a host MYPYPATH added modules to mypy's search path and a fence could pass on a module the pinned environment lacks. The run now gets PATH, locale and temp variables only. A --python-env that prints no version (e.g. /bin/true) raised an uncaught ValueError, and repo scan printed a traceback. It is now the usage error the other --python-env failures already are. Both are covered by a red-then-green test in test_repo_l0_typecheck.py. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/l0/typecheck.py | 14 +++++++++++--- dokimi/tests/test_repo_l0_typecheck.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index d4d64c5a8..294f6fdfc 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -111,8 +111,11 @@ def python_version(check: Check) -> tuple[int, int]: tempfile.gettempdir()) if done.timed_out or done.returncode != 0: raise TypecheckError(f"{check.python} did not report its version: {done.stderr[-300:]}") - major, minor = done.stdout.split()[:2] - return int(major), int(minor) + try: + major, minor = done.stdout.split()[:2] + return int(major), int(minor) + except ValueError: # no output, or output that is not two integers: the path is not a Python + raise TypecheckError(f"{check.python} did not report its version: {done.stdout[-300:]!r}") from None def _label(code: str) -> str: @@ -150,9 +153,14 @@ def _run(check: Check, codes: list[str], target: tuple[int, int] | None = None) return _errors(check.tool, done.stdout) +# What a tool run may inherit from the host. Not the whole environment: MYPYPATH, for one, adds modules to +# mypy's search path, so a fence could pass on a module the pinned environment does not have. +_TOOL_ENV = ("PATH", "LANG", "LC_ALL", "LC_CTYPE", "TMPDIR", "TEMP", "TMP", "SYSTEMROOT") + + def _run_tool(argv: list[str], cwd: str): try: - return process.run(argv, timeout=TIMEOUT, env=process.Env(allow=None), cwd=cwd) + return process.run(argv, timeout=TIMEOUT, env=process.Env.only(_TOOL_ENV), cwd=cwd) except OSError as exc: raise TypecheckError(f"cannot run {argv[0]}: {exc}") from exc diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 9bc2ed879..ed881bff2 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -313,6 +313,27 @@ def test_a_subject_or_host_mypy_config_cannot_suppress_a_fence(tmp_path, monkeyp assert claim["status"] == "fail" +def test_a_host_mypypath_cannot_supply_a_module_the_pinned_environment_lacks(tmp_path, monkeypatch): + extra = tmp_path / "extra" + extra.mkdir() + (extra / "ghost_pkg_mod.py").write_text("def run() -> int:\n return 1\n", encoding="utf-8") + monkeypatch.setenv("MYPYPATH", str(extra)) + tree = _tree(tmp_path, f"{FENCE}python\nimport ghost_pkg_mod\nprint(ghost_pkg_mod.run())\n{FENCE}\n") + [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + + +def test_a_python_env_that_reports_no_version_is_a_usage_error_not_a_traceback(tmp_path, capsys): + silent = tmp_path / "silent-python" + silent.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + silent.chmod(0o755) + args = _scan_args(tmp_path, "--typecheck", "--python-env", str(silent)) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "did not report its version" in capsys.readouterr().err + + def test_the_version_probe_is_not_shadowed_by_a_module_in_the_checkout(tmp_path, monkeypatch): from dokimi.repos.l0 import typecheck as l0_typecheck From 9dfe9b213673f1bc8bab2008ac56e03632721b40 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 00:19:16 -0700 Subject: [PATCH 11/43] fix(typecheck): name listed modules as imports do; keep a stopped run unresolved - grader: --explicit-package-bases, with the source root of each regular package on mypy_path, so a listed namespace module and a src/ layout both resolve as they are imported (pkg.other, pkg.x) - L0: fence syntax is parsed by the repository interpreter, with type_comments and no feature_version approximation - L0: mypy exit 2 is a result only when it names a fence; fences the run did not reach are unresolved, not ok - basedpyright runs with an empty --project, so no config above the run applies Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 26 ++++++- dokimi/dokimi/repos/l0/typecheck.py | 83 ++++++++++++++++---- dokimi/tests/test_repo_l0_typecheck.py | 88 ++++++++++++++++++++-- dokimi/tests/test_repo_typecheck_grader.py | 16 +++- 4 files changed, 187 insertions(+), 26 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 629445eec..10b03b4d6 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -66,6 +66,23 @@ def _errors(stdout: str, listed: set[str] | None, root: str) -> list[dict[str, A return out +def _package_roots(workspace: Path, paths: list[str]) -> list[Path]: + """For each listed file inside regular packages (``__init__.py``), the directory above its top package. + + A namespace directory (no ``__init__.py``) is not a package root: its files are named from the workspace, as + an import names them, and nothing above them is added.""" + roots: list[Path] = [] + for p in paths: + top = None + d = (workspace / p).parent + while d != workspace and (d / "__init__.py").is_file(): + top = d + d = d.parent + if top is not None and top.parent != workspace and top.parent not in roots: + roots.append(top.parent) + return roots + + def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | None = None) -> dict[str, Any]: """Type-check the listed files in the workspace of ``run_dir``; the grader's result dict.""" del task_dir, answer # the answer is not read: the workspace is the subject @@ -87,11 +104,14 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cfg-") try: # An explicit neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are - # not read. mypy_path keeps the workspace root on the import path, so a namespace package resolves. + # not read. Explicit package bases name a listed file the way an import of it does: relative to the + # root it is imported from (pkg.other, not other). So the roots are on mypy_path: the workspace, and + # the source root of each regular package (src/ for src/pkg/__init__.py). + roots = [*_package_roots(workspace, present), workspace] config = Path(neutral, "mypy.ini") - config.write_text(f"[mypy]\nmypy_path = {workspace}\n", encoding="utf-8") + config.write_text(f"[mypy]\nmypy_path = {':'.join(str(r) for r in roots)}\n", encoding="utf-8") stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", f"--config-file={config}", - *[str(workspace / p) for p in present]]) + "--explicit-package-bases", *[str(workspace / p) for p in present]]) finally: shutil.rmtree(cache, ignore_errors=True) shutil.rmtree(neutral, ignore_errors=True) diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index 294f6fdfc..e6558f124 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -11,7 +11,6 @@ from __future__ import annotations -import ast import json import os import sys @@ -58,15 +57,14 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: found = python_fences(scan, files) block: dict[str, Any] = {"tool": check.tool, "tool_version": None, "python": check.python, "fences": len(found), "checked": 0, "fail": 0, "unknown": 0, "syntax": 0} - # The repository's grammar and version are the target: a fence is parsed as that Python parses it. + # The repository's grammar and version are the target: a fence is parsed by that interpreter, and checked as + # that Python. No feature_version: it approximates an older grammar, the interpreter has the real one. target = python_version(check) if check.python is not None and found else None + syntax = _syntax_errors(check, [code for _, _, code in found]) runnable = [] - for f, fence, code in found: - try: - ast.parse(code, feature_version=target) - except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: # `f()` is not Python + for (f, fence, code), why in zip(found, syntax, strict=True): + if why is not None: # `f()` is not Python, or too deep for the parser block["syntax"] += 1 - why = exc.msg if isinstance(exc, SyntaxError) else type(exc).__name__ # too deep: no parse tree scan.add(f, fence.line, "python_fence", _label(code), "unresolved", f"not valid Python ({why}): not type-checked") continue @@ -78,9 +76,16 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: block["unknown"] = len(runnable) return block block["tool_version"] = tool_version(check) - found_errors = _run(check, [code for _, _, code in runnable], target) if runnable else {} # no file: usage error + # no file: usage error. ``blocked``: the tool stopped early, so a fence with no error was not necessarily checked. + found_errors, blocked = _run(check, [code for _, _, code in runnable], target) if runnable else ({}, False) + unchecked = 0 for i, (f, fence, code) in enumerate(runnable): errors = found_errors.get(i, []) + if not errors and blocked: + unchecked += 1 + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + f"{check.tool} stopped before this fence was checked: not type-checked") + continue if not errors: scan.add(f, fence.line, "python_fence", _label(code), "ok", f"{check.tool}: no error") continue @@ -90,7 +95,8 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: where = f"line {line}: " if line else "" # 0: the tool gave no line, so the claim is the fence itself scan.add(f, fence.line + line, "python_fence", _label(code), "fail", f"{check.tool}: {where}{message}{more}"[:MAX_WHY]) - block["checked"] = len(runnable) + block["checked"] = len(runnable) - unchecked + block["unknown"] += unchecked # "unknown" is every fence that was not checked return block @@ -124,8 +130,42 @@ def _label(code: str) -> str: return first[:80] -def _run(check: Check, codes: list[str], target: tuple[int, int] | None = None) -> dict[int, list[tuple[int, str]]]: - """Errors per fence index: (line in the fence, message), in tool order. ``target``: the repository's version.""" +_PARSE = """\ +import ast, json, sys +out = [] +for code in json.load(sys.stdin): + try: + ast.parse(code, type_comments=True) + out.append(None) + except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: + out.append((exc.msg or "invalid syntax") if isinstance(exc, SyntaxError) else type(exc).__name__) +json.dump(out, sys.stdout) +""" + + +def _syntax_errors(check: Check, codes: list[str]) -> list[str | None]: + """Per fence: None when it parses, else why not. The repository's interpreter parses it (its own grammar); + with none given, dokimi's own, ``-I`` and from the temporary directory, so the checkout cannot shadow it.""" + if not codes: + return [] + python = check.python or sys.executable + with tempfile.TemporaryDirectory(prefix="dokimi-typecheck-") as tmp: + done = _run_tool([python, "-I", "-c", _PARSE], tmp, stdin=json.dumps(codes)) + if done.timed_out or done.returncode != 0 or done.truncated: + raise TypecheckError(f"{python} did not parse the fences: {done.stderr[-300:]}") + try: + reasons = json.loads(done.stdout) + except ValueError: + reasons = None + if not isinstance(reasons, list) or len(reasons) != len(codes): + raise TypecheckError(f"{python} did not parse the fences: {done.stdout[-300:]!r}") + return reasons + + +def _run(check: Check, codes: list[str], + target: tuple[int, int] | None = None) -> tuple[dict[int, list[tuple[int, str]]], bool]: + """Errors per fence index: (line in the fence, message), in tool order; and whether the tool stopped early. + ``target``: the repository's version.""" with tempfile.TemporaryDirectory(prefix="dokimi-typecheck-") as tmp: names = [] for i, code in enumerate(codes): @@ -142,15 +182,26 @@ def _run(check: Check, codes: list[str], target: tuple[int, int] | None = None) "--follow-imports=silent", "--no-incremental", "--cache-dir", os.path.join(tmp, "cache"), "--output", "json", *names] else: - argv = [check.tool, "--pythonpath", str(check.python), "--outputjson", *names] + # The project is explicit and empty: a pyrightconfig.json above the temporary directory is not read. + project = os.path.join(tmp, "pyrightconfig.json") + Path(project).write_text("{}\n", encoding="utf-8") + argv = [check.tool, "--project", project, "--pythonpath", str(check.python), "--outputjson", *names] done = _run_tool(argv, tmp) - if done.timed_out or done.returncode not in (0, 1): # 1: the tool found errors; anything else is its failure + # 1: the tool found errors. mypy's 2: it stopped on a blocking one ("errors prevented further checking"), + # which is a result only when it names a fence. Any other exit is the tool's own failure (pyright's 2 is fatal). + blocked = check.tool == "mypy" and done.returncode == 2 + if done.timed_out or (done.returncode not in (0, 1) and not blocked): raise TypecheckError(f"{check.tool} exited {done.returncode}: {done.stderr[-300:]}") if done.truncated: # the middle of the output was dropped, so an error there would be missed raise TypecheckError(f"{check.tool} output was truncated; not every error is known") if done.returncode == 1 and not done.stdout.strip(): # exit 1 with no report: the tool did not run raise TypecheckError(f"{check.tool} exited 1 with no report: {done.stderr[-300:]}") - return _errors(check.tool, done.stdout) + errors = _errors(check.tool, done.stdout) + if blocked: + if not errors: # a usage error, or a blocking one that names no fence + raise TypecheckError(f"{check.tool} exited 2 with no fence report: {done.stderr[-300:]}") + return errors, True + return errors, False # What a tool run may inherit from the host. Not the whole environment: MYPYPATH, for one, adds modules to @@ -158,9 +209,9 @@ def _run(check: Check, codes: list[str], target: tuple[int, int] | None = None) _TOOL_ENV = ("PATH", "LANG", "LC_ALL", "LC_CTYPE", "TMPDIR", "TEMP", "TMP", "SYSTEMROOT") -def _run_tool(argv: list[str], cwd: str): +def _run_tool(argv: list[str], cwd: str, stdin: str | None = None): try: - return process.run(argv, timeout=TIMEOUT, env=process.Env.only(_TOOL_ENV), cwd=cwd) + return process.run(argv, timeout=TIMEOUT, env=process.Env.only(_TOOL_ENV), cwd=cwd, input=stdin) except OSError as exc: raise TypecheckError(f"cannot run {argv[0]}: {exc}") from exc diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index ed881bff2..c5a78f0c5 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -361,10 +361,88 @@ def fake(argv, cwd): assert argv[argv.index("--python-version") + 1] == "3.11" -def test_a_fence_that_is_newer_syntax_than_the_repository_is_checked_against_its_grammar(tmp_path, monkeypatch): - from dokimi.repos.l0 import typecheck as l0_typecheck - - monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 11)) +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="the fence needs a repository interpreter older than 3.12") +def test_a_fence_newer_than_the_repository_grammar_is_a_syntax_error(tmp_path): + # `type` is a 3.12 statement: the repository's interpreter (here, 3.11) is what parses the fence tree = _tree(tmp_path, f"{FENCE}python\ntype Alias = int\n{FENCE}\n") payload = scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) - assert payload["typecheck"]["syntax"] == 1 # `type` is not a 3.11 statement, whatever the host runs + assert payload["typecheck"]["syntax"] == 1 + + +def _fake_tools(monkeypatch, mypy_result, seen: list): + """Stand-in for the tool runs: the version probes answer, the parse passes every fence, mypy gives ``mypy_result``.""" + from dokimi.repos.l0 import typecheck as l0_typecheck + + def fake(argv, cwd, stdin=None): + seen.append(argv) + if "ast" in argv[-1]: # the fence parse, run by the repository's interpreter + return _completed(returncode=0, stdout=json.dumps([None] * len(json.loads(stdin)))) + if "version_info" in argv[-1]: + return _completed(returncode=0, stdout="3 11\n") + if "--version" in argv: + return _completed(returncode=0, stdout="mypy 2.4.0 (compiled: no)\n") + return mypy_result + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + + +def test_a_tool_that_stops_on_a_blocking_error_leaves_the_unchecked_fences_unresolved(tmp_path, monkeypatch): + report = json.dumps({"file": "fence_0.py", "line": 1, "severity": "error", "message": "blocking"}) + _fake_tools(monkeypatch, _completed(returncode=2, stdout=report + "\n"), []) + agents = f"{FENCE}python\nx: int = 1\n{FENCE}\n\n{FENCE}python\ny: int = 2\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["fail", "unresolved"] + assert "blocking" in claims[0]["why"] + assert "stopped before this fence was checked" in claims[1]["why"] # no error reported is not a pass + assert payload["typecheck"]["checked"] == 1 + assert payload["typecheck"]["unknown"] == 1 + + +def test_a_tool_that_exits_two_with_no_fence_report_is_an_error_not_a_pass(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=2, stderr="usage: mypy [-h]")) + with pytest.raises(l0_typecheck.TypecheckError, match="exited 2 with no fence report"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_basedpyright_is_given_an_empty_project_so_no_config_above_the_run_applies(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = {} + + def fake(argv, cwd, stdin=None): + seen["argv"] = argv + seen["config"] = Path(argv[argv.index("--project") + 1]).read_text(encoding="utf-8") # while it exists + return _completed(returncode=0, stdout="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("basedpyright", sys.executable), ["x: int = 1\n"]) + assert seen["argv"][0] == "basedpyright" + assert seen["config"] == "{}\n" + + +def test_a_fence_is_parsed_by_the_repository_interpreter_not_the_host(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + return _completed(returncode=0, stdout=json.dumps([None])) + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + assert l0_typecheck._syntax_errors(Check("mypy", "/opt/repo/python"), ["x = 1\n"]) == [None] + assert seen[0][:3] == ["/opt/repo/python", "-I", "-c"] + + +@pytest.mark.skipif(not Path("/bin/true").exists(), reason="needs /bin/true") +def test_a_true_python_env_is_a_usage_error_not_a_traceback(tmp_path, capsys): + # /bin/true exits 0 with no output: it reports no version, so the scan refuses it cleanly + args = _scan_args(tmp_path, "--typecheck", "--python-env", "/bin/true") + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "did not report its version" in capsys.readouterr().err diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 51ab5dff7..05dec7cfb 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -166,8 +166,20 @@ def test_a_relative_run_directory_is_graded(tmp_path, monkeypatch): def test_a_namespace_package_resolves_from_the_workspace_root(tmp_path): - run = _run(tmp_path, {"pkg/api.py": NAMESPACE_API, "pkg/other.py": NAMESPACE_OTHER}, _task()) - result = typecheck.grade(_task(), tmp_path, "", run) + # both files listed: a listed other.py must be named pkg.other, the name api.py imports it by + task = _task(paths=["pkg/api.py", "pkg/other.py"]) + run = _run(tmp_path, {"pkg/api.py": NAMESPACE_API, "pkg/other.py": NAMESPACE_OTHER}, task) + result = typecheck.grade(task, tmp_path, "", run) + assert result["pass"] is True, result["errors"] + + +def test_a_src_layout_package_resolves_when_both_files_are_listed(tmp_path): + # a regular package under src/: pkg.x is imported from src/, so src/ is a root with the workspace + files = {"src/pkg/__init__.py": "", "src/pkg/x.py": "val: int = 1\n", + "src/pkg/y.py": "from pkg.x import val\n\nprint(val + 1)\n"} + task = _task(paths=["src/pkg/x.py", "src/pkg/y.py"]) + run = _run(tmp_path, files, task) + result = typecheck.grade(task, tmp_path, "", run) assert result["pass"] is True, result["errors"] From 6a9c800fb57bbdb490d01e8ce7c3d492aea45397 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 00:21:31 -0700 Subject: [PATCH 12/43] test(typecheck): a blocking type comment is a failing fence mypy 2.4.0 reports the # type: ??? comment as a syntax error (exit 1), so the fence is a fail; the test pins that the parse passes it through to mypy. Not a regression test: it passes before and after the grader and L0 changes. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_l0_typecheck.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index c5a78f0c5..56f185f91 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -446,3 +446,11 @@ def test_a_true_python_env_is_a_usage_error_not_a_traceback(tmp_path, capsys): args.func(args) assert exc.value.code == 2 assert "did not report its version" in capsys.readouterr().err + + +def test_a_blocking_type_comment_is_a_failing_fence_not_a_tool_failure(tmp_path): + # ast.parse reads no type comments by default, so this passes the parse; mypy then exits 2 on it + tree = _tree(tmp_path, f"{FENCE}python\nx = 1 # type: ???\n{FENCE}\n") + [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + assert "type comment" in claim["why"] From fd9160c69377464e7670719da9a70c299db2f328 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 00:53:01 -0700 Subject: [PATCH 13/43] docs(typecheck): shorten the mypy config comment to the two-line cap The comment block was four lines; check_comment_blocks fails a block of three or more lines that the PR touches. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 10b03b4d6..b2f684dde 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -103,10 +103,8 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non cache = tempfile.mkdtemp(prefix="dokimi-mypy-") neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cfg-") try: - # An explicit neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are - # not read. Explicit package bases name a listed file the way an import of it does: relative to the - # root it is imported from (pkg.other, not other). So the roots are on mypy_path: the workspace, and - # the source root of each regular package (src/ for src/pkg/__init__.py). + # Neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are not read. + # mypy_path is the workspace and each regular package's source root (src/ for src/pkg/__init__.py). roots = [*_package_roots(workspace, present), workspace] config = Path(neutral, "mypy.ini") config.write_text(f"[mypy]\nmypy_path = {':'.join(str(r) for r in roots)}\n", encoding="utf-8") From 5d8b05d3ad798e9b8f29deb19690292c433ee6fb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 01:17:04 -0700 Subject: [PATCH 14/43] fix(repos/graders): restore the recursion limit after in-process mypy mypy's main raises sys.setrecursionlimit for the whole process, so a later RecursionError-based check (journeys' too-deep registry) passed on py3.11 CI. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 3 +++ dokimi/tests/test_repo_typecheck_grader.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index b2f684dde..f0527b76c 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -19,6 +19,7 @@ import posixpath import re import shutil +import sys import tempfile from pathlib import Path from typing import Any @@ -102,6 +103,7 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non if present: cache = tempfile.mkdtemp(prefix="dokimi-mypy-") neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cfg-") + limit = sys.getrecursionlimit() # mypy's main raises it for the whole process try: # Neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are not read. # mypy_path is the workspace and each regular package's source root (src/ for src/pkg/__init__.py). @@ -111,6 +113,7 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", f"--config-file={config}", "--explicit-package-bases", *[str(workspace / p) for p in present]]) finally: + sys.setrecursionlimit(limit) shutil.rmtree(cache, ignore_errors=True) shutil.rmtree(neutral, ignore_errors=True) # 2 is also what mypy gives for a blocking source error (a syntax error, a duplicate module): those diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 05dec7cfb..6600f5e51 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import sys from pathlib import Path import mypy.version @@ -45,6 +46,14 @@ def test_listed_file_that_type_checks_passes(tmp_path): assert result["tool_version"] == PINNED +def test_grading_leaves_the_recursion_limit_as_it_was(tmp_path): + # mypy's main raises the process-wide recursion limit; in-process, that leaked into later code. + before = sys.getrecursionlimit() + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + typecheck.grade(_task(), tmp_path, "", run) + assert sys.getrecursionlimit() == before + + def test_listed_file_with_a_type_error_fails_and_names_the_error(tmp_path): run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) result = typecheck.grade(_task(), tmp_path, "", run) From 8122aa2eed0157413135b97084e44e065bf6b0a5 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 12:59:07 +0000 Subject: [PATCH 15/43] docs(typecheck): the L0 check runs the tool --typecheck-tool selects Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/graders/typecheck.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c4962fc39..82707b1f7 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -639,7 +639,7 @@ It runs mypy in-process in the grading container, so the image's `python3` must import the pinned mypy: a `version` other than the installed one is a grader failure, not a fail. It is not "no new errors against the base": the grader is stdlib-only and cannot run `git` to read the base, and it does not -start basedpyright (the L0 check, §8.2, runs both tools on the host). A repository that +start basedpyright (the L0 check, §8.2, runs mypy or basedpyright on the host, as `--typecheck-tool` selects). A repository that needs that comparison writes it in its own `grade` entry point. `manifest.json` is `{"seeds": [{id, class, file, lines, kw}], "decoys": diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index f0527b76c..2fee719b4 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -9,7 +9,7 @@ It runs mypy in-process: the grader cannot start a child (the import layers allow spawns only in ``dokimi.process``, which the container cannot import), so basedpyright is not offered here. The -host-side L0 check (``repo scan --typecheck``) runs both tools. Standard library only at import +host-side L0 check (``repo scan --typecheck``) runs one, as ``--typecheck-tool`` selects. Standard library only at import time; mypy is imported by :func:`grade` and must be importable by the image's ``python3``. """ From b6a8dacd73a1d28e345c4ff07b8a9c3f12dcbee4 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 12:59:42 +0000 Subject: [PATCH 16/43] test(typecheck): a linked guidance file's fences are counted once Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_l0_typecheck.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 56f185f91..2ce234294 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -454,3 +454,14 @@ def test_a_blocking_type_comment_is_a_failing_fence_not_a_tool_failure(tmp_path) [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) assert claim["status"] == "fail" assert "type comment" in claim["why"] + + +def test_a_linked_guidance_file_has_its_fences_checked_and_counted_once(tmp_path): + root = tmp_path / "r" + root.mkdir() + (root / "AGENTS.md").write_text(f"{FENCE}python\nimport json\n\njson.nosuch()\n{FENCE}\n", encoding="utf-8") + (root / "CLAUDE.md").symlink_to("AGENTS.md") + manifest = parse_manifest(dict(MANIFEST, guidance={"globs": ["AGENTS.md", "CLAUDE.md"]})) + payload = scan_repo(WorkTree(root), manifest, typecheck=Check("mypy", sys.executable)) + assert len(_fence_claims(payload)) == 1 + assert payload["typecheck"]["fences"] == 1 From 9a253f0535c384676832839d49a836b3f82774da Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:30:41 -0800 Subject: [PATCH 17/43] fix(typecheck): the grader keeps a reference copy out of symlinks and MYPYPATH The oracle copied the reference over the workspace with copytree(dirs_exist_ok=True), which writes through an existing destination symlink, so a checkout could make it overwrite a file outside the workspace. It now refuses a destination reached through a link (the workspace root, a directory on the way, or the file) before copying anything. mypy reads MYPYPATH ahead of the neutral mypy_path, so an externally supplied stub directory could satisfy an import the image lacks. MYPYPATH is out of the run during a grade and restored afterwards. Thread: PRRT_kwDOVBg7g86yYW3V Thread: PRRT_kwDOVBg7g86yaepL Thread: PRRT_kwDOVBg7g86yYfCX Thread: PRRT_kwDOVBg7g86yaepR Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 30 +++++++++++++- dokimi/tests/test_repo_typecheck_grader.py | 47 ++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 2fee719b4..941cce25f 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -104,6 +104,9 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non cache = tempfile.mkdtemp(prefix="dokimi-mypy-") neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cfg-") limit = sys.getrecursionlimit() # mypy's main raises it for the whole process + # mypy reads MYPYPATH ahead of mypy_path, so an externally supplied stub directory could stand in for a + # module the image lacks: it is out of the run, and restored afterwards. + mypypath = os.environ.pop("MYPYPATH", None) try: # Neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are not read. # mypy_path is the workspace and each regular package's source root (src/ for src/pkg/__init__.py). @@ -114,6 +117,8 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non "--explicit-package-bases", *[str(workspace / p) for p in present]]) finally: sys.setrecursionlimit(limit) + if mypypath is not None: + os.environ["MYPYPATH"] = mypypath shutil.rmtree(cache, ignore_errors=True) shutil.rmtree(neutral, ignore_errors=True) # 2 is also what mypy gives for a blocking source error (a syntax error, a duplicate module): those @@ -132,12 +137,33 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non "tool": TOOL, "tool_version": mypy.version.__version__} +def _linked_on_the_way(workspace: Path, dest: Path) -> bool: + """True when the workspace root, a directory on the way to ``dest``, or ``dest`` itself is a symbolic link: + a copy would then write to the link's target, which can lie outside the workspace.""" + if workspace.is_symlink(): + return True + current = workspace + for part in dest.relative_to(workspace).parts: + current = current / part + if current.is_symlink(): + return True + return False + + def oracle(task: dict[str, Any], task_dir: Path, run_dir: Path | None = None) -> str: """No reference answer: the grade reads the workspace only. A task with a ``reference/`` directory - has its files copied over the workspace first, so the reference repair is what gets graded.""" + has its files copied over the workspace first, so the reference repair is what gets graded. A destination + reached through a symbolic link in the workspace is refused before anything is copied.""" reference = task_dir / REFERENCE_DIR if reference.is_dir() and run_dir is not None: - shutil.copytree(reference, Path(run_dir) / "workspace", dirs_exist_ok=True) + workspace = Path(run_dir) / "workspace" + for source in reference.rglob("*"): + if source.is_file(): + dest = workspace / source.relative_to(reference) + if _linked_on_the_way(workspace, dest): + raise ValueError(f"{dest.relative_to(workspace).as_posix()} would be written through a " + "symbolic link in the workspace") + shutil.copytree(reference, workspace, dirs_exist_ok=True) return "" diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 6600f5e51..221060769 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import os import sys from pathlib import Path @@ -236,3 +237,49 @@ def test_a_blocking_error_in_an_imported_module_fails_the_listed_file(tmp_path): result = typecheck.grade(_task(), tmp_path, "", run) assert result["pass"] is False assert result["n_errors"] >= 1 + + +def test_an_external_mypypath_cannot_supply_a_module_the_image_lacks(tmp_path, monkeypatch): + stubs = tmp_path / "stubs" # outside the workspace, as an image's or the environment's MYPYPATH would be + stubs.mkdir() + (stubs / "unavailable.pyi").write_text("def value() -> int: ...\n", encoding="utf-8") + monkeypatch.setenv("MYPYPATH", str(stubs)) + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "from unavailable import value\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert any("unavailable" in error for error in result["errors"]) + assert os.environ["MYPYPATH"] == str(stubs) # restored for the caller + + +@pytest.mark.parametrize("linked", ["file", "directory", "workspace"]) +def test_the_oracle_refuses_to_write_through_a_symbolic_link(tmp_path, linked): + outside = tmp_path / "outside" + (outside / "pkg").mkdir(parents=True) + sentinel = outside / "pkg" / "api.py" + sentinel.write_text("ORIGINAL", encoding="utf-8") + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text("REFERENCE", encoding="utf-8") + run = tmp_path / "run" + run.mkdir() + workspace = run / "workspace" + if linked == "file": + (workspace / "pkg").mkdir(parents=True) + (workspace / "pkg" / "api.py").symlink_to(sentinel) + elif linked == "directory": + workspace.mkdir() + (workspace / "pkg").symlink_to(outside / "pkg", target_is_directory=True) + else: + workspace.symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert sentinel.read_text(encoding="utf-8") == "ORIGINAL" + + +def test_the_oracle_copies_the_reference_when_no_link_is_on_the_way(tmp_path): + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + typecheck.oracle(_task(), task_dir, run) + assert (run / "workspace" / "pkg" / "api.py").read_text(encoding="utf-8") == CLEAN From 2745c477086949ec33fb6c518a5fe44275dd1eed Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:30:46 -0800 Subject: [PATCH 18/43] fix(typecheck): a fence cannot import another fence; no tool run without a fence The fences were written as fence_.py in one directory that mypy searches, so 'import fence_0' resolved to another fence and the verdict depended on the other snippets. The files are now fence-.py: a hyphen cannot be imported, so every fence is checked against the repository's environment alone. A repository with no runnable fence (none, or all unparsable) no longer runs the tool's --version probe; the report's tool_version is then null. Pinned, with tests: the --python-executable argv, the exit codes that are the tool's own failure (basedpyright 2, any other), and that only error diagnostics count as fence errors. Thread: PRRT_kwDOVBg7g86yaepV Thread: PRRT_kwDOVBg7g86yaepX Thread: PRRT_kwDOVBg7g86yaepc Thread: PRRT_kwDOVBg7g86yal_R Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/l0/typecheck.py | 13 ++-- dokimi/tests/test_repo_l0_typecheck.py | 97 ++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 10 deletions(-) diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index e6558f124..9e64f71d4 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -75,7 +75,8 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: "unknown: no repository environment (--python-env), not type-checked") block["unknown"] = len(runnable) return block - block["tool_version"] = tool_version(check) + # Nothing to run, no tool run: a version probe would fail on a repository without the tool, for no result. + block["tool_version"] = tool_version(check) if runnable else None # no file: usage error. ``blocked``: the tool stopped early, so a fence with no error was not necessarily checked. found_errors, blocked = _run(check, [code for _, _, code in runnable], target) if runnable else ({}, False) unchecked = 0 @@ -169,8 +170,10 @@ def _run(check: Check, codes: list[str], with tempfile.TemporaryDirectory(prefix="dokimi-typecheck-") as tmp: names = [] for i, code in enumerate(codes): - Path(tmp, f"fence_{i}.py").write_text(code + "\n", encoding="utf-8") - names.append(f"fence_{i}.py") + # A hyphen makes the name unimportable: no ``import`` in one fence can resolve to another fence's file, + # so every fence is checked against the repository's environment alone. + Path(tmp, f"fence-{i}.py").write_text(code + "\n", encoding="utf-8") + names.append(f"fence-{i}.py") if check.tool == "mypy": # The config is explicit and empty: no subject or host mypy settings apply. -I: nothing in the # run directory or PYTHON* variables can stand in for mypy or for the standard library. @@ -242,8 +245,8 @@ def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: def _fence_index(name: str) -> int | None: - """``3`` for ``fence_3.py``; None for any other file (a note about an import, say).""" + """``3`` for ``fence-3.py``; None for any other file (a note about an import, say).""" stem = Path(name).name - if stem.startswith("fence_") and stem.endswith(".py") and stem[6:-3].isdigit(): + if stem.startswith("fence-") and stem.endswith(".py") and stem[6:-3].isdigit(): return int(stem[6:-3]) return None diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 2ce234294..26ae2e526 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -119,9 +119,10 @@ def test_basedpyright_fails_the_same_stale_fence(tmp_path): def test_fence_file_names_map_back_to_their_index_only(): - assert _fence_index("fence_3.py") == 3 - assert _fence_index("elsewhere/x/fence_12.py") == 12 - assert _fence_index("fence_x.py") is None + assert _fence_index("fence-3.py") == 3 + assert _fence_index("elsewhere/x/fence-12.py") == 12 + assert _fence_index("fence-x.py") is None + assert _fence_index("fence_3.py") is None # the name an import could reach: not a fence file assert _fence_index("pkg/other.py") is None @@ -286,7 +287,7 @@ def test_a_fence_too_deep_for_the_parser_is_unresolved_not_a_crash(tmp_path): def test_a_line_less_mypy_error_is_the_fence_line_not_the_line_above(tmp_path): from dokimi.repos.l0.typecheck import _errors - report = json.dumps({"file": "fence_0.py", "line": -1, "severity": "error", "message": "no line"}) + report = json.dumps({"file": "fence-0.py", "line": -1, "severity": "error", "message": "no line"}) assert _errors("mypy", report) == {0: [(0, "no line")]} @@ -387,7 +388,7 @@ def fake(argv, cwd, stdin=None): def test_a_tool_that_stops_on_a_blocking_error_leaves_the_unchecked_fences_unresolved(tmp_path, monkeypatch): - report = json.dumps({"file": "fence_0.py", "line": 1, "severity": "error", "message": "blocking"}) + report = json.dumps({"file": "fence-0.py", "line": 1, "severity": "error", "message": "blocking"}) _fake_tools(monkeypatch, _completed(returncode=2, stdout=report + "\n"), []) agents = f"{FENCE}python\nx: int = 1\n{FENCE}\n\n{FENCE}python\ny: int = 2\n{FENCE}\n" payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) @@ -465,3 +466,89 @@ def test_a_linked_guidance_file_has_its_fences_checked_and_counted_once(tmp_path payload = scan_repo(WorkTree(root), manifest, typecheck=Check("mypy", sys.executable)) assert len(_fence_claims(payload)) == 1 assert payload["typecheck"]["fences"] == 1 + + +def test_a_fence_cannot_import_another_fence_as_a_module(tmp_path): + agents = f"{FENCE}python\nvalue: int = 1\n{FENCE}\n\n{FENCE}python\nimport fence_0\nprint(fence_0.value)\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + assert [c["status"] for c in _fence_claims(payload)] == ["ok", "fail"] + + +def test_mypy_is_given_the_repository_interpreter(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + return _completed(returncode=0, stdout="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("mypy", "/opt/repo/bin/python"), ["x: int = 1\n"]) + argv = seen[0] + assert argv[argv.index("--python-executable") + 1] == "/opt/repo/bin/python" + + +def test_nothing_to_check_runs_no_tool_not_even_a_version_probe(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + return _completed(returncode=1, stderr="No module named mypy") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + payload = scan_repo(_tree(tmp_path, "# Toy\n\nNo code here.\n"), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + assert seen == [] + assert payload["typecheck"]["tool_version"] is None + + +def test_fences_that_do_not_parse_run_no_tool_either(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + if "version_info" in argv[-1]: + return _completed(returncode=0, stdout="3 11\n") + return _completed(returncode=0, stdout=json.dumps(["invalid syntax"])) # the parse: one fence, not valid + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + payload = scan_repo(_tree(tmp_path, f"{FENCE}python\nx = (\n{FENCE}\n"), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + assert all("--version" not in argv and "mypy" not in argv for argv in seen) # no tool run + assert payload["typecheck"]["syntax"] == 1 + assert payload["typecheck"]["tool_version"] is None + + +def test_basedpyright_exit_two_is_the_tools_own_failure(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=2, stdout="{}", stderr="fatal")) + with pytest.raises(l0_typecheck.TypecheckError, match="exited 2"): + l0_typecheck._run(Check("basedpyright", sys.executable), ["x: int = 1\n"]) + + +def test_an_exit_code_outside_zero_to_two_is_the_tools_own_failure(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=3, stdout="", stderr="crashed")) + with pytest.raises(l0_typecheck.TypecheckError, match="exited 3"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_only_error_diagnostics_count_as_fence_errors(): + from dokimi.repos.l0 import typecheck as l0_typecheck + + mypy_out = "\n".join(json.dumps({"file": "fence-0.py", "line": n, "severity": sev, "message": "m"}) + for n, sev in [(1, "error"), (2, "note"), (3, "warning")]) + assert l0_typecheck._errors("mypy", mypy_out) == {0: [(1, "m")]} + pyright_out = json.dumps({"generalDiagnostics": [ + {"file": "/tmp/x/fence-0.py", "severity": sev, "message": sev, "range": {"start": {"line": n}}} + for n, sev in [(0, "error"), (1, "warning"), (2, "information")]]}) + assert l0_typecheck._errors("basedpyright", pyright_out) == {0: [(1, "error")]} From 4a24291b8ad7cc7d110d5663766f5fe507133274 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:33:12 -0800 Subject: [PATCH 19/43] fix(typecheck): the oracle also refuses an empty reference directory under a link copytree creates the reference's directories as well as its files, so an empty directory whose parent is a workspace symlink was created at the link's target. The check now covers every entry of the reference, not only its files. Thread: PRRT_kwDOVBg7g86yaepL Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 11 +++++------ dokimi/tests/test_repo_typecheck_grader.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 941cce25f..ceab1a3b3 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -157,12 +157,11 @@ def oracle(task: dict[str, Any], task_dir: Path, run_dir: Path | None = None) -> reference = task_dir / REFERENCE_DIR if reference.is_dir() and run_dir is not None: workspace = Path(run_dir) / "workspace" - for source in reference.rglob("*"): - if source.is_file(): - dest = workspace / source.relative_to(reference) - if _linked_on_the_way(workspace, dest): - raise ValueError(f"{dest.relative_to(workspace).as_posix()} would be written through a " - "symbolic link in the workspace") + for source in reference.rglob("*"): # directories too: copytree creates an empty one through a link + dest = workspace / source.relative_to(reference) + if _linked_on_the_way(workspace, dest): + raise ValueError(f"{dest.relative_to(workspace).as_posix()} would be written through a " + "symbolic link in the workspace") shutil.copytree(reference, workspace, dirs_exist_ok=True) return "" diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 221060769..688715666 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -283,3 +283,16 @@ def test_the_oracle_copies_the_reference_when_no_link_is_on_the_way(tmp_path): run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) typecheck.oracle(_task(), task_dir, run) assert (run / "workspace" / "pkg" / "api.py").read_text(encoding="utf-8") == CLEAN + + +def test_the_oracle_refuses_an_empty_reference_directory_created_through_a_link(tmp_path): + outside = tmp_path / "outside" + outside.mkdir() + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg" / "sub").mkdir(parents=True) # no file under it + run = tmp_path / "run" + (run / "workspace").mkdir(parents=True) + (run / "workspace" / "pkg").symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert not (outside / "sub").exists() From dc5732fab51e45684a7b2aa34f56db316a388fe0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 10:02:57 -0800 Subject: [PATCH 20/43] fix(typecheck): a malformed tool report is a failure; refuse links in the reference - A nonempty mypy or basedpyright report that is not the tool's JSON now raises TypecheckError, not zero errors. - The oracle refuses a symbolic link inside reference/: copytree follows it and the walk did not see the entries behind it. - The B108 finding (a literal /tmp path in a test) uses tmp_path. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 6 ++++- dokimi/dokimi/repos/l0/typecheck.py | 29 ++++++++++++++-------- dokimi/tests/test_repo_l0_typecheck.py | 20 +++++++++++++-- dokimi/tests/test_repo_typecheck_grader.py | 21 ++++++++++++++++ 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index ceab1a3b3..6f47abe9f 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -153,11 +153,15 @@ def _linked_on_the_way(workspace: Path, dest: Path) -> bool: def oracle(task: dict[str, Any], task_dir: Path, run_dir: Path | None = None) -> str: """No reference answer: the grade reads the workspace only. A task with a ``reference/`` directory has its files copied over the workspace first, so the reference repair is what gets graded. A destination - reached through a symbolic link in the workspace is refused before anything is copied.""" + reached through a symbolic link in the workspace is refused before anything is copied. A symbolic link in the + reference is refused too: copytree would follow it, and the entries behind it are not walked here.""" reference = task_dir / REFERENCE_DIR if reference.is_dir() and run_dir is not None: workspace = Path(run_dir) / "workspace" for source in reference.rglob("*"): # directories too: copytree creates an empty one through a link + if source.is_symlink(): + raise ValueError(f"reference/{source.relative_to(reference).as_posix()} is a symbolic link; " + "the reference is copied without links") dest = workspace / source.relative_to(reference) if _linked_on_the_way(workspace, dest): raise ValueError(f"{dest.relative_to(workspace).as_posix()} would be written through a " diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index 9e64f71d4..ea5d2f8b2 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -222,21 +222,30 @@ def _run_tool(argv: list[str], cwd: str, stdin: str | None = None): def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: """Parse the tool's JSON output into errors keyed by fence index.""" out: dict[int, list[tuple[int, str]]] = {} + diagnostics = [] + # A report we cannot read is a tool failure, not a clean run: the fences it would have flagged pass otherwise. if tool == "mypy": # one JSON object per line - diagnostics = [] for text in stdout.splitlines(): + if not text.strip(): + continue try: d = json.loads(text) - except ValueError: - continue - if isinstance(d, dict) and d.get("severity") == "error": - line = int(d.get("line") or 0) # mypy writes -1 for an error with no line - diagnostics.append((str(d.get("file")), max(line, 0), str(d.get("message")))) + if not isinstance(d, dict): + raise ValueError("not an object") + if d.get("severity") == "error": + line = int(d.get("line") or 0) # mypy writes -1 for an error with no line + diagnostics.append((str(d.get("file")), max(line, 0), str(d.get("message")))) + except (ValueError, TypeError) as exc: + raise TypecheckError(f"mypy wrote a line that is not a diagnostic in its report: {text[:200]!r}") \ + from exc else: # basedpyright: one document; its ranges are zero-based - doc = json.loads(stdout or "{}") - diagnostics = [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, - str(d.get("message")) + (f" [{d['rule']}]" if d.get("rule") else "")) - for d in doc.get("generalDiagnostics", []) if d.get("severity") == "error"] + try: + doc = json.loads(stdout or "{}") + diagnostics = [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, + str(d.get("message")) + (f" [{d['rule']}]" if d.get("rule") else "")) + for d in doc.get("generalDiagnostics", []) if d.get("severity") == "error"] + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise TypecheckError(f"basedpyright wrote a report that is not in its JSON format: {exc}") from exc for name, line, message in diagnostics: index = _fence_index(name) if index is not None: diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 26ae2e526..773a33189 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -542,13 +542,29 @@ def test_an_exit_code_outside_zero_to_two_is_the_tools_own_failure(monkeypatch): l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) -def test_only_error_diagnostics_count_as_fence_errors(): +def test_only_error_diagnostics_count_as_fence_errors(tmp_path): from dokimi.repos.l0 import typecheck as l0_typecheck mypy_out = "\n".join(json.dumps({"file": "fence-0.py", "line": n, "severity": sev, "message": "m"}) for n, sev in [(1, "error"), (2, "note"), (3, "warning")]) assert l0_typecheck._errors("mypy", mypy_out) == {0: [(1, "m")]} pyright_out = json.dumps({"generalDiagnostics": [ - {"file": "/tmp/x/fence-0.py", "severity": sev, "message": sev, "range": {"start": {"line": n}}} + {"file": str(tmp_path / "fence-0.py"), "severity": sev, "message": sev, "range": {"start": {"line": n}}} for n, sev in [(0, "error"), (1, "warning"), (2, "information")]]}) assert l0_typecheck._errors("basedpyright", pyright_out) == {0: [(1, "error")]} + + +@pytest.mark.parametrize("tool,stdout", [ + ("mypy", "not JSON"), + ("mypy", "[1, 2]"), + ("basedpyright", "not JSON"), + ("basedpyright", '{"generalDiagnostics": [{"severity": "error", "message": "m"}]}'), # no range +]) +def test_a_nonempty_report_that_is_not_the_tools_json_is_a_tool_failure(monkeypatch, tool, stdout): + # Exit 1 says the tool found errors; a report we cannot read must not pass every fence as clean. + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=1, stdout=stdout, stderr="")) + with pytest.raises(l0_typecheck.TypecheckError, match="report"): + l0_typecheck._run(Check(tool, sys.executable), ["x: int = 1\n"]) diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 688715666..f4d04a94f 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -276,6 +276,27 @@ def test_the_oracle_refuses_to_write_through_a_symbolic_link(tmp_path, linked): assert sentinel.read_text(encoding="utf-8") == "ORIGINAL" +def test_the_oracle_refuses_a_linked_reference_directory_with_a_linked_destination(tmp_path): + # copytree follows a linked reference directory and writes its entries through a linked destination + # file; the walk of the reference does not see inside the link, so the link itself is refused. + repairs = tmp_path / "repairs" + repairs.mkdir() + (repairs / "api.py").write_text("REFERENCE", encoding="utf-8") + outside = tmp_path / "outside" + outside.mkdir() + sentinel = outside / "api.py" + sentinel.write_text("ORIGINAL", encoding="utf-8") + task_dir = tmp_path / "task" + (task_dir / "reference").mkdir(parents=True) + (task_dir / "reference" / "pkg").symlink_to(repairs, target_is_directory=True) + run = tmp_path / "run" + (run / "workspace" / "pkg").mkdir(parents=True) + (run / "workspace" / "pkg" / "api.py").symlink_to(sentinel) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert sentinel.read_text(encoding="utf-8") == "ORIGINAL" + + def test_the_oracle_copies_the_reference_when_no_link_is_on_the_way(tmp_path): task_dir = tmp_path / "task" (task_dir / "reference" / "pkg").mkdir(parents=True) From 75f338f7234e768d1f903ec2a72584c166d01476 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:10:09 -0800 Subject: [PATCH 21/43] fix(typecheck): reference/ is optional; a report without its fields is a tool failure Without reference/ a dokimi:typecheck task has no oracle step: the self-check runs the null step alone, and the listed files must fail as they are. The grade is unchanged. A tool report that is valid JSON but has no diagnostic fields ("{}", a mypy line with no severity, basedpyright without a generalDiagnostics list) is a TypecheckError, not a clean run. Empty basedpyright output is a failure too. Signed-off-by: Nikolay Petrov --- .../20261031_120000_napetrov_repo-typecheck.md | 3 +++ dokimi/docs/repo-evaluation-contract.md | 9 ++++++--- dokimi/dokimi/repos/l0/typecheck.py | 13 ++++++++----- dokimi/dokimi/repos/selfcheck.py | 14 +++++++++++++- dokimi/tests/test_repo_l0_typecheck.py | 6 +++++- dokimi/tests/test_repo_typecheck_grader.py | 17 +++++++++++++++++ 6 files changed, 52 insertions(+), 10 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md index 307d5bbc4..9b5411766 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md @@ -6,3 +6,6 @@ each Python fence of the guidance against `--python-env`'s site-packages: a fence with an error is a `fail` stale-API candidate, a fence that does not parse or has no interpreter is `unresolved`. The report gains a `typecheck` block, `null` without the flag. See `docs/repo-evaluation-contract.md` §4.5 and §8.2. +- A `dokimi:typecheck` task needs no `reference/` directory: with one, its files are the repair the grade + reads; without one, the listed files are graded as they are and the self-check runs the null step only. +- A tool report without its diagnostic fields (for example `{}`) is a tool failure, not a clean run. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 82707b1f7..2b6b689a1 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -524,7 +524,8 @@ measure the patch's text, such as a guidance line, not the tool. For every task, before any paid run: - the null workspace (prepared, untouched) must fail the gate -- the workspace after `oracle` must pass the gate +- the workspace after `oracle` must pass the gate (a `dokimi:typecheck` task with no `reference/` + has no oracle step: its null step alone must fail, §4.5) - for score-only tasks, the oracle must reach `score.max` on `score.metric` - the agent container must not be able to read any file under `tasks` or `{source_dir}` (§1.1) @@ -633,8 +634,10 @@ listed file" (an error in a module a listed file imports belongs to that module, not to the listed file), and a listed file that is missing fails. The exception is a blocking error (a syntax error, a duplicate module) that stops mypy before it reaches the listed files: then that error is the grade's, and the grade fails. -The task's `oracle` copies its `reference/` directory, when it has one, over the -workspace, so a reference repair is what gets graded; its answer is empty. +`reference/` is optional. The task's `oracle` copies its `reference/` directory, when it has one, over the +workspace, so a reference repair is what gets graded; its answer is empty. Without `reference/` the grade +reads the listed files as they are, and the self-check (§4.3) runs the null step only: the untouched +listed files must fail, since there is no reference repair for the oracle step to pass. It runs mypy in-process in the grading container, so the image's `python3` must import the pinned mypy: a `version` other than the installed one is a grader failure, not a fail. It is not "no new errors against the base": the diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index ea5d2f8b2..20a2814fa 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -230,9 +230,10 @@ def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: continue try: d = json.loads(text) - if not isinstance(d, dict): - raise ValueError("not an object") - if d.get("severity") == "error": + # A valid JSON object is not a diagnostic: without these fields "{}" would read as a clean run. + if not isinstance(d, dict) or not {"file", "line", "severity", "message"} <= d.keys(): + raise ValueError("not a diagnostic") + if d["severity"] == "error": line = int(d.get("line") or 0) # mypy writes -1 for an error with no line diagnostics.append((str(d.get("file")), max(line, 0), str(d.get("message")))) except (ValueError, TypeError) as exc: @@ -240,10 +241,12 @@ def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: from exc else: # basedpyright: one document; its ranges are zero-based try: - doc = json.loads(stdout or "{}") + doc = json.loads(stdout) # empty output is not a report either + if not isinstance(doc, dict) or not isinstance(doc.get("generalDiagnostics"), list): + raise ValueError("no generalDiagnostics list") # a missing list is not an empty one diagnostics = [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, str(d.get("message")) + (f" [{d['rule']}]" if d.get("rule") else "")) - for d in doc.get("generalDiagnostics", []) if d.get("severity") == "error"] + for d in doc["generalDiagnostics"] if d.get("severity") == "error"] except (ValueError, KeyError, TypeError, AttributeError) as exc: raise TypecheckError(f"basedpyright wrote a report that is not in its JSON format: {exc}") from exc for name, line, message in diagnostics: diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index 3e3a1a782..f4ab89965 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -31,6 +31,8 @@ from typing import Any from .container import CONTENT_DIR, RUN_DIR, SOURCE_DIR, ContainerError, ContainerSpec, docker_network +from .graders import builtin_name +from .graders.typecheck import REFERENCE_DIR from .grading import EvalContext, outcome, run_entry from .host import probe_host, shortfall_kinds, unmet, validate_tasks from .runner import agent_layout, container_name @@ -82,7 +84,8 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, reason, unsupported_by, facts = _host_reason(ctx, task, work_dir, host) # Only what the hardware decides is skipped: the null check (it passes where the bug cannot show) and, # unless the miss is CPUs alone, the oracle. Preparing the workspace and isolation hold anywhere. - run_oracle = not reason or (unsupported_by == "host" and shortfall_kinds(task, facts) <= {"cpus"}) + run_oracle = _oracle_applies(task) and ( + not reason or (unsupported_by == "host" and shortfall_kinds(task, facts) <= {"cpus"})) for kind, run_it in (("null", not reason), ("oracle", run_oracle)): _check_step(ctx, task, work_dir / f"{task.id}__{kind}", kind, run_it, net, res) if not res["problems"]: @@ -95,6 +98,15 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, return res +def _oracle_applies(task: Task) -> bool: + """False for a built-in ``dokimi:typecheck`` task with no ``reference/``: its oracle has no repair to apply + (the grade reads the workspace as it is), so the self-check runs the null step only, and the untouched + listed files must fail the gate.""" + if builtin_name(task.raw.get("grader")) != "typecheck": + return True + return (task.dir / REFERENCE_DIR).is_dir() + + def _host_reason(ctx: EvalContext, task: Task, work_dir: Path, host: dict[str, Any] | None) -> tuple[str | None, str | None, dict[str, Any] | None]: """``(reason, unsupported_by, facts)``; reason and unsupported_by are ``None`` when the host suffices.""" diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 773a33189..8c8b59b60 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -417,7 +417,7 @@ def test_basedpyright_is_given_an_empty_project_so_no_config_above_the_run_appli def fake(argv, cwd, stdin=None): seen["argv"] = argv seen["config"] = Path(argv[argv.index("--project") + 1]).read_text(encoding="utf-8") # while it exists - return _completed(returncode=0, stdout="") + return _completed(returncode=0, stdout='{"generalDiagnostics": []}') # basedpyright always writes its report monkeypatch.setattr(l0_typecheck, "_run_tool", fake) l0_typecheck._run(Check("basedpyright", sys.executable), ["x: int = 1\n"]) @@ -559,6 +559,10 @@ def test_only_error_diagnostics_count_as_fence_errors(tmp_path): ("mypy", "[1, 2]"), ("basedpyright", "not JSON"), ("basedpyright", '{"generalDiagnostics": [{"severity": "error", "message": "m"}]}'), # no range + ("mypy", "{}"), # valid JSON, no diagnostic fields: not a clean run + ("mypy", '{"file": "fence-0.py", "line": 1, "message": "m"}'), # no severity + ("basedpyright", "{}"), # no generalDiagnostics list + ("basedpyright", '{"generalDiagnostics": 3}'), ]) def test_a_nonempty_report_that_is_not_the_tools_json_is_a_tool_failure(monkeypatch, tool, stdout): # Exit 1 says the tool found errors; a report we cannot read must not pass every fence as clean. diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index f4d04a94f..a0296e3eb 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -317,3 +317,20 @@ def test_the_oracle_refuses_an_empty_reference_directory_created_through_a_link( with pytest.raises(ValueError, match="symbolic link"): typecheck.oracle(_task(), task_dir, run) assert not (outside / "sub").exists() + + +def test_a_typecheck_task_without_reference_runs_no_oracle_step(tmp_path): + # reference/ is optional: without it the grade reads the workspace as it is, and the self-check has no + # repair to apply, so it runs the null step alone (the untouched listed files must fail the gate). + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + with_reference = tmp_path / "with" + (with_reference / "reference").mkdir(parents=True) + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": "1", "paths": ["pkg/api.py"]} + assert selfcheck._oracle_applies(Task("t", raw, bare)) is False + assert selfcheck._oracle_applies(Task("t", raw, with_reference)) is True + assert selfcheck._oracle_applies(Task("t", {"grader": "dokimi:review_seeds"}, bare)) is True + assert selfcheck._oracle_applies(Task("t", {}, bare)) is True From 1a737d5b7b8eda3b9f9572df678defbc61aef37f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 13:00:37 -0800 Subject: [PATCH 22/43] fix(typecheck): split _errors under the complexity cap; refuse a linked workspace with an empty reference - _errors (complexity 11) failed the growth ratchet in CI (Tests py3.11/py3.14). The mypy and basedpyright parsers are now _mypy_errors and _basedpyright_errors; behaviour is unchanged. - oracle checks the workspace root before the reference walk, so an empty reference/ no longer copies its mode onto a symlinked workspace. - Tests: the root case with an empty reference; _linked_on_the_way on a link in the middle of a path; the check_task gate that skips the oracle step for a typecheck task without reference/. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 3 ++ dokimi/dokimi/repos/l0/typecheck.py | 60 ++++++++++++---------- dokimi/tests/test_repo_typecheck_grader.py | 46 +++++++++++++++++ 3 files changed, 83 insertions(+), 26 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 6f47abe9f..fec50e9d5 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -158,6 +158,9 @@ def oracle(task: dict[str, Any], task_dir: Path, run_dir: Path | None = None) -> reference = task_dir / REFERENCE_DIR if reference.is_dir() and run_dir is not None: workspace = Path(run_dir) / "workspace" + # checked even for an empty reference: copytree still copies the mode onto a linked root + if _linked_on_the_way(workspace, workspace): + raise ValueError("the workspace is a symbolic link; the reference would be written through it") for source in reference.rglob("*"): # directories too: copytree creates an empty one through a link if source.is_symlink(): raise ValueError(f"reference/{source.relative_to(reference).as_posix()} is a symbolic link; " diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index 20a2814fa..8b41dbcd8 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -222,33 +222,8 @@ def _run_tool(argv: list[str], cwd: str, stdin: str | None = None): def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: """Parse the tool's JSON output into errors keyed by fence index.""" out: dict[int, list[tuple[int, str]]] = {} - diagnostics = [] # A report we cannot read is a tool failure, not a clean run: the fences it would have flagged pass otherwise. - if tool == "mypy": # one JSON object per line - for text in stdout.splitlines(): - if not text.strip(): - continue - try: - d = json.loads(text) - # A valid JSON object is not a diagnostic: without these fields "{}" would read as a clean run. - if not isinstance(d, dict) or not {"file", "line", "severity", "message"} <= d.keys(): - raise ValueError("not a diagnostic") - if d["severity"] == "error": - line = int(d.get("line") or 0) # mypy writes -1 for an error with no line - diagnostics.append((str(d.get("file")), max(line, 0), str(d.get("message")))) - except (ValueError, TypeError) as exc: - raise TypecheckError(f"mypy wrote a line that is not a diagnostic in its report: {text[:200]!r}") \ - from exc - else: # basedpyright: one document; its ranges are zero-based - try: - doc = json.loads(stdout) # empty output is not a report either - if not isinstance(doc, dict) or not isinstance(doc.get("generalDiagnostics"), list): - raise ValueError("no generalDiagnostics list") # a missing list is not an empty one - diagnostics = [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, - str(d.get("message")) + (f" [{d['rule']}]" if d.get("rule") else "")) - for d in doc["generalDiagnostics"] if d.get("severity") == "error"] - except (ValueError, KeyError, TypeError, AttributeError) as exc: - raise TypecheckError(f"basedpyright wrote a report that is not in its JSON format: {exc}") from exc + diagnostics = _mypy_errors(stdout) if tool == "mypy" else _basedpyright_errors(stdout) for name, line, message in diagnostics: index = _fence_index(name) if index is not None: @@ -256,6 +231,39 @@ def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: return out +def _mypy_errors(stdout: str) -> list[tuple[str, int, str]]: + """The error diagnostics in mypy's report: one JSON object per line.""" + diagnostics = [] + for text in stdout.splitlines(): + if not text.strip(): + continue + try: + d = json.loads(text) + # A valid JSON object is not a diagnostic: without these fields "{}" would read as a clean run. + if not isinstance(d, dict) or not {"file", "line", "severity", "message"} <= d.keys(): + raise ValueError("not a diagnostic") + if d["severity"] == "error": + line = int(d.get("line") or 0) # mypy writes -1 for an error with no line + diagnostics.append((str(d.get("file")), max(line, 0), str(d.get("message")))) + except (ValueError, TypeError) as exc: + raise TypecheckError(f"mypy wrote a line that is not a diagnostic in its report: {text[:200]!r}") \ + from exc + return diagnostics + + +def _basedpyright_errors(stdout: str) -> list[tuple[str, int, str]]: + """The error diagnostics in basedpyright's report: one document whose ranges are zero-based.""" + try: + doc = json.loads(stdout) # empty output is not a report either + if not isinstance(doc, dict) or not isinstance(doc.get("generalDiagnostics"), list): + raise ValueError("no generalDiagnostics list") # a missing list is not an empty one + return [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, + str(d.get("message")) + (f" [{d['rule']}]" if d.get("rule") else "")) + for d in doc["generalDiagnostics"] if d.get("severity") == "error"] + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise TypecheckError(f"basedpyright wrote a report that is not in its JSON format: {exc}") from exc + + def _fence_index(name: str) -> int | None: """``3`` for ``fence-3.py``; None for any other file (a note about an import, say).""" stem = Path(name).name diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index a0296e3eb..54dc123a4 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -319,6 +319,52 @@ def test_the_oracle_refuses_an_empty_reference_directory_created_through_a_link( assert not (outside / "sub").exists() +def test_the_oracle_refuses_a_linked_workspace_even_with_an_empty_reference(tmp_path): + # no entry in reference/, so the walk checks nothing; copytree still copies the mode onto the linked root + outside = tmp_path / "outside" + outside.mkdir(mode=0o700) + task_dir = tmp_path / "task" + (task_dir / "reference").mkdir(parents=True) + run = tmp_path / "run" + run.mkdir() + (run / "workspace").symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert outside.stat().st_mode & 0o777 == 0o700 + + +def test_linked_on_the_way_sees_a_link_in_the_middle_of_a_path(tmp_path): + # the oracle's walk also reaches this through the reference's directories; the helper is pinned on its own + outside = tmp_path / "outside" + (outside / "c").mkdir(parents=True) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "a").symlink_to(outside, target_is_directory=True) + assert typecheck._linked_on_the_way(workspace, workspace / "a" / "b" / "c.py") is True + assert typecheck._linked_on_the_way(workspace, workspace / "d" / "c.py") is False + + +def test_check_task_runs_no_oracle_step_for_a_typecheck_task_without_reference(tmp_path, monkeypatch): + # the gate in check_task itself: a mutated `_oracle_applies(...) and` would run the oracle here + from types import SimpleNamespace + + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": "1", "paths": ["pkg/api.py"]} + ran = [] + monkeypatch.setattr(selfcheck, "_host_reason", lambda ctx, task, work_dir, host: (None, None, None)) + monkeypatch.setattr(selfcheck, "_check_step", + lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) + monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + res = selfcheck.check_task(ctx, Task("t", raw, bare), tmp_path / "work") + assert ran == [("null", True), ("oracle", False)] + assert res["verdict"] == "ok" + + def test_a_typecheck_task_without_reference_runs_no_oracle_step(tmp_path): # reference/ is optional: without it the grade reads the workspace as it is, and the self-check has no # repair to apply, so it runs the null step alone (the untouched listed files must fail the gate). From e291dca46563d56fee6ffd5d374873620d8dc88c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 17:10:16 -0800 Subject: [PATCH 23/43] fix(typecheck): refuse line-separator paths, pin no-colour mypy output - _safe_path also refuses a path that splitlines() would break, so a listed file name with U+2028 or a newline cannot split mypy's output. - mypy runs with --no-color-output, so MYPY_FORCE_COLOR in the grader environment cannot put escape codes in the parsed error lines. - Tests: colour pin, separator parametrize cases, and an oracle-gate test that a typecheck task with reference/ runs the oracle step. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 12 ++++++--- dokimi/tests/test_repo_typecheck_grader.py | 31 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index fec50e9d5..87576a4ba 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -33,8 +33,10 @@ def _safe_path(path: object) -> bool: - """A workspace-relative path: no leading slash, no ``..`` segment, not empty.""" - return isinstance(path, str) and bool(path) and not path.startswith("/") and ".." not in path.split("/") + """A workspace-relative path: no leading slash, no ``..`` segment, not empty, and one line: a line separator in + a name would split it in mypy's output, so the error on that file would not be matched to it.""" + return (isinstance(path, str) and bool(path) and not path.startswith("/") and ".." not in path.split("/") + and path.splitlines() == [path]) def task_problems(raw: dict[str, Any]) -> list[str]: @@ -113,8 +115,10 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non roots = [*_package_roots(workspace, present), workspace] config = Path(neutral, "mypy.ini") config.write_text(f"[mypy]\nmypy_path = {':'.join(str(r) for r in roots)}\n", encoding="utf-8") - stdout, stderr, status = api.run(["--no-incremental", f"--cache-dir={cache}", f"--config-file={config}", - "--explicit-package-bases", *[str(workspace / p) for p in present]]) + # plain lines: the image's MYPY_FORCE_COLOR would put escapes around the file name and the marker + stdout, stderr, status = api.run(["--no-incremental", "--no-color-output", f"--cache-dir={cache}", + f"--config-file={config}", "--explicit-package-bases", + *[str(workspace / p) for p in present]]) finally: sys.setrecursionlimit(limit) if mypypath is not None: diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 54dc123a4..0910cf50c 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -112,6 +112,8 @@ def test_the_entry_point_writes_the_metrics_that_the_gate_reads(tmp_path): {"paths": ["/etc/passwd"]}, {"paths": ["../outside.py"]}, {"paths": "pkg/api.py"}, + {"paths": ["odd\nname/api.py"]}, # a line separator splits the name in mypy's output + {"paths": ["odd\u2028name/api.py"]}, ]) def test_task_problems_refuse_bad_fields(over): task = _task() @@ -153,6 +155,15 @@ def test_the_subjects_mypy_ini_cannot_suppress_a_listed_file(tmp_path): assert result["n_errors"] == 1 +def test_a_colored_mypy_image_still_fails_a_listed_file_with_a_type_error(tmp_path, monkeypatch): + # the image's env is kept for the grade: MYPY_FORCE_COLOR puts ANSI escapes around the file name and the marker + monkeypatch.setenv("MYPY_FORCE_COLOR", "1") + run = _run(tmp_path, {"pkg/api.py": WRONG}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + def test_a_listed_path_with_a_colon_is_matched(): out = typecheck._errors("a:b.py:3: error: boom [misc]", {"a:b.py"}, "") assert [(e["file"], e["line"]) for e in out] == [("a:b.py", 3)] @@ -365,6 +376,26 @@ def test_check_task_runs_no_oracle_step_for_a_typecheck_task_without_reference(t assert res["verdict"] == "ok" +def test_check_task_runs_the_oracle_step_for_a_typecheck_task_with_reference(tmp_path, monkeypatch): + # the mirror of the test above: a `False and (...)` gate would skip the oracle of a task that has a repair + from types import SimpleNamespace + + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + with_reference = tmp_path / "with" + (with_reference / "reference").mkdir(parents=True) + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": "1", "paths": ["pkg/api.py"]} + ran = [] + monkeypatch.setattr(selfcheck, "_host_reason", lambda ctx, task, work_dir, host: (None, None, None)) + monkeypatch.setattr(selfcheck, "_check_step", + lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) + monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + selfcheck.check_task(ctx, Task("t", raw, with_reference), tmp_path / "work") + assert ran == [("null", True), ("oracle", True)] + + def test_a_typecheck_task_without_reference_runs_no_oracle_step(tmp_path): # reference/ is optional: without it the grade reads the workspace as it is, and the self-check has no # repair to apply, so it runs the null step alone (the untouched listed files must fail the gate). From 3c0f97b9feed0f59a14a3f2c5fa33e1fe842ec3b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 18:27:25 -0800 Subject: [PATCH 24/43] fix(typecheck): match a mypy line to a listed file by the listed name A listed path may hold ': error: ' itself; the first separator in the line then ends the name too early and the error is lost. Each listed name is tried longest first and must be followed by a diagnostic. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 32 +++++++++++++++++++--- dokimi/tests/test_repo_typecheck_grader.py | 17 ++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 87576a4ba..c6c462427 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -30,6 +30,7 @@ REFERENCE_DIR = "reference" # task dir: the files of a reference repair, copied over the workspace by oracle MAX_ERRORS = 50 # reported in details; the count is in the metrics _ERROR = re.compile(r"^(?P.+?)(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") +_TAIL = re.compile(r"^(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") def _safe_path(path: object) -> bool: @@ -53,16 +54,39 @@ def task_problems(raw: dict[str, Any]) -> list[str]: return problems +def _split(line: str, listed: set[str] | None, root: str) -> tuple[str, re.Match[str]] | None: + """The file a mypy line is about, and the match of the diagnostic after it; None for a line that is not one. + + With ``listed``, the file is a listed name, normalised as the names are: a name may hold ": error: " itself, so + the first separator in the line is not always the end of the name. Each separator is tried as the end of the + name, and the last one that gives a listed name is taken (the longest name).""" + if listed is None: + m = _ERROR.match(line) + if m is None: + return None + name = m.group("file") + return posixpath.normpath(name[len(root):] if name.startswith(root) else name), m + text = line[len(root):] if line.startswith(root) else line + best: tuple[str, re.Match[str]] | None = None + for k, ch in enumerate(text): + if ch != ":": + continue + tail = _TAIL.match(text[k:]) + name = posixpath.normpath(text[:k]) + if tail is not None and name in listed: + best = (name, tail) + return best + + def _errors(stdout: str, listed: set[str] | None, root: str) -> list[dict[str, Any]]: """The error lines of mypy's output on a listed file (any file when ``listed`` is None), in output order; ``root`` is stripped from the names.""" out = [] for line in stdout.splitlines(): - m = _ERROR.match(line) - if not m: + hit = _split(line, listed, root) + if hit is None: continue - name = m.group("file") - name = posixpath.normpath(name[len(root):] if name.startswith(root) else name) + name, m = hit if listed is None or name in listed: out.append({"file": name, "line": int(m.group("line") or 0), "code": m.group("code") or "", "message": m.group("message")}) diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 0910cf50c..defe76b37 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -169,6 +169,23 @@ def test_a_listed_path_with_a_colon_is_matched(): assert [(e["file"], e["line"]) for e in out] == [("a:b.py", 3)] +def test_a_listed_path_containing_the_error_separator_is_matched(): + # the name itself holds ": error: "; the first separator in the line is not the end of the name + out = typecheck._errors("pkg/a: error: b.py:2: error: incompatible return value [return-value]", + {"pkg/a: error: b.py"}, "") + assert [(e["file"], e["line"], e["code"]) for e in out] == [("pkg/a: error: b.py", 2, "return-value")] + + +def test_a_listed_name_that_is_a_prefix_of_another_does_not_take_its_errors(): + out = typecheck._errors("pkg/a: error: b.py:2: error: boom", {"pkg/a", "pkg/a: error: b.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("pkg/a: error: b.py", 2)] + + +def test_a_mypy_spelling_of_a_listed_file_is_normalised_before_matching(): + out = typecheck._errors("./pkg//a.py:3: error: boom [misc]", {"pkg/a.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("pkg/a.py", 3)] + + def test_a_line_less_error_on_a_listed_file_is_counted(): out = typecheck._errors("pkg/a.py: error: Duplicate module named 'a'", {"pkg/a.py"}, "") assert [(e["file"], e["line"]) for e in out] == [("pkg/a.py", 0)] From 0ed179cb4be88cb173b518d1238ab14ec18a3fe0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 20:28:22 -0800 Subject: [PATCH 25/43] fix(typecheck): reject a source root mypy_path would split; skip POSIX-script test on Windows Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 3 +++ dokimi/tests/test_repo_l0_typecheck.py | 1 + dokimi/tests/test_repo_typecheck_grader.py | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index c6c462427..dddb20506 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -137,6 +137,9 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non # Neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are not read. # mypy_path is the workspace and each regular package's source root (src/ for src/pkg/__init__.py). roots = [*_package_roots(workspace, present), workspace] + bad = [str(r) for r in roots if re.search("[,:]", os.path.splitdrive(str(r))[1])] + if bad: # mypy splits mypy_path on , and :, so such a root would be read as two + raise ValueError(f"a source root has , or : in its name, which mypy_path cannot hold: {bad[0]}") config = Path(neutral, "mypy.ini") config.write_text(f"[mypy]\nmypy_path = {':'.join(str(r) for r in roots)}\n", encoding="utf-8") # plain lines: the image's MYPY_FORCE_COLOR would put escapes around the file name and the marker diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 8c8b59b60..1f89a2e9d 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -324,6 +324,7 @@ def test_a_host_mypypath_cannot_supply_a_module_the_pinned_environment_lacks(tmp assert claim["status"] == "fail" +@pytest.mark.skipif(sys.platform == "win32", reason="the stand-in interpreter is a POSIX shell script") def test_a_python_env_that_reports_no_version_is_a_usage_error_not_a_traceback(tmp_path, capsys): silent = tmp_path / "silent-python" silent.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index defe76b37..5ed26a83b 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -221,6 +221,16 @@ def test_a_src_layout_package_resolves_when_both_files_are_listed(tmp_path): assert result["pass"] is True, result["errors"] +@pytest.mark.parametrize("sep", [",", ":"]) +def test_a_source_root_with_a_mypy_path_delimiter_is_a_task_error(tmp_path, sep): + files = {f"src{sep}lib/pkg/__init__.py": "", f"src{sep}lib/pkg/x.py": "val: int = 1\n", + f"src{sep}lib/pkg/y.py": "from pkg.x import val\n\nprint(val + 1)\n"} + task = _task(paths=[f"src{sep}lib/pkg/x.py", f"src{sep}lib/pkg/y.py"]) + run = _run(tmp_path, files, task) + with pytest.raises(ValueError, match="mypy_path"): + typecheck.grade(task, tmp_path, "", run) + + def test_a_blocking_syntax_error_in_a_listed_file_fails_the_grade(tmp_path): run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "def broken(:\n pass\n"}, _task()) result = typecheck.grade(_task(), tmp_path, "", run) From 5f69d26b5bbdff1564ff99d8a8344f538f7715e8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 08:14:57 -0800 Subject: [PATCH 26/43] fix(typecheck): a listed path reached through a link is missing, not graded is_file() and mypy both follow symbolic links, so a listed file that the agent linked to the hidden reference repair was graded on that repair. A link on the way to a listed file, or a linked workspace root, now makes the path missing before any check or mypy run. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/graders/typecheck.py | 8 ++++++-- dokimi/tests/test_repo_typecheck_grader.py | 23 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index dddb20506..f2826b3f1 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -121,9 +121,13 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non if mypy.version.__version__ != task["version"]: raise ValueError(f"the image's mypy is {mypy.version.__version__}, the task pins {task['version']}") - workspace = (Path(run_dir or "/eval/run") / "workspace").resolve() # absolute: mypy gets these paths + linked_root = Path(run_dir or "/eval/run") / "workspace" # unresolved: a link at the root must be seen + workspace = linked_root.resolve() # absolute: mypy gets these paths paths = [posixpath.normpath(p) for p in task["paths"]] - missing = [p for p in paths if not (workspace / p).is_file()] + # is_file() and mypy both follow links, and the grading container mounts the hidden reference: a listed path + # with a link on the way (or a linked root) is missing, never read. + missing = [p for p in paths + if _linked_on_the_way(linked_root, linked_root / p) or not (workspace / p).is_file()] present = [p for p in paths if p not in missing] errors: list[dict[str, Any]] = [] if present: diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 5ed26a83b..3833751ec 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -289,6 +289,29 @@ def test_an_external_mypypath_cannot_supply_a_module_the_image_lacks(tmp_path, m assert os.environ["MYPYPATH"] == str(stubs) # restored for the caller +@pytest.mark.parametrize("linked", ["file", "directory", "workspace"]) +def test_a_listed_path_reached_through_a_link_is_missing_not_graded(tmp_path, linked): + # The agent links the listed file (or its directory, or the workspace) to a clean repair outside the workspace: + # is_file() and mypy would follow the link and pass it. The link is refused, so the file is missing. + outside = tmp_path / "outside" + (outside / "pkg").mkdir(parents=True) + (outside / "pkg" / "__init__.py").write_text("", encoding="utf-8") + (outside / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + workspace = tmp_path / "run" / "workspace" + if linked == "file": + (workspace / "pkg").mkdir(parents=True) + (workspace / "pkg" / "__init__.py").write_text("", encoding="utf-8") + (workspace / "pkg" / "api.py").symlink_to(outside / "pkg" / "api.py") + elif linked == "directory": + workspace.mkdir(parents=True) + (workspace / "pkg").symlink_to(outside / "pkg", target_is_directory=True) + else: + workspace.parent.mkdir(parents=True) + workspace.symlink_to(outside, target_is_directory=True) + result = typecheck.grade(_task(), tmp_path, "", tmp_path / "run") + assert result["pass"] is False and result["n_missing"] == 1 and result["missing"] == ["pkg/api.py"] + + @pytest.mark.parametrize("linked", ["file", "directory", "workspace"]) def test_the_oracle_refuses_to_write_through_a_symbolic_link(tmp_path, linked): outside = tmp_path / "outside" From c9f0e362d4e1508818cbd6727988f46ee4aa31c3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 09:44:35 -0800 Subject: [PATCH 27/43] fix(typecheck): refuse a grade with a link out of the workspace; set basedpyright's version A link whose target lies outside the workspace makes mypy read content this run cannot see, so the grade is refused as a grader error. basedpyright gets the repository interpreter's version as pythonVersion in its project file, since --pythonpath only sets import paths. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 7 +++++-- dokimi/dokimi/repos/graders/typecheck.py | 17 +++++++++++++++++ dokimi/dokimi/repos/l0/typecheck.py | 4 +++- dokimi/tests/test_repo_l0_typecheck.py | 14 ++++++++++++++ dokimi/tests/test_repo_typecheck_grader.py | 13 +++++++++++++ 5 files changed, 52 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 8d612cb9f..d423bef56 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -659,7 +659,9 @@ failed the task, and on a task with `score` that is a failed run with score `dokimi:typecheck` checks the listed files only. Its pass is "no error on a listed file" (an error in a module a listed file imports belongs to that module, -not to the listed file), and a listed file that is missing fails. The exception +not to the listed file), and a listed file that is missing fails. A symbolic link in the workspace whose target +lies outside it refuses the whole grade (a grader error, not a pass), listed or not: mypy follows an import through a +link, so an unlisted module could supply what a listed file imports. The exception is a blocking error (a syntax error, a duplicate module) that stops mypy before it reaches the listed files: then that error is the grade's, and the grade fails. `reference/` is optional. The task's `oracle` copies its `reference/` directory, when it has one, over the @@ -1507,7 +1509,8 @@ the tool, its version, the interpreter (`null`: unknown) and the counts flag. `--python-env` and `--typecheck-tool` need `--typecheck`. The fences are parsed with the grammar of the repository interpreter's version, and mypy runs with that `--python-version` and an explicit empty config, so the subject's and the host's -mypy settings do not apply. The tools run on the host through +mypy settings do not apply. basedpyright gets the same version as `pythonVersion` +in its explicit project file, since `--pythonpath` alone sets the import paths. The tools run on the host through `dokimi.process`; one run covers every fence of the repository. ```bash diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index f2826b3f1..3c9e619ed 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -110,6 +110,19 @@ def _package_roots(workspace: Path, paths: list[str]) -> list[Path]: return roots +def _links_outside(workspace: Path) -> list[str]: + """The symbolic links under the (resolved) workspace, a link to a directory included, whose target lies outside + it. mypy follows an import through a link, so a link to a file this run cannot see (a hidden repair) would be read + as the agent's own code, unlisted modules included. A link whose target stays inside is read as the file it names.""" + found = [] + for dirpath, dirnames, filenames in os.walk(workspace): + for name in (*dirnames, *filenames): + path = Path(dirpath, name) + if path.is_symlink() and not path.resolve().is_relative_to(workspace): + found.append(path.relative_to(workspace).as_posix()) + return found + + def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | None = None) -> dict[str, Any]: """Type-check the listed files in the workspace of ``run_dir``; the grader's result dict.""" del task_dir, answer # the answer is not read: the workspace is the subject @@ -131,6 +144,10 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non present = [p for p in paths if p not in missing] errors: list[dict[str, Any]] = [] if present: + outside = _links_outside(workspace) + if outside: # refused, not missing: the import of an unlisted module through it would be graded + raise ValueError(f"{outside[0]} is a symbolic link to a file outside the workspace; mypy would follow it " + "to content this run cannot see") cache = tempfile.mkdtemp(prefix="dokimi-mypy-") neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cfg-") limit = sys.getrecursionlimit() # mypy's main raises it for the whole process diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index 8b41dbcd8..d9412a549 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -186,8 +186,10 @@ def _run(check: Check, codes: list[str], "--output", "json", *names] else: # The project is explicit and empty: a pyrightconfig.json above the temporary directory is not read. + # the target version is set here: --pythonpath gives the import paths, not the version checked against + settings = {} if target is None else {"pythonVersion": f"{target[0]}.{target[1]}"} project = os.path.join(tmp, "pyrightconfig.json") - Path(project).write_text("{}\n", encoding="utf-8") + Path(project).write_text(json.dumps(settings) + "\n", encoding="utf-8") argv = [check.tool, "--project", project, "--pythonpath", str(check.python), "--outputjson", *names] done = _run_tool(argv, tmp) # 1: the tool found errors. mypy's 2: it stopped on a blocking one ("errors prevented further checking"), diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 1f89a2e9d..34d503b42 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -410,6 +410,20 @@ def test_a_tool_that_exits_two_with_no_fence_report_is_an_error_not_a_pass(monke l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) +def test_basedpyright_is_told_the_repository_interpreters_version(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = {} + + def fake(argv, cwd, stdin=None): + seen["config"] = json.loads(Path(argv[argv.index("--project") + 1]).read_text(encoding="utf-8")) + return _completed(returncode=0, stdout='{"generalDiagnostics": []}') + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("basedpyright", sys.executable), ["x: int = 1\n"], (3, 11)) + assert seen["config"] == {"pythonVersion": "3.11"} # not the interpreter's own version: the target + + def test_basedpyright_is_given_an_empty_project_so_no_config_above_the_run_applies(monkeypatch): from dokimi.repos.l0 import typecheck as l0_typecheck diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 3833751ec..ae89bdbd1 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -289,6 +289,19 @@ def test_an_external_mypypath_cannot_supply_a_module_the_image_lacks(tmp_path, m assert os.environ["MYPYPATH"] == str(stubs) # restored for the caller +def test_a_listed_file_importing_an_unlisted_link_to_a_hidden_repair_is_refused(tmp_path): + # api.py stays a regular file and imports pkg.repair, which is not listed and is a link to a clean repair outside + # the workspace: mypy follows the import through the link and would pass api.py on a module this run cannot see + outside = tmp_path / "outside" + outside.mkdir() + (outside / "repair.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "from pkg.repair import add\nassert add(1, 2) == 3\n"}, + _task()) + (run / "workspace" / "pkg" / "repair.py").symlink_to(outside / "repair.py") + with pytest.raises(ValueError, match="pkg/repair.py is a symbolic link to a file outside the workspace"): + typecheck.grade(_task(), tmp_path, "", run) + + @pytest.mark.parametrize("linked", ["file", "directory", "workspace"]) def test_a_listed_path_reached_through_a_link_is_missing_not_graded(tmp_path, linked): # The agent links the listed file (or its directory, or the workspace) to a clean repair outside the workspace: From 67bbc3a00f5faa1ae1ff143d084e72ec2af791ff Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 10:18:03 -0800 Subject: [PATCH 28/43] docs(repo): say when an external link refuses the typecheck grade The contract said a link out of the workspace always refuses the grade. The grader refuses only when mypy runs, that is when a listed file is present; a listed path reached through a link is missing, so with every listed file missing the grade fails with n_missing and no grader error. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index d423bef56..2132bb392 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -659,9 +659,12 @@ failed the task, and on a task with `score` that is a failed run with score `dokimi:typecheck` checks the listed files only. Its pass is "no error on a listed file" (an error in a module a listed file imports belongs to that module, -not to the listed file), and a listed file that is missing fails. A symbolic link in the workspace whose target -lies outside it refuses the whole grade (a grader error, not a pass), listed or not: mypy follows an import through a -link, so an unlisted module could supply what a listed file imports. The exception +not to the listed file), and a listed file that is missing fails. A listed path +reached through a symbolic link is missing, not read. A symbolic link in the workspace whose target +lies outside it refuses the whole grade (a grader error, not a pass), listed or not, whenever mypy runs, that is +when some listed file is present: mypy follows an import through a link, so an unlisted module could supply what a +listed file imports. When every listed file is missing, mypy does not run, and the grade is a failure with +`n_missing` set, not a grader error, even if a link lies outside. The exception is a blocking error (a syntax error, a duplicate module) that stops mypy before it reaches the listed files: then that error is the grade's, and the grade fails. `reference/` is optional. The task's `oracle` copies its `reference/` directory, when it has one, over the From 12ff812823df1716ef6849f9c6874d877dd6ffa3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 11:23:27 -0800 Subject: [PATCH 29/43] fix(typecheck): refuse silenced mypy; unresolve fragments and pre-3.10 targets - A `# type: ignore` or `# mypy:` comment in a listed file, or a .pyi in the workspace, fails the grade (n_suppressions); the grader cannot pass by silencing mypy. - mypy runs with --show-absolute-path, so a type error is graded from any working directory; paths equal after normpath are a task problem. - repo scan: a fence that is only missing context is unresolved; a repository older than mypy's floor (3.10) is unresolved, mypy not run. - --python-env must be a file; the CLI names --typecheck-tool in errors. - tasks check: a typecheck task with no reference and a missing listed path is a problem (its null grade is the only step). Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 8 +-- dokimi/dokimi/commands/repo.py | 7 +-- dokimi/dokimi/repos/graders/typecheck.py | 27 ++++++++-- dokimi/dokimi/repos/l0/typecheck.py | 57 +++++++++++++++++--- dokimi/dokimi/repos/selfcheck.py | 15 +++++- dokimi/tests/test_repo_l0_typecheck.py | 48 ++++++++++++++++- dokimi/tests/test_repo_typecheck_grader.py | 61 ++++++++++++++++++++++ 7 files changed, 202 insertions(+), 21 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 2132bb392..1890476e5 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -655,11 +655,11 @@ failed the task, and on a task with `score` that is a failed run with score |---|---|---|---|---| | `dokimi:review_seeds` | `manifest.json` (`manifest`); oracle answer `oracle.json` (`oracle_answer`), a list of findings | `null`; for a manifest with no seeds (a clean-diff control), `true` when no finding is `blocker` or `major` | `n_findings`, `n_severe`, `n_decoys`, `unmatched`, `recall` (`null` with no seeds) | `seeds_hit`, `decoys_flagged`, `recall_by_class` (`"hit/total"`) | | `dokimi:review_replay` | `gold.json` (`gold`); `task.json` `window`, default 5 | `null` (score-only) | `n_findings`, `n_severe`, `n_threads`, `recall` (= `recall_all`), `recall_addressed`, `precision_proxy` | `threads_hit`, `matches`, `recall_by_kind`, `decoys_flagged` (always `[]`) | -| `dokimi:typecheck` | none: the task's own fields `tool` (`mypy`), `version` (the image's mypy, exactly), `paths` (workspace-relative files) | `true` when no error is reported on a listed file and every listed file exists, else `false` | `n_errors`, `n_missing`, `n_files` | `errors` (first 50, `file:line: message [code]`), `missing`, `tool`, `tool_version` | +| `dokimi:typecheck` | none: the task's own fields `tool` (`mypy`), `version` (the image's mypy, exactly), `paths` (workspace-relative files) | `true` when no error is reported on a listed file, every listed file exists, and nothing silences mypy, else `false` | `n_errors`, `n_missing`, `n_suppressions`, `n_files` | `errors` (first 50, `file:line: message [code]`, and each silenced spot), `missing`, `tool`, `tool_version` | `dokimi:typecheck` checks the listed files only. Its pass is "no error on a listed file" (an error in a module a listed file imports belongs to that module, -not to the listed file), and a listed file that is missing fails. A listed path +not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), is a suppression, and the grade counts it in `n_suppressions`. A listed path reached through a symbolic link is missing, not read. A symbolic link in the workspace whose target lies outside it refuses the whole grade (a grader error, not a pass), listed or not, whenever mypy runs, that is when some listed file is present: mypy follows an import through a link, so an unlisted module could supply what a @@ -1497,7 +1497,9 @@ claim: guidance file, and its `why` names the first error (`mypy: line N: message`). - `unresolved`: the fence is not valid Python (a placeholder such as `f()`), is too deeply nested for the parser, or no `--python-env` was given, so nothing - was checked. The reason says which. + was checked. It is also unresolved when its only errors are missing context (a + name an earlier fence defines, `await`/`return` outside a function), or when the + repository is older than the tool checks (mypy: Python 3.10). The reason says which. `--python-env PYTHON` names the repository's interpreter, a virtualenv with its pinned dependencies installed (and the repository itself, if its fences import diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 3a1bb268c..c2cfe3a28 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -137,11 +137,12 @@ def _typecheck_or_exit(args: argparse.Namespace): sys.exit(EXIT_USAGE) return None tool = args.typecheck_tool or "mypy" - if args.python_env is not None and not os.access(args.python_env, os.X_OK): + # a directory is X_OK too: only a regular file can be the interpreter + if args.python_env is not None and not (os.path.isfile(args.python_env) and os.access(args.python_env, os.X_OK)): print(f"✗ --python-env {args.python_env}: not an executable interpreter", file=sys.stderr) sys.exit(EXIT_USAGE) if tool != "mypy" and shutil.which(tool) is None: - print(f"✗ --typecheck {tool}: not on PATH", file=sys.stderr) + print(f"✗ --typecheck-tool {tool}: not on PATH", file=sys.stderr) sys.exit(EXIT_USAGE) python = os.path.abspath(args.python_env) if args.python_env is not None else None # the tool runs elsewhere return Check(tool, python) @@ -160,7 +161,7 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: try: payload = scan_repo(tree, manifest, static, typecheck=check) except TypecheckError as exc: - print(f"✗ --typecheck {check.tool}: {exc}", file=sys.stderr) + print(f"✗ --typecheck-tool {check.tool}: {exc}", file=sys.stderr) sys.exit(EXIT_USAGE) if args.out: try: diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 3c9e619ed..0a2ad685b 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -5,7 +5,7 @@ task.json: ``"tool": "mypy"``, ``"version"`` (exactly the image's mypy version, else a grader failure) and ``"paths"`` (workspace-relative files). ``pass`` is true when no error is reported -on a listed file and every listed file exists. Errors in other modules are not counted. +on a listed file, every listed file exists and nothing silences mypy. Errors in other modules are not counted. It runs mypy in-process: the grader cannot start a child (the import layers allow spawns only in ``dokimi.process``, which the container cannot import), so basedpyright is not offered here. The @@ -31,6 +31,9 @@ MAX_ERRORS = 50 # reported in details; the count is in the metrics _ERROR = re.compile(r"^(?P.+?)(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") _TAIL = re.compile(r"^(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") +# A comment that silences mypy: ``# type: ignore`` on a line, or a mypy directive comment (ignore-errors, ...). +# A listed file with one is not graded clean, whatever mypy then reports. +_SILENCE = re.compile(r"#\s*(?:type:\s*ignore|mypy:)") def _safe_path(path: object) -> bool: @@ -51,6 +54,8 @@ def task_problems(raw: dict[str, Any]) -> list[str]: paths = raw.get("paths") if not isinstance(paths, list) or not paths or not all(_safe_path(p) for p in paths): problems.append("paths must be a non-empty list of workspace-relative file paths") + elif len({posixpath.normpath(p) for p in paths}) != len(paths): # "a.py" and "./a.py": one file, a duplicate module + problems.append("paths must be distinct after normalising (\"a.py\" and \"./a.py\" are one file)") return problems @@ -110,6 +115,19 @@ def _package_roots(workspace: Path, paths: list[str]) -> list[Path]: return roots +def _silenced(workspace: Path, listed: list[str]) -> list[str]: + """Where the workspace silences mypy: ``file:line`` for a comment, the path for a stub (``.pyi`` shadows a module). + + A ``.pyi`` anywhere is refused: mypy_path puts the workspace first, so ``__getattr__ -> Any`` hides every attribute.""" + found = [] + for p in listed: + text = (workspace / p).read_text(encoding="utf-8", errors="replace") + found += [f"{p}:{n}" for n, line in enumerate(text.splitlines(), 1) if _SILENCE.search(line)] + for dirpath, _, filenames in os.walk(workspace): + found += [Path(dirpath, name).relative_to(workspace).as_posix() for name in filenames if name.endswith(".pyi")] + return found + + def _links_outside(workspace: Path) -> list[str]: """The symbolic links under the (resolved) workspace, a link to a directory included, whose target lies outside it. mypy follows an import through a link, so a link to a file this run cannot see (a hidden repair) would be read @@ -166,6 +184,7 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non # plain lines: the image's MYPY_FORCE_COLOR would put escapes around the file name and the marker stdout, stderr, status = api.run(["--no-incremental", "--no-color-output", f"--cache-dir={cache}", f"--config-file={config}", "--explicit-package-bases", + "--show-absolute-path", # cwd-independent names, matched to the listed ones *[str(workspace / p) for p in present]]) finally: sys.setrecursionlimit(limit) @@ -183,9 +202,11 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non # mypy stopped at a blocking error in a file it did not reach ("errors prevented further # checking"): the listed files were not checked, so that error is the grade's errors = _errors(stdout, None, root) + silenced = _silenced(workspace, present) shown = [f"{e['file']}:{e['line']}: {e['message']}" + (f" [{e['code']}]" if e["code"] else "") for e in errors] - return {"pass": not errors and not missing, "n_errors": len(errors), "n_missing": len(missing), - "n_files": len(paths), "errors": shown[:MAX_ERRORS], "missing": missing, + shown += [f"{s}: mypy is silenced here (fix the code instead)" for s in silenced] + return {"pass": not errors and not missing and not silenced, "n_errors": len(errors), "n_missing": len(missing), + "n_suppressions": len(silenced), "n_files": len(paths), "errors": shown[:MAX_ERRORS], "missing": missing, "tool": TOOL, "tool_version": mypy.version.__version__} diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index d9412a549..a93b0cce4 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -13,6 +13,7 @@ import json import os +import re import sys import tempfile import textwrap @@ -28,6 +29,12 @@ TOOLS = ("mypy", "basedpyright") TIMEOUT = 600 # seconds for one run over every fence of a repository MAX_WHY = 200 +# The oldest repository Python each tool accepts: the pinned mypy 2.x refuses --python-version 3.9 and exits 2. +MIN_PYTHON = {"mypy": (3, 10)} +# Errors a fence has only because it is a fragment of a longer example: a name an earlier fence defines, or +# await/return/yield outside a function. Any other error is real, whatever else the fence has. +_FRAGMENT = re.compile(r'^(?:Name "[^"]+"|"[^"]+") is not defined(?: \[reportUndefinedVariable\])?$' + r'|^"(?:await|return|yield)" outside function$') class TypecheckError(RuntimeError): @@ -77,28 +84,62 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: return block # Nothing to run, no tool run: a version probe would fail on a repository without the tool, for no result. block["tool_version"] = tool_version(check) if runnable else None + if _older_than_the_tool(scan, check, runnable, target): + block["unknown"] = len(runnable) + return block # no file: usage error. ``blocked``: the tool stopped early, so a fence with no error was not necessarily checked. found_errors, blocked = _run(check, [code for _, _, code in runnable], target) if runnable else ({}, False) - unchecked = 0 + fails, unchecked = _claims(scan, check.tool, runnable, found_errors, blocked) + block["fail"] = fails + block["checked"] = len(runnable) - unchecked + block["unknown"] += unchecked # "unknown" is every fence that was not checked + return block + + +def _claims(scan: Scan, tool: str, runnable: list[tuple[str, Fence, str]], + found_errors: dict[int, list[tuple[int, str]]], blocked: bool) -> tuple[int, int]: + """One claim per runnable fence from the tool's errors; (failing fences, fences not checked).""" + fails = unchecked = 0 for i, (f, fence, code) in enumerate(runnable): errors = found_errors.get(i, []) if not errors and blocked: unchecked += 1 scan.add(f, fence.line, "python_fence", _label(code), "unresolved", - f"{check.tool} stopped before this fence was checked: not type-checked") + f"{tool} stopped before this fence was checked: not type-checked") continue if not errors: - scan.add(f, fence.line, "python_fence", _label(code), "ok", f"{check.tool}: no error") + scan.add(f, fence.line, "python_fence", _label(code), "ok", f"{tool}: no error") continue - block["fail"] += 1 + if _is_fragment(errors): + unchecked += 1 + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + f"{tool}: {errors[0][1]} (a fragment of a longer example: not type-checked)"[:MAX_WHY]) + continue + fails += 1 line, message = errors[0] more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" where = f"line {line}: " if line else "" # 0: the tool gave no line, so the claim is the fence itself scan.add(f, fence.line + line, "python_fence", _label(code), "fail", - f"{check.tool}: {where}{message}{more}"[:MAX_WHY]) - block["checked"] = len(runnable) - unchecked - block["unknown"] += unchecked # "unknown" is every fence that was not checked - return block + f"{tool}: {where}{message}{more}"[:MAX_WHY]) + return fails, unchecked + + +def _older_than_the_tool(scan: Scan, check: Check, runnable: list[tuple[str, Fence, str]], + target: tuple[int, int] | None) -> bool: + """True, with each runnable fence unresolved, when the repository is older than the tool checks (no run).""" + floor = MIN_PYTHON.get(check.tool, (0, 0)) + if target is None or target >= floor: + return False + for f, fence, code in runnable: + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + f"{check.tool} checks Python {floor[0]}.{floor[1]} and later; the repository is " + f"{target[0]}.{target[1]}: not type-checked") + return True + + +def _is_fragment(errors: list[tuple[int, str]]) -> bool: + """True when every error of a fence is one its missing context causes (see ``_FRAGMENT``).""" + return all(_FRAGMENT.match(message) for _, message in errors) def tool_version(check: Check) -> str: diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index f4ab89965..35b7c92e4 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -33,7 +33,7 @@ from .container import CONTENT_DIR, RUN_DIR, SOURCE_DIR, ContainerError, ContainerSpec, docker_network from .graders import builtin_name from .graders.typecheck import REFERENCE_DIR -from .grading import EvalContext, outcome, run_entry +from .grading import EvalContext, outcome, read_grade_json, run_entry from .host import probe_host, shortfall_kinds, unmet, validate_tasks from .runner import agent_layout, container_name from .workspace import PrepError, Task, prepare_workspace @@ -148,6 +148,16 @@ def _prepare(ctx: EvalContext, task: Task, rd: Path) -> None: (rd / "events.jsonl").write_text("", encoding="utf-8") +def _missing_listed(rd: Path) -> list[str]: + """For a typecheck task with no reference the null grade is the only step: a listed path it reports missing + fails that step, so no run could pass the task.""" + doc, _ = read_grade_json(rd / "grade.json") + missing = doc.get("missing") if isinstance(doc, dict) else None + if not isinstance(missing, list) or not missing: + return [] + return [f"null: listed path {', '.join(str(p) for p in missing)} is missing; with no reference no run can pass"] + + def _verdict_problems(task: Task, kind: str, grade: dict[str, Any], rd: Path) -> list[str]: if grade["status"] == "grader_failure": return [f"{kind}: grader failure ({'; '.join(grade['reasons'])}); see {rd / 'grade.log'}"] @@ -155,7 +165,8 @@ def _verdict_problems(task: Task, kind: str, grade: dict[str, Any], rd: Path) -> reached = top is not None and grade["score"] is not None and grade["score"] >= top if kind == "null": out = ["null: the untouched workspace passes the gate"] if grade["success"] else [] - return out + ([f"null: the untouched workspace reaches score.max {top}"] if reached else []) + out += [f"null: the untouched workspace reaches score.max {top}"] if reached else [] + return out + ([] if _oracle_applies(task) else _missing_listed(rd)) if grade["score_only"] and task.score: # outcome() fails pass: null when the task has no `score` return [] if reached else [f"oracle: {task.score['metric']} = {grade['score']}, below score.max {top}"] out = [] if grade["success"] else ["oracle: the reference solution does not pass the gate"] diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 34d503b42..40d36075d 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -218,7 +218,7 @@ def broken(check, codes, target=None): with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 2 - assert "--typecheck mypy: mypy exited 2: broken" in capsys.readouterr().err + assert "--typecheck-tool mypy: mypy exited 2: broken" in capsys.readouterr().err def test_basedpyright_missing_from_path_is_a_usage_error(tmp_path, capsys, monkeypatch): @@ -229,7 +229,7 @@ def test_basedpyright_missing_from_path_is_a_usage_error(tmp_path, capsys, monke with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 2 - assert "--typecheck basedpyright: not on PATH" in capsys.readouterr().err + assert "--typecheck-tool basedpyright: not on PATH" in capsys.readouterr().err def test_an_interpreter_with_nothing_to_check_runs_no_tool(tmp_path): @@ -587,3 +587,47 @@ def test_a_nonempty_report_that_is_not_the_tools_json_is_a_tool_failure(monkeypa returncode=1, stdout=stdout, stderr="")) with pytest.raises(l0_typecheck.TypecheckError, match="report"): l0_typecheck._run(Check(tool, sys.executable), ["x: int = 1\n"]) + + +def test_a_repository_older_than_mypy_supports_is_unresolved_and_mypy_is_not_run(tmp_path, monkeypatch): + # mypy 2.x refuses --python-version 3.9 and exits 2: the fences are unknown for that reason, not a tool failure + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 9)) + + def no_run(*args, **kwargs): + raise AssertionError("mypy must not run for a target it refuses") + + monkeypatch.setattr(l0_typecheck, "_run", no_run) + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["unresolved", "unresolved", "unresolved"] + assert "3.9" in claims[0]["why"] and "3.10" in claims[0]["why"] + assert payload["typecheck"]["checked"] == 0 + + +def test_a_fence_that_is_a_fragment_of_a_longer_example_is_unresolved_not_stale(tmp_path): + # `client` is defined by an earlier fence, and `await` at top level is only valid in an async body + fragments = f'{FENCE}python\nclient.decode("{{}}")\n{FENCE}\n\n{FENCE}python\nawait asyncio.sleep(1)\n{FENCE}\n' + claims = _fence_claims(scan_repo(_tree(tmp_path, fragments), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved", "unresolved"] + assert all("longer example" in c["why"] for c in claims) + + +def test_a_fragment_with_a_real_error_still_fails(tmp_path): + fence = f'{FENCE}python\nclient.decode("{{}}")\nx: int = "a"\n{FENCE}\n' + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + + +def test_the_scan_command_refuses_a_python_env_that_is_a_directory(tmp_path, capsys): + # os.access(X_OK) is true for a directory: only a regular file can be the interpreter + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", str(tmp_path)]) + from dokimi.commands.repo import _typecheck_or_exit + + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "not an executable interpreter" in capsys.readouterr().err diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index ae89bdbd1..144e8484c 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -474,3 +474,64 @@ def test_a_typecheck_task_without_reference_runs_no_oracle_step(tmp_path): assert selfcheck._oracle_applies(Task("t", raw, with_reference)) is True assert selfcheck._oracle_applies(Task("t", {"grader": "dokimi:review_seeds"}, bare)) is True assert selfcheck._oracle_applies(Task("t", {}, bare)) is True + + +@pytest.mark.parametrize("source", [ + "def add(a: int, b: int) -> int:\n return f'{a}{b}' # type: ignore\n", + "# mypy: ignore-errors\n" + WRONG, + "# mypy: disable-error-code=return-value\n" + WRONG, +]) +def test_a_suppression_in_a_listed_file_fails_the_grade(tmp_path, source): + # the agent must fix the code, not hide the error: a listed file with a suppression never passes + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_a_stub_in_the_workspace_that_hides_a_module_fails_the_grade(tmp_path): + # mypy_path puts the workspace first: a stub there replaces json, and its __getattr__ hides json.nosuch() + stub = "from typing import Any\n\ndef __getattr__(name: str) -> Any: ...\n" + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "import json\n\nprint(json.nosuch())\n", + "json.pyi": stub}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_a_clean_listed_file_has_no_suppressions(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_suppressions"] == 0 + + +def test_a_type_error_is_graded_from_any_working_directory(tmp_path, monkeypatch): + # mypy prints names relative to the cwd: "workspace/pkg/api.py" matched no listed name, so the error was dropped + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + monkeypatch.chdir(run) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + +def test_two_listed_names_for_one_file_are_a_task_error(): + # "a.py" and "./a.py" are one file: mypy reports it as a duplicate module, so the task could never pass + problems = typecheck.task_problems(_task(paths=["pkg/api.py", "./pkg/api.py"])) + assert any("distinct" in p for p in problems) + + +def test_a_typecheck_task_without_reference_whose_listed_path_is_missing_is_a_problem(tmp_path): + # with no reference the null grade is the only step: a typo'd path fails it, so no run could ever pass + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": PINNED, "paths": ["src/pkg/modle.py"]} + rd = tmp_path / "rd" + rd.mkdir() + (rd / "grade.json").write_text(json.dumps({"pass": False, "missing": ["src/pkg/modle.py"]}), encoding="utf-8") + grade = {"status": "graded", "success": False, "score": None, "score_only": False, "pass": False} + problems = selfcheck._verdict_problems(Task("t", raw, bare), "null", grade, rd) + assert any("missing" in p and "src/pkg/modle.py" in p for p in problems) From ac47ae99e633042dc719fcf31bfe392b7479bd46 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 14:16:01 -0800 Subject: [PATCH 30/43] docs(repo): name the mypy pin for repo scan --typecheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The help and contract §8.2 called mypy "the host's pinned copy", but pip install dokimi does not install it: the pin is requirements-test.txt (mypy==2.4.0), and a missing mypy exits 2. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 9 +++++---- dokimi/dokimi/commands/repo.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 1890476e5..066efb12e 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1506,10 +1506,11 @@ pinned dependencies installed (and the repository itself, if its fences import it). The fences are checked against that site-packages with imports followed silently (`--follow-imports=silent`), so only errors in the fence count. A module the environment does not have is a `fail`: a fence that imports an -uninstalled package is a fence the pinned environment cannot run. mypy is the -host's pinned copy; basedpyright must be on `PATH` and is recorded with the -version it reports, not pinned by dokimi. The report's `typecheck` block names -the tool, its version, the interpreter (`null`: unknown) and the counts +uninstalled package is a fence the pinned environment cannot run. `pip install +dokimi` does not install mypy: the tested pin is `requirements-test.txt` +(`mypy==2.4.0`), and a missing mypy exits 2. basedpyright must be on `PATH` and +is recorded with the version it reports, not pinned by dokimi. The report's +`typecheck` block names the tool, its version, the interpreter (`null`: unknown) and the counts (`fences`, `checked`, `fail`, `unknown`, `syntax`); it is `null` without the flag. `--python-env` and `--typecheck-tool` need `--typecheck`. The fences are parsed with the grammar of the repository interpreter's version, and mypy runs with that diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index c2cfe3a28..ceea4e1b2 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -677,8 +677,9 @@ def register(sub, positive_int) -> None: "reported and makes the command exit 1") scan_p.add_argument("--typecheck", action="store_true", default=False, help="Type-check each Python fence of the guidance (§8.2): one claim per fence, fail when the " - "tool reports an error. The tool is mypy (the host's pinned copy) unless --typecheck-tool names " - "basedpyright, which must be on PATH. Without --python-env the fences are unknown (unresolved), " + "tool reports an error. The tool is mypy unless --typecheck-tool names basedpyright (on PATH). " + "mypy is not installed by pip install dokimi: the tested pin is requirements-test.txt " + "(mypy==2.4.0); a missing mypy exits 2. Without --python-env the fences are unknown (unresolved), " "not checked") scan_p.add_argument("--typecheck-tool", default=None, dest="typecheck_tool", choices=("mypy", "basedpyright"), help="The tool --typecheck runs (default mypy). Needs --typecheck") From 47a95a6e86c470aeeb0aeba86b70de1c70087344 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 16:21:57 -0800 Subject: [PATCH 31/43] fix(typecheck): suppressions are comments; basedpyright fragments; 3.7 parse - A listed file is suppressed by a `# type: ignore` or `# mypy:` comment, read with tokenize: a marker in a string or docstring no longer fails a clean file. A file the tokenizer cannot read is still scanned line by line. - basedpyright words top-level await/return/yield its own way (with an optional rule suffix): those fences are unresolved, as mypy's are. - The fence parser omits type_comments on Python 3.7 (the keyword is 3.8+). - Contract 4.5: dokimi:typecheck imports mypy (3.10+) and oracle copies reference/. - The Check.tool comment names mypy's real pin (requirements-test.txt). Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 10 +++-- dokimi/dokimi/repos/graders/typecheck.py | 14 +++++- dokimi/dokimi/repos/l0/typecheck.py | 17 ++++++-- dokimi/tests/test_repo_l0_typecheck.py | 50 ++++++++++++++++++++++ dokimi/tests/test_repo_typecheck_grader.py | 29 +++++++++++++ 5 files changed, 112 insertions(+), 8 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 066efb12e..5c403d0c4 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -602,7 +602,8 @@ repository's checks with its own without knowing their rules. A task whose `grader` is `dokimi:` is graded by the framework, not by the repository's `grade` and `oracle` entry points; any other name goes to those entry points. Such a task is data only: its directory holds -`task.json`, the patch and the JSON files the grader reads. The `dokimi:` +`task.json`, the patch and the JSON files the grader reads (and, for +`dokimi:typecheck`, an optional `reference/` directory, §4.5 below). The `dokimi:` prefix is the framework's: an unknown `dokimi:` name fails `tasks check` and the run's grade, and is never handed to the repository. @@ -612,7 +613,9 @@ framework's grader code is mounted read-only at `/eval/graders` (never into the agent's container) and runs as `python3 -I -B /eval/graders/main.py grade|oracle /eval/run`, where `` is the task's directory under `{eval_dir}`. It uses the -standard library only and the image's `python3` (3.8 or newer). `-I` keeps +standard library only and the image's `python3` (3.8 or newer), except +`dokimi:typecheck`, which imports the image's mypy (the pinned one, on Python +3.10 or newer). `-I` keeps `PYTHONPATH` from `image.env` and the workspace off `sys.path`; `-B` prevents writing `.pyc` files to the read-only mount. An image with no `python3` on `PATH` is a grader failure that says so: the grader exits 127 and its @@ -626,7 +629,8 @@ other exit 127 is reported as a plain `grade exited 127`. `pass: null` with `errors`. A metric the task's `gate` names that the grader did not produce is `null`. - `oracle` writes the reference answer to `answer.txt` and leaves the - workspace alone. + workspace alone, except `dokimi:typecheck`, whose `oracle` copies `reference/` + over the workspace (see its paragraph below). A review task with `pass: null` is scored, not gated: give it `score`. A `gate` on its metrics is not applied (`success` is null), and without `score` the run diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 0a2ad685b..1c2961be7 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -15,12 +15,14 @@ from __future__ import annotations +import io import os import posixpath import re import shutil import sys import tempfile +import tokenize from pathlib import Path from typing import Any @@ -115,6 +117,16 @@ def _package_roots(workspace: Path, paths: list[str]) -> list[Path]: return roots +def _comments(text: str) -> list[tuple[int, str]]: + """The comments of a Python source, with their lines: a marker inside a string or a docstring is not one. + A source the tokenizer cannot read is taken line by line, so a marker in it still counts.""" + try: + tokens = tokenize.generate_tokens(io.StringIO(text).readline) + return [(tok.start[0], tok.string) for tok in tokens if tok.type == tokenize.COMMENT] + except (tokenize.TokenError, SyntaxError): + return list(enumerate(text.splitlines(), 1)) + + def _silenced(workspace: Path, listed: list[str]) -> list[str]: """Where the workspace silences mypy: ``file:line`` for a comment, the path for a stub (``.pyi`` shadows a module). @@ -122,7 +134,7 @@ def _silenced(workspace: Path, listed: list[str]) -> list[str]: found = [] for p in listed: text = (workspace / p).read_text(encoding="utf-8", errors="replace") - found += [f"{p}:{n}" for n, line in enumerate(text.splitlines(), 1) if _SILENCE.search(line)] + found += [f"{p}:{n}" for n, comment in _comments(text) if _SILENCE.search(comment)] for dirpath, _, filenames in os.walk(workspace): found += [Path(dirpath, name).relative_to(workspace).as_posix() for name in filenames if name.endswith(".pyi")] return found diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index a93b0cce4..b51494270 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -32,9 +32,13 @@ # The oldest repository Python each tool accepts: the pinned mypy 2.x refuses --python-version 3.9 and exits 2. MIN_PYTHON = {"mypy": (3, 10)} # Errors a fence has only because it is a fragment of a longer example: a name an earlier fence defines, or -# await/return/yield outside a function. Any other error is real, whatever else the fence has. +# await/return/yield outside a function. Each tool words them its own way (mypy, then basedpyright, which may add +# a rule name). Any other error is real, whatever else the fence has. _FRAGMENT = re.compile(r'^(?:Name "[^"]+"|"[^"]+") is not defined(?: \[reportUndefinedVariable\])?$' - r'|^"(?:await|return|yield)" outside function$') + r'|^"(?:await|return|yield)" outside function$' + r'|^"await" allowed only within async function(?: \[\w+\])?$' + r'|^"return" can be used only within a function(?: \[\w+\])?$' + r'|^"yield" not allowed outside of a function or lambda(?: \[\w+\])?$') class TypecheckError(RuntimeError): @@ -43,7 +47,7 @@ class TypecheckError(RuntimeError): @dataclass(frozen=True) class Check: - tool: str # ``mypy`` (the host's pinned copy) or ``basedpyright`` (on PATH) + tool: str # ``mypy`` (dokimi's own interpreter, pinned in requirements-test.txt) or ``basedpyright`` (on PATH) python: str | None = None # the repository's interpreter; None means unknown, nothing is checked @@ -174,10 +178,15 @@ def _label(code: str) -> str: _PARSE = """\ import ast, json, sys +def parse(code): + try: + ast.parse(code, type_comments=True) + except TypeError: # Python 3.7: ast.parse has no type_comments keyword (it came in 3.8) + ast.parse(code) out = [] for code in json.load(sys.stdin): try: - ast.parse(code, type_comments=True) + parse(code) out.append(None) except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: out.append((exc.msg or "invalid syntax") if isinstance(exc, SyntaxError) else type(exc).__name__) diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 40d36075d..4108306ef 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -622,6 +622,56 @@ def test_a_fragment_with_a_real_error_still_fails(tmp_path): assert claim["status"] == "fail" +def test_basedpyright_top_level_await_return_and_yield_are_fragments_not_stale(tmp_path, monkeypatch): + # basedpyright words these in its own way (pyright's messages, an optional rule suffix): not mypy's wording + from dokimi.repos.l0 import typecheck as l0_typecheck + + fences = ["await asyncio.sleep(1)", "return 1", "yield 1"] + messages = ['"await" allowed only within async function', '"return" can be used only within a function', + '"yield" not allowed outside of a function or lambda'] + + def fake(argv, cwd, stdin=None): + if stdin is not None: # the fences parse + return _completed(returncode=0, stdout=json.dumps([None] * len(fences))) + if argv[-1] == "--version": + return _completed(returncode=0, stdout="basedpyright 1.31.6\n") + report = {"generalDiagnostics": [ + {"file": f"fence-{i}.py", "severity": "error", "message": message, "rule": "reportGeneralTypeIssues", + "range": {"start": {"line": 0}}} for i, message in enumerate(messages)]} + return _completed(returncode=1, stdout=json.dumps(report), stderr="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 12)) + payload = scan_repo(_tree(tmp_path, "".join(f"{FENCE}python\n{code}\n{FENCE}\n\n" for code in fences)), + parse_manifest(MANIFEST), typecheck=Check("basedpyright", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["unresolved"] * 3 + assert all("longer example" in c["why"] for c in claims) + assert payload["typecheck"]["fail"] == 0 + + +def test_the_fence_parser_runs_where_ast_parse_has_no_type_comments_keyword(monkeypatch): + # Python 3.7's ast.parse takes no type_comments keyword: the fence is parsed without it, not a crash + import ast + import io + + from dokimi.repos.l0 import typecheck as l0_typecheck + + real_parse = ast.parse + + def parse_3_7(source, filename="", mode="exec", *, feature_version=None): + return real_parse(source, filename, mode, feature_version=feature_version) + + monkeypatch.setattr(ast, "parse", parse_3_7) + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(["x: int = 1", "f("]))) + out = io.StringIO() + monkeypatch.setattr(sys, "stdout", out) + exec(compile(l0_typecheck._PARSE, "", "exec"), {"__name__": "__main__"}) + valid, invalid = json.loads(out.getvalue()) + assert valid is None + assert invalid is not None + + def test_the_scan_command_refuses_a_python_env_that_is_a_directory(tmp_path, capsys): # os.access(X_OK) is true for a directory: only a regular file can be the interpreter args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", str(tmp_path)]) diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 144e8484c..8027bdafa 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -506,6 +506,35 @@ def test_a_clean_listed_file_has_no_suppressions(tmp_path): assert result["n_suppressions"] == 0 +@pytest.mark.parametrize("source", [ + 'MESSAGE = "# type: ignore"\n' + CLEAN, + '"""Show the marker:\n\n # mypy: ignore-errors\n x = 1 # type: ignore\n"""\n' + CLEAN, +]) +def test_a_marker_inside_a_string_is_not_a_suppression(tmp_path, source): + # mypy reads comments only: the text of a string or a docstring that shows a marker silences nothing + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_suppressions"] == 0 + + +def test_a_genuine_suppression_after_a_docstring_is_named_on_its_own_line(tmp_path): + source = '"""Show:\n\n # mypy: ignore-errors\n"""\nVALUE: int = 1 # type: ignore\n' + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + assert result["errors"] == ["pkg/api.py:5: mypy is silenced here (fix the code instead)"] + + +def test_a_source_that_does_not_tokenize_still_counts_its_marker(tmp_path): + # no comment stream to read: the marker is found line by line, and the file fails mypy's parse anyway + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "def add(:\n return 1 # type: ignore\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + def test_a_type_error_is_graded_from_any_working_directory(tmp_path, monkeypatch): # mypy prints names relative to the cwd: "workspace/pkg/api.py" matched no listed name, so the error was dropped run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) From 7f5bddec21c813320e1a5b8b12f2419d98279563 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 16:53:35 -0800 Subject: [PATCH 32/43] fix(typecheck): two-line fragment comment; nosec the test's exec of its own parse script Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/l0/typecheck.py | 5 ++--- dokimi/tests/test_repo_l0_typecheck.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index b51494270..bbfa1d32d 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -31,9 +31,8 @@ MAX_WHY = 200 # The oldest repository Python each tool accepts: the pinned mypy 2.x refuses --python-version 3.9 and exits 2. MIN_PYTHON = {"mypy": (3, 10)} -# Errors a fence has only because it is a fragment of a longer example: a name an earlier fence defines, or -# await/return/yield outside a function. Each tool words them its own way (mypy, then basedpyright, which may add -# a rule name). Any other error is real, whatever else the fence has. +# Fragment-only errors (a name an earlier fence defines, await/return/yield outside a function), in mypy's or +# basedpyright's wording; any other error is real. _FRAGMENT = re.compile(r'^(?:Name "[^"]+"|"[^"]+") is not defined(?: \[reportUndefinedVariable\])?$' r'|^"(?:await|return|yield)" outside function$' r'|^"await" allowed only within async function(?: \[\w+\])?$' diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 4108306ef..8bf0c1645 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -666,7 +666,7 @@ def parse_3_7(source, filename="", mode="exec", *, feature_version=None monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(["x: int = 1", "f("]))) out = io.StringIO() monkeypatch.setattr(sys, "stdout", out) - exec(compile(l0_typecheck._PARSE, "", "exec"), {"__name__": "__main__"}) + exec(compile(l0_typecheck._PARSE, "", "exec"), {"__name__": "__main__"}) # nosec B102 - our own constant valid, invalid = json.loads(out.getvalue()) assert valid is None assert invalid is not None From a6340ce42f0ecf9c268e5068c28abd72ddb7bbbd Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 22:28:17 -0800 Subject: [PATCH 33/43] fix(typecheck): match native mypy paths; read details.missing; name the real error - graders: a diagnostic with the native separator (Windows) matches its POSIX listed name - selfcheck: a no-reference missing path is read from details (the grader puts lists there) - l0: the claim names the first real error, not a fragment error from the same fence - repo scan --typecheck: an interpreter failure is no longer labelled --typecheck-tool - l0: the unparsed-fence reason names the parsing interpreter Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo.py | 2 +- dokimi/dokimi/repos/graders/typecheck.py | 18 ++++++++----- dokimi/dokimi/repos/l0/typecheck.py | 5 ++-- dokimi/dokimi/repos/selfcheck.py | 3 ++- dokimi/tests/test_repo_l0_typecheck.py | 30 +++++++++++++++++++--- dokimi/tests/test_repo_typecheck_grader.py | 17 +++++++++++- 6 files changed, 61 insertions(+), 14 deletions(-) diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index ceea4e1b2..f64aa3c08 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -161,7 +161,7 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: try: payload = scan_repo(tree, manifest, static, typecheck=check) except TypecheckError as exc: - print(f"✗ --typecheck-tool {check.tool}: {exc}", file=sys.stderr) + print(f"✗ repo scan --typecheck: {exc}", file=sys.stderr) # the error says whether the tool or the interpreter sys.exit(EXIT_USAGE) if args.out: try: diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 1c2961be7..4dac1241d 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -61,7 +61,13 @@ def task_problems(raw: dict[str, Any]) -> list[str]: return problems -def _split(line: str, listed: set[str] | None, root: str) -> tuple[str, re.Match[str]] | None: +def _name(text: str, sep: str) -> str: + """A diagnostic's file name as the listed names are: POSIX separators, normalised. Only ``sep`` (the native + separator, which mypy prints on Windows) is converted, so a backslash in a POSIX name stays one.""" + return posixpath.normpath(text.replace(sep, "/")) + + +def _split(line: str, listed: set[str] | None, root: str, sep: str = os.sep) -> tuple[str, re.Match[str]] | None: """The file a mypy line is about, and the match of the diagnostic after it; None for a line that is not one. With ``listed``, the file is a listed name, normalised as the names are: a name may hold ": error: " itself, so @@ -72,25 +78,25 @@ def _split(line: str, listed: set[str] | None, root: str) -> tuple[str, re.Match if m is None: return None name = m.group("file") - return posixpath.normpath(name[len(root):] if name.startswith(root) else name), m + return _name(name[len(root):] if name.startswith(root) else name, sep), m text = line[len(root):] if line.startswith(root) else line best: tuple[str, re.Match[str]] | None = None for k, ch in enumerate(text): if ch != ":": continue tail = _TAIL.match(text[k:]) - name = posixpath.normpath(text[:k]) + name = _name(text[:k], sep) if tail is not None and name in listed: best = (name, tail) return best -def _errors(stdout: str, listed: set[str] | None, root: str) -> list[dict[str, Any]]: +def _errors(stdout: str, listed: set[str] | None, root: str, sep: str = os.sep) -> list[dict[str, Any]]: """The error lines of mypy's output on a listed file (any file when ``listed`` is None), in output - order; ``root`` is stripped from the names.""" + order; ``root`` is stripped from the names, and ``sep`` (the separator mypy printed) is read as ``/``.""" out = [] for line in stdout.splitlines(): - hit = _split(line, listed, root) + hit = _split(line, listed, root, sep) if hit is None: continue name, m = hit diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index bbfa1d32d..a93573a1b 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -76,7 +76,7 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: if why is not None: # `f()` is not Python, or too deep for the parser block["syntax"] += 1 scan.add(f, fence.line, "python_fence", _label(code), "unresolved", - f"not valid Python ({why}): not type-checked") + f"not parsed by {check.python or 'dokimi'} ({why}): not type-checked") continue runnable.append((f, fence, code)) if check.python is None: @@ -119,7 +119,8 @@ def _claims(scan: Scan, tool: str, runnable: list[tuple[str, Fence, str]], f"{tool}: {errors[0][1]} (a fragment of a longer example: not type-checked)"[:MAX_WHY]) continue fails += 1 - line, message = errors[0] + # the first real error names the claim: a fragment error of the same fence is only counted + line, message = next(e for e in errors if not _FRAGMENT.match(e[1])) more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" where = f"line {line}: " if line else "" # 0: the tool gave no line, so the claim is the fence itself scan.add(f, fence.line + line, "python_fence", _label(code), "fail", diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index 35b7c92e4..f4cc9aa30 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -152,7 +152,8 @@ def _missing_listed(rd: Path) -> list[str]: """For a typecheck task with no reference the null grade is the only step: a listed path it reports missing fails that step, so no run could pass the task.""" doc, _ = read_grade_json(rd / "grade.json") - missing = doc.get("missing") if isinstance(doc, dict) else None + details = doc.get("details") if isinstance(doc, dict) else None # the grader's lists are under details + missing = details.get("missing") if isinstance(details, dict) else None if not isinstance(missing, list) or not missing: return [] return [f"null: listed path {', '.join(str(p) for p in missing)} is missing; with no reference no run can pass"] diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 8bf0c1645..f8ff0f757 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -84,7 +84,7 @@ def test_a_fence_that_is_not_python_is_unresolved_not_failed(tmp_path): typecheck=Check("mypy", sys.executable)) placeholder = {c["line"]: c for c in _fence_claims(payload)}[15] # opens on line 15 assert placeholder["status"] == "unresolved" - assert "not valid Python" in placeholder["why"] + assert "not parsed by" in placeholder["why"] and "invalid syntax" in placeholder["why"] assert payload["typecheck"]["syntax"] == 1 @@ -92,7 +92,7 @@ def test_without_an_interpreter_every_checkable_fence_is_unknown(tmp_path): payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", None)) fences = _fence_claims(payload) assert {c["status"] for c in fences} == {"unresolved"} - assert all("unknown" in c["why"] for c in fences if "not valid" not in c["why"]) + assert all("unknown" in c["why"] for c in fences if "not parsed by" not in c["why"]) assert payload["typecheck"] == {"tool": "mypy", "tool_version": None, "python": None, "fences": 3, "checked": 0, "fail": 0, "unknown": 2, "syntax": 1} @@ -218,7 +218,22 @@ def broken(check, codes, target=None): with pytest.raises(SystemExit) as exc: args.func(args) assert exc.value.code == 2 - assert "--typecheck-tool mypy: mypy exited 2: broken" in capsys.readouterr().err + assert "repo scan --typecheck: mypy exited 2: broken" in capsys.readouterr().err + + +def test_an_interpreter_that_reports_no_version_is_not_labelled_as_the_tool(tmp_path, capsys, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + def no_version(check): + raise l0_typecheck.TypecheckError("/venv/bin/python did not report its version: ''") + + monkeypatch.setattr(l0_typecheck, "python_version", no_version) + args = _scan_args(tmp_path, "--typecheck", "--python-env", sys.executable) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "did not report its version" in err and "--typecheck-tool" not in err def test_basedpyright_missing_from_path_is_a_usage_error(tmp_path, capsys, monkeypatch): @@ -622,6 +637,15 @@ def test_a_fragment_with_a_real_error_still_fails(tmp_path): assert claim["status"] == "fail" +def test_the_claim_names_the_first_real_error_when_a_fragment_error_comes_first(tmp_path): + # `client` is an earlier fence's name: its error is a fragment, so the real error is the one the claim names + fence = f'{FENCE}python\nclient.decode("{{}}")\nx: int = "a"\n{FENCE}\n' + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + assert "Incompatible types" in claim["why"] and "not defined" not in claim["why"] + + def test_basedpyright_top_level_await_return_and_yield_are_fragments_not_stale(tmp_path, monkeypatch): # basedpyright words these in its own way (pyright's messages, an optional rule suffix): not mypy's wording from dokimi.repos.l0 import typecheck as l0_typecheck diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 8027bdafa..7214b640b 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -544,6 +544,19 @@ def test_a_type_error_is_graded_from_any_working_directory(tmp_path, monkeypatch assert result["n_errors"] == 1 +def test_a_windows_diagnostic_path_matches_the_listed_name(): + # mypy on Windows prints backslash paths; the listed name is POSIX, so the error is still this file's + stdout = "C:\\run\\workspace\\pkg\\api.py:2: error: boom\n" + errors = typecheck._errors(stdout, {"pkg/api.py"}, "C:\\run\\workspace\\", sep="\\") + assert errors == [{"file": "pkg/api.py", "line": 2, "code": "", "message": "boom"}] + + +def test_a_backslash_in_a_posix_name_is_a_literal_character(): + # only the native separator is converted: on POSIX a file may be named with a backslash + errors = typecheck._errors("ws/a\\b.py:1: error: boom\n", {"a\\b.py"}, "ws/", sep="/") + assert [e["file"] for e in errors] == ["a\\b.py"] + + def test_two_listed_names_for_one_file_are_a_task_error(): # "a.py" and "./a.py" are one file: mypy reports it as a duplicate module, so the task could never pass problems = typecheck.task_problems(_task(paths=["pkg/api.py", "./pkg/api.py"])) @@ -560,7 +573,9 @@ def test_a_typecheck_task_without_reference_whose_listed_path_is_missing_is_a_pr raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": PINNED, "paths": ["src/pkg/modle.py"]} rd = tmp_path / "rd" rd.mkdir() - (rd / "grade.json").write_text(json.dumps({"pass": False, "missing": ["src/pkg/modle.py"]}), encoding="utf-8") + # the shape main.grade writes: a list is under details, not at the top level + mets, det = main.split_result({"pass": False, "n_missing": 1, "missing": ["src/pkg/modle.py"]}) + (rd / "grade.json").write_text(json.dumps({"pass": False, "metrics": mets, "details": det}), encoding="utf-8") grade = {"status": "graded", "success": False, "score": None, "score_only": False, "pass": False} problems = selfcheck._verdict_problems(Task("t", raw, bare), "null", grade, rd) assert any("missing" in p and "src/pkg/modle.py" in p for p in problems) From 89887adc2c9a06f64bbae0e0d92ea887b940df71 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 01:33:22 -0800 Subject: [PATCH 34/43] fix(typecheck): run mypy again past a fence that blocks it mypy stops on a blocking error (a top-level yield, break outside a loop), so one such fence left every other fence unresolved and --fail-on-fail passed a stale fence. The fences the blocking error did not name are checked in another run. Signed-off-by: Alexander Rybkin --- dokimi/docs/repo-evaluation-contract.md | 3 +- dokimi/dokimi/repos/l0/typecheck.py | 32 ++++++++++++------- dokimi/tests/test_repo_l0_typecheck.py | 41 ++++++++++++++++++++----- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5c403d0c4..cb03f9d50 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1521,7 +1521,8 @@ grammar of the repository interpreter's version, and mypy runs with that `--python-version` and an explicit empty config, so the subject's and the host's mypy settings do not apply. basedpyright gets the same version as `pythonVersion` in its explicit project file, since `--pythonpath` alone sets the import paths. The tools run on the host through -`dokimi.process`; one run covers every fence of the repository. +`dokimi.process`; one run covers every fence of the repository. When mypy stops on a blocking error, it runs +again on the fences that error did not name, so one fence cannot leave the others unchecked. ```bash python cli.py repo scan --manifest --typecheck --python-env /bin/python diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index a93573a1b..4952d7bd8 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -5,8 +5,9 @@ Each fence that parses becomes a ``python_fence`` claim: ``ok`` when the tool reports no error, ``fail`` (a stale-API candidate) when it does, and ``unresolved`` when it does not parse or no -repository interpreter was given, so nothing was checked. One tool run covers every fence; the -interpreter's site-packages are the pinned dependencies, and imports are not followed past the fence. +repository interpreter was given, so nothing was checked. One tool run covers every fence (mypy runs again +without the fences a blocking error named); the interpreter's site-packages are the pinned dependencies, +and imports are not followed past the fence. """ from __future__ import annotations @@ -90,9 +91,8 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: if _older_than_the_tool(scan, check, runnable, target): block["unknown"] = len(runnable) return block - # no file: usage error. ``blocked``: the tool stopped early, so a fence with no error was not necessarily checked. - found_errors, blocked = _run(check, [code for _, _, code in runnable], target) if runnable else ({}, False) - fails, unchecked = _claims(scan, check.tool, runnable, found_errors, blocked) + found_errors = _run_past_blocking(check, [code for _, _, code in runnable], target) if runnable else {} + fails, unchecked = _claims(scan, check.tool, runnable, found_errors) block["fail"] = fails block["checked"] = len(runnable) - unchecked block["unknown"] += unchecked # "unknown" is every fence that was not checked @@ -100,16 +100,11 @@ def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: def _claims(scan: Scan, tool: str, runnable: list[tuple[str, Fence, str]], - found_errors: dict[int, list[tuple[int, str]]], blocked: bool) -> tuple[int, int]: + found_errors: dict[int, list[tuple[int, str]]]) -> tuple[int, int]: """One claim per runnable fence from the tool's errors; (failing fences, fences not checked).""" fails = unchecked = 0 for i, (f, fence, code) in enumerate(runnable): errors = found_errors.get(i, []) - if not errors and blocked: - unchecked += 1 - scan.add(f, fence.line, "python_fence", _label(code), "unresolved", - f"{tool} stopped before this fence was checked: not type-checked") - continue if not errors: scan.add(f, fence.line, "python_fence", _label(code), "ok", f"{tool}: no error") continue @@ -213,6 +208,21 @@ def _syntax_errors(check: Check, codes: list[str]) -> list[str | None]: return reasons +def _run_past_blocking(check: Check, codes: list[str], + target: tuple[int, int] | None) -> dict[int, list[tuple[int, str]]]: + """Errors per fence index. A run mypy stopped on a blocking error is run again without the fences it named, + so that one fence cannot leave every other fence unchecked; each round drops at least one fence.""" + errors: dict[int, list[tuple[int, str]]] = {} + todo = list(range(len(codes))) + while todo: + found, blocked = _run(check, [codes[i] for i in todo], target) + errors.update((todo[k], found_errors) for k, found_errors in found.items()) + if not blocked: + break + todo = [i for k, i in enumerate(todo) if k not in found] + return errors + + def _run(check: Check, codes: list[str], target: tuple[int, int] | None = None) -> tuple[dict[int, list[tuple[int, str]]], bool]: """Errors per fence index: (line in the fence, message), in tool order; and whether the tool stopped early. diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index f8ff0f757..148c1b0f3 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -403,17 +403,42 @@ def fake(argv, cwd, stdin=None): monkeypatch.setattr(l0_typecheck, "_run_tool", fake) -def test_a_tool_that_stops_on_a_blocking_error_leaves_the_unchecked_fences_unresolved(tmp_path, monkeypatch): - report = json.dumps({"file": "fence-0.py", "line": 1, "severity": "error", "message": "blocking"}) - _fake_tools(monkeypatch, _completed(returncode=2, stdout=report + "\n"), []) +def test_a_tool_that_stops_on_a_blocking_error_is_run_again_on_the_fences_it_did_not_name(tmp_path, monkeypatch): + blocking = json.dumps({"file": "fence-0.py", "line": 1, "severity": "error", "message": "blocking"}) + stale = json.dumps({"file": "fence-0.py", "line": 1, "severity": "error", "message": "stale"}) + runs: list = [] + _fake_tools(monkeypatch, None, runs) + from dokimi.repos.l0 import typecheck as l0_typecheck + + fake = l0_typecheck._run_tool + + def per_run(argv, cwd, stdin=None): + done = fake(argv, cwd, stdin) + if done is not None: + return done + mypy_runs = [a for a in runs if "--output" in a] + report = blocking if len(mypy_runs) == 1 else stale # the second run has only the other fence, as fence-0 + return _completed(returncode=2 if len(mypy_runs) == 1 else 1, stdout=report + "\n") + + monkeypatch.setattr(l0_typecheck, "_run_tool", per_run) agents = f"{FENCE}python\nx: int = 1\n{FENCE}\n\n{FENCE}python\ny: int = 2\n{FENCE}\n" payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) claims = _fence_claims(payload) - assert [c["status"] for c in claims] == ["fail", "unresolved"] - assert "blocking" in claims[0]["why"] - assert "stopped before this fence was checked" in claims[1]["why"] # no error reported is not a pass - assert payload["typecheck"]["checked"] == 1 - assert payload["typecheck"]["unknown"] == 1 + assert [c["status"] for c in claims] == ["fail", "fail"] + assert "blocking" in claims[0]["why"] and "stale" in claims[1]["why"] + assert [a[-1] for a in runs if "--output" in a] == ["fence-1.py", "fence-0.py"] + assert payload["typecheck"]["checked"] == 2 + + +def test_a_blocking_fragment_fence_does_not_hide_a_stale_fence(tmp_path): + # a top-level yield stops mypy and is itself a fragment: the stale fence must still be named, not unresolved + agents = (f"{FENCE}python\nconn = open('/dev/null')\nyield conn\n{FENCE}\n\n" + f"{FENCE}python\nimport json\njson.dumps(1).nosuch()\n{FENCE}\n") + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["unresolved", "fail"] + assert "nosuch" in claims[1]["why"] + assert payload["typecheck"]["fail"] == 1 def test_a_tool_that_exits_two_with_no_fence_report_is_an_error_not_a_pass(monkeypatch): From 5d603ab8cb9fb1268ebdddd5fc58f24ddc014866 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 01:34:28 -0800 Subject: [PATCH 35/43] fix(typecheck): read a top-level async with/for as a fragment A top-level async with or async for lacks the same context as a top-level await, yet both tools' messages for it were read as a stale API, so --fail-on-fail failed correct async snippets. Signed-off-by: Alexander Rybkin --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/l0/typecheck.py | 6 +++-- dokimi/tests/test_repo_l0_typecheck.py | 32 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index cb03f9d50..1aca21846 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1502,7 +1502,7 @@ claim: - `unresolved`: the fence is not valid Python (a placeholder such as `f()`), is too deeply nested for the parser, or no `--python-env` was given, so nothing was checked. It is also unresolved when its only errors are missing context (a - name an earlier fence defines, `await`/`return` outside a function), or when the + name an earlier fence defines, `await`/`return`, `async with`/`async for` outside a function), or when the repository is older than the tool checks (mypy: Python 3.10). The reason says which. `--python-env PYTHON` names the repository's interpreter, a virtualenv with its diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index 4952d7bd8..cc0d4a03f 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -32,10 +32,12 @@ MAX_WHY = 200 # The oldest repository Python each tool accepts: the pinned mypy 2.x refuses --python-version 3.9 and exits 2. MIN_PYTHON = {"mypy": (3, 10)} -# Fragment-only errors (a name an earlier fence defines, await/return/yield outside a function), in mypy's or -# basedpyright's wording; any other error is real. +# Fragment-only errors (a name an earlier fence defines, await/return/yield or async with/for outside a function), +# in mypy's or basedpyright's wording; any other error is real. _FRAGMENT = re.compile(r'^(?:Name "[^"]+"|"[^"]+") is not defined(?: \[reportUndefinedVariable\])?$' r'|^"(?:await|return|yield)" outside function$' + r'|^"async (?:with|for)" outside async function$' + r'|^Use of "async" not allowed outside of async function(?: \[\w+\])?$' r'|^"await" allowed only within async function(?: \[\w+\])?$' r'|^"return" can be used only within a function(?: \[\w+\])?$' r'|^"yield" not allowed outside of a function or lambda(?: \[\w+\])?$') diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 148c1b0f3..9e51ced1f 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -662,6 +662,38 @@ def test_a_fragment_with_a_real_error_still_fails(tmp_path): assert claim["status"] == "fail" +def test_a_top_level_async_with_or_async_for_is_a_fragment_not_stale(tmp_path): + # the same missing context as a top-level await: the snippet's body runs inside an async function + fences = ["import asyncio\nasync with asyncio.timeout(1):\n await asyncio.sleep(0)", + "async def gen():\n yield 1\nasync for x in gen():\n print(x)", + "import asyncio\nimport json\nasync with asyncio.timeout(1):\n json.dumps(1).nosuch()"] + claims = _fence_claims(scan_repo(_tree(tmp_path, "".join(f"{FENCE}python\n{c}\n{FENCE}\n\n" for c in fences)), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved", "unresolved", "fail"] + assert "nosuch" in claims[2]["why"] + + +def test_basedpyright_top_level_async_with_is_a_fragment_not_stale(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + def fake(argv, cwd, stdin=None): + if stdin is not None: + return _completed(returncode=0, stdout=json.dumps([None])) + if argv[-1] == "--version": + return _completed(returncode=0, stdout="basedpyright 1.40.2\n") + report = {"generalDiagnostics": [ + {"file": "fence-0.py", "severity": "error", "message": 'Use of "async" not allowed outside of async function', + "rule": "reportGeneralTypeIssues", "range": {"start": {"line": 1}}}]} + return _completed(returncode=1, stdout=json.dumps(report), stderr="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 12)) + fence = f"{FENCE}python\nimport asyncio\nasync with asyncio.timeout(1):\n pass\n{FENCE}\n" + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("basedpyright", sys.executable))) + assert claim["status"] == "unresolved" + + def test_the_claim_names_the_first_real_error_when_a_fragment_error_comes_first(tmp_path): # `client` is an earlier fence's name: its error is a fragment, so the real error is the one the claim names fence = f'{FENCE}python\nclient.decode("{{}}")\nx: int = "a"\n{FENCE}\n' From a740ae2face97abfdc52d2ef325e26ed2e98eb41 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 01:37:19 -0800 Subject: [PATCH 36/43] docs(contract): any undefined name in a fence is unresolved Section 8.2 said only a name an earlier fence defines is missing context, but every undefined name is, an omitted import included (the PR's own fragment test pins `asyncio` with no import). Say so, and that a removed builtin such as `unicode` is unresolved rather than a fail. Signed-off-by: Alexander Rybkin --- dokimi/docs/repo-evaluation-contract.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 1aca21846..d9534ce0b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1501,8 +1501,9 @@ claim: guidance file, and its `why` names the first error (`mypy: line N: message`). - `unresolved`: the fence is not valid Python (a placeholder such as `f()`), is too deeply nested for the parser, or no `--python-env` was given, so nothing - was checked. It is also unresolved when its only errors are missing context (a - name an earlier fence defines, `await`/`return`, `async with`/`async for` outside a function), or when the + was checked. It is also unresolved when its only errors are missing context (any + undefined name, as from an earlier fence or an omitted import, so a removed builtin such as + `unicode` is unresolved too; `await`/`return`, `async with`/`async for` outside a function), or when the repository is older than the tool checks (mypy: Python 3.10). The reason says which. `--python-env PYTHON` names the repository's interpreter, a virtualenv with its From 7c75b0023fb21a8c3d4770cd4a4c9da8251dd871 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 01:37:45 -0800 Subject: [PATCH 37/43] fix(typecheck): check unannotated function bodies in the grader The grader's mypy run lacked --check-untyped-defs, which the L0 check passes, so an agent could make a failing grade pass by deleting a function's annotations instead of fixing its body. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/repos/graders/typecheck.py | 1 + dokimi/tests/test_repo_typecheck_grader.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 4dac1241d..b23ac7bb8 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -202,6 +202,7 @@ def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | Non # plain lines: the image's MYPY_FORCE_COLOR would put escapes around the file name and the marker stdout, stderr, status = api.run(["--no-incremental", "--no-color-output", f"--cache-dir={cache}", f"--config-file={config}", "--explicit-package-bases", + "--check-untyped-defs", # as L0: stripped annotations hide nothing "--show-absolute-path", # cwd-independent names, matched to the listed ones *[str(workspace / p) for p in present]]) finally: diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 7214b640b..d7fc51bd6 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -499,6 +499,15 @@ def test_a_stub_in_the_workspace_that_hides_a_module_fails_the_grade(tmp_path): assert result["n_suppressions"] == 1 +def test_an_error_in_an_unannotated_function_still_fails_the_grade(tmp_path): + # without --check-untyped-defs mypy skips the body: deleting the annotations would turn a fail into a pass + source = "def add(a, b):\n n: int = 'x'\n return a + n\n" + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + def test_a_clean_listed_file_has_no_suppressions(tmp_path): run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) result = typecheck.grade(_task(), tmp_path, "", run) From 9b0bb4118f504bf5f2b3b280ab59e5cab78851b6 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 01:38:31 -0800 Subject: [PATCH 38/43] fix(typecheck): count no_type_check in a listed file as a suppression @no_type_check (under any alias) makes mypy skip the function, so a listed file with a type error graded pass with no suppression counted. Signed-off-by: Alexander Rybkin --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/graders/typecheck.py | 12 ++++++++++++ dokimi/tests/test_repo_typecheck_grader.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index d9534ce0b..915ff9ba4 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -663,7 +663,7 @@ failed the task, and on a task with `score` that is a failed run with score `dokimi:typecheck` checks the listed files only. Its pass is "no error on a listed file" (an error in a module a listed file imports belongs to that module, -not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), is a suppression, and the grade counts it in `n_suppressions`. A listed path +not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment or the name `no_type_check` in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), is a suppression, and the grade counts it in `n_suppressions`. A listed path reached through a symbolic link is missing, not read. A symbolic link in the workspace whose target lies outside it refuses the whole grade (a grader error, not a pass), listed or not, whenever mypy runs, that is when some listed file is present: mypy follows an import through a link, so an unlisted module could supply what a diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index b23ac7bb8..0f0a9a811 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -36,6 +36,7 @@ # A comment that silences mypy: ``# type: ignore`` on a line, or a mypy directive comment (ignore-errors, ...). # A listed file with one is not graded clean, whatever mypy then reports. _SILENCE = re.compile(r"#\s*(?:type:\s*ignore|mypy:)") +_NO_TYPE_CHECK = "no_type_check" # typing's decorator: mypy skips the body it wraps, under any import alias def _safe_path(path: object) -> bool: @@ -133,6 +134,16 @@ def _comments(text: str) -> list[tuple[int, str]]: return list(enumerate(text.splitlines(), 1)) +def _no_type_check(text: str) -> list[int]: + """The lines where a source names ``no_type_check`` as code (not in a comment or a string); line by line for a + source the tokenizer cannot read.""" + try: + tokens = tokenize.generate_tokens(io.StringIO(text).readline) + return [tok.start[0] for tok in tokens if tok.type == tokenize.NAME and tok.string == _NO_TYPE_CHECK] + except (tokenize.TokenError, SyntaxError): + return [n for n, line in enumerate(text.splitlines(), 1) if _NO_TYPE_CHECK in line] + + def _silenced(workspace: Path, listed: list[str]) -> list[str]: """Where the workspace silences mypy: ``file:line`` for a comment, the path for a stub (``.pyi`` shadows a module). @@ -141,6 +152,7 @@ def _silenced(workspace: Path, listed: list[str]) -> list[str]: for p in listed: text = (workspace / p).read_text(encoding="utf-8", errors="replace") found += [f"{p}:{n}" for n, comment in _comments(text) if _SILENCE.search(comment)] + found += [f"{p}:{n}" for n in _no_type_check(text)] for dirpath, _, filenames in os.walk(workspace): found += [Path(dirpath, name).relative_to(workspace).as_posix() for name in filenames if name.endswith(".pyi")] return found diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index d7fc51bd6..e545f6c76 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -508,6 +508,25 @@ def test_an_error_in_an_unannotated_function_still_fails_the_grade(tmp_path): assert result["n_errors"] == 1 +@pytest.mark.parametrize("source", [ + "from typing import no_type_check\n\n@no_type_check\n" + WRONG, + "import typing\n\n@typing.no_type_check\n" + WRONG, + "from typing import no_type_check as ok\n\n@ok\n" + WRONG, + "from typing import no_type_check\n\n@no_type_check\ndef add(:\n return 1\n", # not tokenizable: by line +]) +def test_no_type_check_in_a_listed_file_is_a_suppression(tmp_path, source): + # mypy skips a function under @no_type_check: the error is hidden, not fixed + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] >= 1 + + +def test_no_type_check_in_a_string_is_not_a_suppression(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": 'NOTE = "no_type_check"\n' + CLEAN}, _task()) + assert typecheck.grade(_task(), tmp_path, "", run)["pass"] is True + + def test_a_clean_listed_file_has_no_suppressions(tmp_path): run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) result = typecheck.grade(_task(), tmp_path, "", run) From 132a88afe446d09b216d5471cefae78bbfccadfc Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 01:39:18 -0800 Subject: [PATCH 39/43] fix(typecheck): refuse a workspace module named as a stdlib one The workspace is first on mypy_path, so a json.py replaces json exactly as the refused json.pyi does: a __getattr__ -> Any there hid the listed file's error and the grade passed. It now counts as a suppression. Signed-off-by: Alexander Rybkin --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/graders/typecheck.py | 10 ++++++++-- dokimi/tests/test_repo_typecheck_grader.py | 23 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 915ff9ba4..d8afc41dc 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -663,7 +663,7 @@ failed the task, and on a task with `score` that is a failed run with score `dokimi:typecheck` checks the listed files only. Its pass is "no error on a listed file" (an error in a module a listed file imports belongs to that module, -not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment or the name `no_type_check` in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), is a suppression, and the grade counts it in `n_suppressions`. A listed path +not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment or the name `no_type_check` in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), or a top-level module or package named as a standard-library one (`json.py`, the same replacement), is a suppression, and the grade counts it in `n_suppressions`. A listed path reached through a symbolic link is missing, not read. A symbolic link in the workspace whose target lies outside it refuses the whole grade (a grader error, not a pass), listed or not, whenever mypy runs, that is when some listed file is present: mypy follows an import through a link, so an unlisted module could supply what a diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index 0f0a9a811..f92f65d5c 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -145,9 +145,10 @@ def _no_type_check(text: str) -> list[int]: def _silenced(workspace: Path, listed: list[str]) -> list[str]: - """Where the workspace silences mypy: ``file:line`` for a comment, the path for a stub (``.pyi`` shadows a module). + """Where the workspace silences mypy: ``file:line`` for a comment, the path for a stub (``.pyi`` shadows a module) + or for a top-level module named as a standard-library one (``json.py``), on the workspace or a source root. - A ``.pyi`` anywhere is refused: mypy_path puts the workspace first, so ``__getattr__ -> Any`` hides every attribute.""" + Both are refused: mypy_path puts the workspace first, so ``__getattr__ -> Any`` hides every attribute.""" found = [] for p in listed: text = (workspace / p).read_text(encoding="utf-8", errors="replace") @@ -155,6 +156,11 @@ def _silenced(workspace: Path, listed: list[str]) -> list[str]: found += [f"{p}:{n}" for n in _no_type_check(text)] for dirpath, _, filenames in os.walk(workspace): found += [Path(dirpath, name).relative_to(workspace).as_posix() for name in filenames if name.endswith(".pyi")] + for base in (workspace, *_package_roots(workspace, listed)): + for entry in sorted(base.iterdir()) if base.is_dir() else []: + module = entry.stem if entry.suffix == ".py" else entry.name if (entry / "__init__.py").is_file() else "" + if module in sys.stdlib_module_names: + found.append(entry.relative_to(workspace).as_posix()) return found diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index e545f6c76..5515fd3c1 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -527,6 +527,29 @@ def test_no_type_check_in_a_string_is_not_a_suppression(tmp_path): assert typecheck.grade(_task(), tmp_path, "", run)["pass"] is True +@pytest.mark.parametrize("shadow", [ + {"json.py": "from typing import Any\n\ndef __getattr__(name: str) -> Any: ...\n"}, + {"json.py": "def nosuch() -> None: ...\n"}, + {"json/__init__.py": "def nosuch() -> None: ...\n"}, +]) +def test_a_workspace_module_named_as_a_standard_library_one_fails_the_grade(tmp_path, shadow): + # the .py twin of a json.pyi: the workspace is first on mypy_path, so it replaces json and hides json.nosuch() + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "import json\n\nprint(json.nosuch())\n", **shadow}, + _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_a_standard_library_name_on_a_source_root_fails_the_grade(tmp_path): + files = {"src/pkg/__init__.py": "", "src/pkg/api.py": "import json\n\nprint(json.nosuch())\n", + "src/json.py": "def nosuch() -> None: ...\n"} + run = _run(tmp_path, files, _task(paths=["src/pkg/api.py"])) + result = typecheck.grade(_task(paths=["src/pkg/api.py"]), tmp_path, "", run) + assert result["pass"] is False + assert "src/json.py: mypy is silenced here (fix the code instead)" in result["errors"] + + def test_a_clean_listed_file_has_no_suppressions(tmp_path): run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) result = typecheck.grade(_task(), tmp_path, "", run) From 1ad296f5782143fb5a289ee73ce278308bb2ae37 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 01:40:08 -0800 Subject: [PATCH 40/43] fix(selfcheck): report a no-reference typecheck task that cannot pass With no reference/ the null grade is the only step. A suppression the workspace already has (a repository .pyi) fails every grade, so even a run that repairs the code cannot pass, yet tasks check reported ok. Signed-off-by: Alexander Rybkin --- .../20261031_120000_napetrov_repo-typecheck.md | 3 ++- dokimi/dokimi/repos/selfcheck.py | 15 ++++++++++----- dokimi/tests/test_repo_typecheck_grader.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md index 9b5411766..21d88383e 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md @@ -7,5 +7,6 @@ stale-API candidate, a fence that does not parse or has no interpreter is `unresolved`. The report gains a `typecheck` block, `null` without the flag. See `docs/repo-evaluation-contract.md` §4.5 and §8.2. - A `dokimi:typecheck` task needs no `reference/` directory: with one, its files are the repair the grade - reads; without one, the listed files are graded as they are and the self-check runs the null step only. + reads; without one, the listed files are graded as they are and the self-check runs the null step only, + reporting a listed path that is missing or a suppression the workspace already has (no run could pass). - A tool report without its diagnostic fields (for example `{}`) is a tool failure, not a clean run. diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index f4cc9aa30..8aff71098 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -149,14 +149,19 @@ def _prepare(ctx: EvalContext, task: Task, rd: Path) -> None: def _missing_listed(rd: Path) -> list[str]: - """For a typecheck task with no reference the null grade is the only step: a listed path it reports missing - fails that step, so no run could pass the task.""" + """For a typecheck task with no reference the null grade is the only step: a listed path it reports missing, + or a suppression already in the workspace, fails that step, so no run that repairs the code could pass.""" doc, _ = read_grade_json(rd / "grade.json") details = doc.get("details") if isinstance(doc, dict) else None # the grader's lists are under details + metrics = doc.get("metrics") if isinstance(doc, dict) else None missing = details.get("missing") if isinstance(details, dict) else None - if not isinstance(missing, list) or not missing: - return [] - return [f"null: listed path {', '.join(str(p) for p in missing)} is missing; with no reference no run can pass"] + out = [] + if isinstance(missing, list) and missing: + out.append(f"null: listed path {', '.join(str(p) for p in missing)} is missing; with no reference no run " + "can pass") + if isinstance(metrics, dict) and isinstance(metrics.get("n_suppressions"), int) and metrics["n_suppressions"] > 0: + out.append("null: the untouched workspace already silences mypy; with no reference a code repair cannot pass") + return out def _verdict_problems(task: Task, kind: str, grade: dict[str, Any], rd: Path) -> list[str]: diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 5515fd3c1..28773c471 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -630,3 +630,21 @@ def test_a_typecheck_task_without_reference_whose_listed_path_is_missing_is_a_pr grade = {"status": "graded", "success": False, "score": None, "score_only": False, "pass": False} problems = selfcheck._verdict_problems(Task("t", raw, bare), "null", grade, rd) assert any("missing" in p and "src/pkg/modle.py" in p for p in problems) + + +def test_a_typecheck_task_without_reference_whose_workspace_already_silences_mypy_is_a_problem(tmp_path): + # a repository stub fails every grade, so with no reference even a run that repairs the code cannot pass + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + files = {"pkg/__init__.py": "", "pkg/api.py": CLEAN, "pkg/_ext.pyi": "def f() -> int: ...\n"} + run = _run(tmp_path, files, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["n_errors"] == 0 and result["n_suppressions"] == 1 # repaired, and still failing + mets, det = main.split_result(result) + (run / "grade.json").write_text(json.dumps({"pass": False, "metrics": mets, "details": det}), encoding="utf-8") + grade = {"status": "graded", "success": False, "score": None, "score_only": False, "pass": False} + problems = selfcheck._verdict_problems(Task("t", _task(tool="mypy"), bare), "null", grade, run) + assert any("silences mypy" in p for p in problems) From bc9bc0ed2cd347cbd05678a73baec094dda5204d Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 3 Nov 2026 03:37:43 -0800 Subject: [PATCH 41/43] refactor(repo): move the scan typecheck check and text report to repo_scan.py With main merged in, commands/repo.py reached 823 lines, over the growth ratchet's 800 cap. The two helpers this branch adds move out unchanged; repo.py imports them back. Signed-off-by: Alexander Rybkin --- dokimi/dokimi/commands/repo.py | 56 +----------------------- dokimi/dokimi/commands/repo_scan.py | 66 +++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 55 deletions(-) create mode 100644 dokimi/dokimi/commands/repo_scan.py diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 6e746e7e3..4498f8d0e 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -8,14 +8,13 @@ import argparse import json import math -import os import re -import shutil import sys from pathlib import Path from typing import Any from dokimi.exit_codes import EXIT_USAGE +from dokimi.commands.repo_scan import _print_scan_text, _typecheck_or_exit def _normalize_whitespace(text: str, max_len: int | None = 100) -> str: @@ -124,30 +123,6 @@ def _read_static(path: str) -> Any: sys.exit(EXIT_USAGE) -def _typecheck_or_exit(args: argparse.Namespace): - """The ``--typecheck`` check, or None; ``--python-env`` needs it and must name an interpreter.""" - from dokimi.repos.l0 import Check - - if not args.typecheck: - if args.python_env is not None: - print("✗ --python-env needs --typecheck", file=sys.stderr) - sys.exit(EXIT_USAGE) - if args.typecheck_tool is not None: - print("✗ --typecheck-tool needs --typecheck", file=sys.stderr) - sys.exit(EXIT_USAGE) - return None - tool = args.typecheck_tool or "mypy" - # a directory is X_OK too: only a regular file can be the interpreter - if args.python_env is not None and not (os.path.isfile(args.python_env) and os.access(args.python_env, os.X_OK)): - print(f"✗ --python-env {args.python_env}: not an executable interpreter", file=sys.stderr) - sys.exit(EXIT_USAGE) - if tool != "mypy" and shutil.which(tool) is None: - print(f"✗ --typecheck-tool {tool}: not on PATH", file=sys.stderr) - sys.exit(EXIT_USAGE) - python = os.path.abspath(args.python_env) if args.python_env is not None else None # the tool runs elsewhere - return Check(tool, python) - - def cmd_repo_scan(args: argparse.Namespace) -> None: """L0: check every mechanically checkable claim in the guidance against the same tree.""" from dokimi.artifacts import save_artifact, stamp @@ -182,35 +157,6 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: sys.exit(1) -def _print_scan_text(manifest, payload: dict, out: str | None) -> None: - """The text report of ``repo scan``: the counts, the warnings, the typecheck and static lines, the FAIL lines.""" - c = payload["counts"] - detected = " (detected)" if payload["adapters_detected"] else "" - print(f"{manifest.id}: {len(payload['files'])} guidance files in {payload['repo']['tree']}; " - f"adapters: {', '.join(payload['adapters']) or 'none'}{detected}") - print(f" claims: {c['total']} (ok {c['ok']}, unresolved {c['unresolved']}, fail {c['fail']})") - for warning in payload["warnings"]: - print(f" warning: {warning}") - if payload["incomplete"]: - print(f" incomplete: {'; '.join(payload['incomplete_reasons'])} (repo scan --fail-on-fail exits 1)") - tc = payload["typecheck"] - if tc is not None: - where = tc["python"] or "no --python-env: unknown" - print(f" typecheck {tc['tool']} {tc['tool_version'] or ''} ({where}): {tc['fences']} python fences, " - f"{tc['checked']} checked, {tc['fail']} fail, {tc['unknown']} unknown, {tc['syntax']} not Python") - static_info = payload["static"] - if static_info is not None and not static_info["errors"]: - r = payload["by_source"]["repository"] - print(f" repository static: {r['total']} claims, {r['fail']} fail; " - f"{static_info['superseded']} framework claims replaced") - for claim in payload["claims"]: - if claim["status"].split("_", 1)[0] == "fail": - print(f" FAIL {claim['file']}:{claim['line']} [{claim.get('kind')}] " - f"{str(claim.get('claim'))[:90]} -- {claim.get('why')}") - if out: - print(f"wrote {out}") - - def _scan_gate(payload: dict, fail_on_fail: bool) -> None: """--fail-on-fail: exit 1 on a failed claim or an incomplete scan, saying on stderr what the scan missed.""" if not fail_on_fail or not (payload["counts"]["fail"] or payload["incomplete"]): diff --git a/dokimi/dokimi/commands/repo_scan.py b/dokimi/dokimi/commands/repo_scan.py new file mode 100644 index 000000000..9078e3a26 --- /dev/null +++ b/dokimi/dokimi/commands/repo_scan.py @@ -0,0 +1,66 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo scan` helpers kept out of commands/repo.py, which is at its size cap: the `--typecheck` check and the text report.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import sys + +from dokimi.exit_codes import EXIT_USAGE + + +def _typecheck_or_exit(args: argparse.Namespace): + """The ``--typecheck`` check, or None; ``--python-env`` needs it and must name an interpreter.""" + from dokimi.repos.l0 import Check + + if not args.typecheck: + if args.python_env is not None: + print("✗ --python-env needs --typecheck", file=sys.stderr) + sys.exit(EXIT_USAGE) + if args.typecheck_tool is not None: + print("✗ --typecheck-tool needs --typecheck", file=sys.stderr) + sys.exit(EXIT_USAGE) + return None + tool = args.typecheck_tool or "mypy" + # a directory is X_OK too: only a regular file can be the interpreter + if args.python_env is not None and not (os.path.isfile(args.python_env) and os.access(args.python_env, os.X_OK)): + print(f"✗ --python-env {args.python_env}: not an executable interpreter", file=sys.stderr) + sys.exit(EXIT_USAGE) + if tool != "mypy" and shutil.which(tool) is None: + print(f"✗ --typecheck-tool {tool}: not on PATH", file=sys.stderr) + sys.exit(EXIT_USAGE) + python = os.path.abspath(args.python_env) if args.python_env is not None else None # the tool runs elsewhere + return Check(tool, python) + + +def _print_scan_text(manifest, payload: dict, out: str | None) -> None: + """The text report of ``repo scan``: the counts, the warnings, the typecheck and static lines, the FAIL lines.""" + c = payload["counts"] + detected = " (detected)" if payload["adapters_detected"] else "" + print(f"{manifest.id}: {len(payload['files'])} guidance files in {payload['repo']['tree']}; " + f"adapters: {', '.join(payload['adapters']) or 'none'}{detected}") + print(f" claims: {c['total']} (ok {c['ok']}, unresolved {c['unresolved']}, fail {c['fail']})") + for warning in payload["warnings"]: + print(f" warning: {warning}") + if payload["incomplete"]: + print(f" incomplete: {'; '.join(payload['incomplete_reasons'])} (repo scan --fail-on-fail exits 1)") + tc = payload["typecheck"] + if tc is not None: + where = tc["python"] or "no --python-env: unknown" + print(f" typecheck {tc['tool']} {tc['tool_version'] or ''} ({where}): {tc['fences']} python fences, " + f"{tc['checked']} checked, {tc['fail']} fail, {tc['unknown']} unknown, {tc['syntax']} not Python") + static_info = payload["static"] + if static_info is not None and not static_info["errors"]: + r = payload["by_source"]["repository"] + print(f" repository static: {r['total']} claims, {r['fail']} fail; " + f"{static_info['superseded']} framework claims replaced") + for claim in payload["claims"]: + if claim["status"].split("_", 1)[0] == "fail": + print(f" FAIL {claim['file']}:{claim['line']} [{claim.get('kind')}] " + f"{str(claim.get('claim'))[:90]} -- {claim.get('why')}") + if out: + print(f"wrote {out}") From 9fcddfb83ee29682ed8262c9040565eb1e09f86e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 3 Nov 2026 13:57:34 +0000 Subject: [PATCH 42/43] test(repo): pin that only python-tagged fences become claims Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_l0_typecheck.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 9e51ced1f..192c4bc5b 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -523,6 +523,14 @@ def test_a_linked_guidance_file_has_its_fences_checked_and_counted_once(tmp_path assert payload["typecheck"]["fences"] == 1 +def test_only_python_tagged_fences_become_claims(tmp_path): + bad = "import json\n\njson.nosuch()\n" + agents = f"{FENCE}python\n{bad}{FENCE}\n\n{FENCE}bash\n{bad}{FENCE}\n\n{FENCE}json\n{bad}{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + assert payload["typecheck"]["fences"] == 1 + assert [c["status"] for c in _fence_claims(payload)] == ["fail"] + + def test_a_fence_cannot_import_another_fence_as_a_module(tmp_path): agents = f"{FENCE}python\nvalue: int = 1\n{FENCE}\n\n{FENCE}python\nimport fence_0\nprint(fence_0.value)\n{FENCE}\n" payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) From f7cb3c50359401fa37783b005b40918c19a05bc9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 3 Nov 2026 08:06:38 -0800 Subject: [PATCH 43/43] fix(typecheck): a missing module is unresolved; more fragments; installed-package shadows refused MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - L0: a fence whose only errors are a module the repository environment lacks is unresolved, not fail; a real error in the same fence still fails it. The contract §8.2 says so. - L0: break, continue, yield from outside a loop or function, and nonlocal at module level are fragments (mypy's wording, and basedpyright's where known), not stale calls. - grader: a workspace .py or package named as a module the image's site-packages provides is a suppression, as a standard-library name already was (contract §4.5). - tests: a real-mypy top-level return, the Python 3.10 boundary, the two grader tests that cannot run on Windows are skipped there. Signed-off-by: Nikolay Petrov --- ...20261031_120000_napetrov_repo-typecheck.md | 3 + dokimi/docs/repo-evaluation-contract.md | 12 ++- dokimi/dokimi/repos/graders/typecheck.py | 23 +++++- dokimi/dokimi/repos/l0/typecheck.py | 43 ++++++++--- dokimi/tests/test_repo_l0_typecheck.py | 76 ++++++++++++++++++- dokimi/tests/test_repo_typecheck_grader.py | 22 ++++++ 6 files changed, 160 insertions(+), 19 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md index 21d88383e..ab84b88e1 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md @@ -10,3 +10,6 @@ reads; without one, the listed files are graded as they are and the self-check runs the null step only, reporting a listed path that is missing or a suppression the workspace already has (no run could pass). - A tool report without its diagnostic fields (for example `{}`) is a tool failure, not a clean run. +- A fence that imports a module the environment lacks is `unresolved`, not `fail`. A fence whose only errors are + a statement that needs a function or loop around it (`break`, `continue`, `yield from`, `nonlocal`) is unresolved too. +- The grader refuses a workspace module named as one the image's site-packages provides, as it does a standard-library name. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 58da0a1e1..051b269d5 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -663,7 +663,7 @@ failed the task, and on a task with `score` that is a failed run with score `dokimi:typecheck` checks the listed files only. Its pass is "no error on a listed file" (an error in a module a listed file imports belongs to that module, -not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment or the name `no_type_check` in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), or a top-level module or package named as a standard-library one (`json.py`, the same replacement), is a suppression, and the grade counts it in `n_suppressions`. A listed path +not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment or the name `no_type_check` in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), or a top-level module or package named as a standard-library one or one the image's site-packages provides (`json.py`, `requests/__init__.py`, the same replacement), is a suppression, and the grade counts it in `n_suppressions`. A listed path reached through a symbolic link is missing, not read. A symbolic link in the workspace whose target lies outside it refuses the whole grade (a grader error, not a pass), listed or not, whenever mypy runs, that is when some listed file is present: mypy follows an import through a link, so an unlisted module could supply what a @@ -1549,15 +1549,19 @@ claim: is too deeply nested for the parser, or no `--python-env` was given, so nothing was checked. It is also unresolved when its only errors are missing context (any undefined name, as from an earlier fence or an omitted import, so a removed builtin such as - `unicode` is unresolved too; `await`/`return`, `async with`/`async for` outside a function), or when the + `unicode` is unresolved too; `await`/`return`/`yield`/`yield from`, `break`/`continue` outside a loop, + `async with`/`async for` outside a function, `nonlocal` at module level), or a module the + environment does not have (see below), or when the repository is older than the tool checks (mypy: Python 3.10). The reason says which. `--python-env PYTHON` names the repository's interpreter, a virtualenv with its pinned dependencies installed (and the repository itself, if its fences import it). The fences are checked against that site-packages with imports followed silently (`--follow-imports=silent`), so only errors in the fence count. A -module the environment does not have is a `fail`: a fence that imports an -uninstalled package is a fence the pinned environment cannot run. `pip install +module the environment does not have is `unresolved`, not `fail`: the fence's own +import fails, so nothing it says about that module can be checked, and a fence that +imports an uninstalled package is not a stale-API candidate. A real error in the same fence still +fails it. `pip install dokimi` does not install mypy: the tested pin is `requirements-test.txt` (`mypy==2.4.0`), and a missing mypy exits 2. basedpyright must be on `PATH` and is recorded with the version it reports, not pinned by dokimi. The report's diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py index f92f65d5c..7cac0c445 100644 --- a/dokimi/dokimi/repos/graders/typecheck.py +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -15,12 +15,14 @@ from __future__ import annotations +import importlib.util import io import os import posixpath import re import shutil import sys +import sysconfig import tempfile import tokenize from pathlib import Path @@ -144,9 +146,26 @@ def _no_type_check(text: str) -> list[int]: return [n for n, line in enumerate(text.splitlines(), 1) if _NO_TYPE_CHECK in line] +def _image_provides(name: str) -> bool: + """True when the image's site-packages provides the top-level module ``name``. The module is found, not imported; + a dotted name is not looked up (find_spec would import its parent).""" + if not name.isidentifier(): + return False + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): # a module in sys.modules without a spec + return False + if spec is None: + return False + found = [spec.origin or "", *(spec.submodule_search_locations or [])] + site = [Path(sysconfig.get_path(key)).resolve() for key in ("purelib", "platlib")] + return any(os.path.isabs(loc) and Path(loc).resolve().is_relative_to(d) for loc in found for d in site) + + def _silenced(workspace: Path, listed: list[str]) -> list[str]: """Where the workspace silences mypy: ``file:line`` for a comment, the path for a stub (``.pyi`` shadows a module) - or for a top-level module named as a standard-library one (``json.py``), on the workspace or a source root. + or for a top-level module named as a standard-library one or one the image's site-packages provides + (``json.py``, ``requests/__init__.py``), on the workspace or a source root. Both are refused: mypy_path puts the workspace first, so ``__getattr__ -> Any`` hides every attribute.""" found = [] @@ -159,7 +178,7 @@ def _silenced(workspace: Path, listed: list[str]) -> list[str]: for base in (workspace, *_package_roots(workspace, listed)): for entry in sorted(base.iterdir()) if base.is_dir() else []: module = entry.stem if entry.suffix == ".py" else entry.name if (entry / "__init__.py").is_file() else "" - if module in sys.stdlib_module_names: + if module in sys.stdlib_module_names or _image_provides(module): found.append(entry.relative_to(workspace).as_posix()) return found diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py index cc0d4a03f..d5bef0e87 100644 --- a/dokimi/dokimi/repos/l0/typecheck.py +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -32,15 +32,21 @@ MAX_WHY = 200 # The oldest repository Python each tool accepts: the pinned mypy 2.x refuses --python-version 3.9 and exits 2. MIN_PYTHON = {"mypy": (3, 10)} -# Fragment-only errors (a name an earlier fence defines, await/return/yield or async with/for outside a function), -# in mypy's or basedpyright's wording; any other error is real. +# Fragment-only errors (a name an earlier fence defines, a statement that needs a function, loop or async body +# around it), in mypy's or basedpyright's wording; any other error is real. _FRAGMENT = re.compile(r'^(?:Name "[^"]+"|"[^"]+") is not defined(?: \[reportUndefinedVariable\])?$' - r'|^"(?:await|return|yield)" outside function$' + r'|^"(?:await|return|yield|yield from|continue|break)" outside (?:function|loop)$' r'|^"async (?:with|for)" outside async function$' + r'|^(?:Nonlocal|nonlocal) declaration not allowed at module level(?: \[\w+\])?$' r'|^Use of "async" not allowed outside of async function(?: \[\w+\])?$' r'|^"await" allowed only within async function(?: \[\w+\])?$' r'|^"return" can be used only within a function(?: \[\w+\])?$' - r'|^"yield" not allowed outside of a function or lambda(?: \[\w+\])?$') + r'|^"(?:break|continue)" can be used only within a loop(?: \[\w+\])?$' + r'|^"(?:yield|yield from)" not allowed outside of a function or lambda(?: \[\w+\])?$') +# A module the repository environment does not have: the fence's own import fails, so nothing it says about that +# module can be checked (in mypy's or basedpyright's wording). A missing module is not a stale call. +_MISSING_MODULE = re.compile(r'^Cannot find implementation or library stub for module named "[^"]+"$' + r'|^Import "[^"]+" could not be resolved(?: \[\w+\])?$') class TypecheckError(RuntimeError): @@ -110,14 +116,14 @@ def _claims(scan: Scan, tool: str, runnable: list[tuple[str, Fence, str]], if not errors: scan.add(f, fence.line, "python_fence", _label(code), "ok", f"{tool}: no error") continue - if _is_fragment(errors): + if _is_unchecked(errors): unchecked += 1 scan.add(f, fence.line, "python_fence", _label(code), "unresolved", - f"{tool}: {errors[0][1]} (a fragment of a longer example: not type-checked)"[:MAX_WHY]) + f"{tool}: {errors[0][1]} ({_unchecked_reason(errors)}: not type-checked)"[:MAX_WHY]) continue fails += 1 - # the first real error names the claim: a fragment error of the same fence is only counted - line, message = next(e for e in errors if not _FRAGMENT.match(e[1])) + # the first real error names the claim: a fragment or missing-module error of the same fence is only counted + line, message = next(e for e in errors if not _missing_context(e[1])) more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" where = f"line {line}: " if line else "" # 0: the tool gave no line, so the claim is the fence itself scan.add(f, fence.line + line, "python_fence", _label(code), "fail", @@ -138,9 +144,24 @@ def _older_than_the_tool(scan: Scan, check: Check, runnable: list[tuple[str, Fen return True -def _is_fragment(errors: list[tuple[int, str]]) -> bool: - """True when every error of a fence is one its missing context causes (see ``_FRAGMENT``).""" - return all(_FRAGMENT.match(message) for _, message in errors) +def _missing_context(message: str) -> bool: + """True for an error the fence's missing context or a module the environment lacks causes (see the patterns).""" + return _FRAGMENT.match(message) is not None or _MISSING_MODULE.match(message) is not None + + +def _is_unchecked(errors: list[tuple[int, str]]) -> bool: + """True when every error of a fence is one of those (see ``_missing_context``): nothing real was found.""" + return all(_missing_context(message) for _, message in errors) + + +def _unchecked_reason(errors: list[tuple[int, str]]) -> str: + """Why the fence is not checked, for the claim: the fence is a fragment, or a module is missing (or both).""" + missing = [m for _, m in errors if _MISSING_MODULE.match(m)] + if len(missing) == len(errors): + return "a module the repository environment lacks" + if missing: + return "a fragment of a longer example, or a module the repository environment lacks" + return "a fragment of a longer example" def tool_version(check: Check) -> str: diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py index 192c4bc5b..0e44bb39e 100644 --- a/dokimi/tests/test_repo_l0_typecheck.py +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -336,7 +336,8 @@ def test_a_host_mypypath_cannot_supply_a_module_the_pinned_environment_lacks(tmp monkeypatch.setenv("MYPYPATH", str(extra)) tree = _tree(tmp_path, f"{FENCE}python\nimport ghost_pkg_mod\nprint(ghost_pkg_mod.run())\n{FENCE}\n") [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) - assert claim["status"] == "fail" + # the module is missing from the pinned environment: unresolved, and never checked against the host's copy + assert claim["status"] == "unresolved" @pytest.mark.skipif(sys.platform == "win32", reason="the stand-in interpreter is a POSIX shell script") @@ -534,7 +535,8 @@ def test_only_python_tagged_fences_become_claims(tmp_path): def test_a_fence_cannot_import_another_fence_as_a_module(tmp_path): agents = f"{FENCE}python\nvalue: int = 1\n{FENCE}\n\n{FENCE}python\nimport fence_0\nprint(fence_0.value)\n{FENCE}\n" payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) - assert [c["status"] for c in _fence_claims(payload)] == ["ok", "fail"] + # fence_0 is not a module of the environment: the import is unresolved, and the fence is not passed by another's file + assert [c["status"] for c in _fence_claims(payload)] == ["ok", "unresolved"] def test_mypy_is_given_the_repository_interpreter(monkeypatch): @@ -739,6 +741,76 @@ def fake(argv, cwd, stdin=None): assert payload["typecheck"]["fail"] == 0 +@pytest.mark.parametrize("statement", ["yield from range(3)", "continue", "break", "nonlocal x"]) +def test_a_statement_that_needs_a_function_or_loop_around_it_is_a_fragment_not_stale(tmp_path, statement): + # mypy's wording for each outside a function or loop: the fence is a piece of a longer example + claims = _fence_claims(scan_repo(_tree(tmp_path, f"{FENCE}python\n{statement}\n{FENCE}\n"), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved"] + assert "longer example" in claims[0]["why"] + + +def test_a_top_level_return_is_a_fragment_not_stale(tmp_path): + # the real-mypy twin of the fake basedpyright case below: "return" outside function stays unresolved + claims = _fence_claims(scan_repo(_tree(tmp_path, f"{FENCE}python\nreturn 1\n{FENCE}\n"), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved"] + + +def test_a_repository_on_python_3_10_is_checked_not_unresolved(tmp_path, monkeypatch): + # mypy 2.x accepts --python-version 3.10: only 3.9 and older are unresolved, 3.10 is checked + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 10)) + agents = f"{FENCE}python\nimport json\nprint(json.dumps(1))\n{FENCE}\n\n{FENCE}python\nx: int = 'a'\n{FENCE}\n" + claims = _fence_claims(scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["ok", "fail"] + + +def test_a_module_the_repository_environment_lacks_is_unresolved_not_stale(tmp_path): + claims = _fence_claims(scan_repo(_tree(tmp_path, f"{FENCE}python\nimport nosuchmodule_dokimi\n{FENCE}\n"), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved"] + assert "not type-checked" in claims[0]["why"] and "lacks" in claims[0]["why"] + + +def test_a_missing_module_with_a_real_error_fails_on_the_real_error(tmp_path): + fence = f"{FENCE}python\nimport nosuchmodule_dokimi\nx: int = 'a'\n{FENCE}\n" + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + assert "line 2" in claim["why"] and "nosuchmodule_dokimi" not in claim["why"] + + +def test_basedpyright_break_continue_nonlocal_and_a_missing_import_are_not_stale(tmp_path, monkeypatch): + # pyright's own messages (not run here: basedpyright is not installed in CI's test image) + from dokimi.repos.l0 import typecheck as l0_typecheck + + fences = ["break", "continue", "yield from range(3)", "nonlocal x", "import nosuchmodule_dokimi"] + messages = ['"break" can be used only within a loop', '"continue" can be used only within a loop', + '"yield from" not allowed outside of a function or lambda', + "Nonlocal declaration not allowed at module level", + 'Import "nosuchmodule_dokimi" could not be resolved'] + + def fake(argv, cwd, stdin=None): + if stdin is not None: # the fences parse + return _completed(returncode=0, stdout=json.dumps([None] * len(fences))) + if argv[-1] == "--version": + return _completed(returncode=0, stdout="basedpyright 1.31.6\n") + report = {"generalDiagnostics": [ + {"file": f"fence-{i}.py", "severity": "error", "message": message, "rule": "reportGeneralTypeIssues", + "range": {"start": {"line": 0}}} for i, message in enumerate(messages)]} + return _completed(returncode=1, stdout=json.dumps(report), stderr="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 12)) + payload = scan_repo(_tree(tmp_path, "".join(f"{FENCE}python\n{code}\n{FENCE}\n\n" for code in fences)), + parse_manifest(MANIFEST), typecheck=Check("basedpyright", sys.executable)) + assert [c["status"] for c in _fence_claims(payload)] == ["unresolved"] * len(fences) + assert payload["typecheck"]["fail"] == 0 + + def test_the_fence_parser_runs_where_ast_parse_has_no_type_comments_keyword(monkeypatch): # Python 3.7's ast.parse takes no type_comments keyword: the fence is parsed without it, not a crash import ast diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 28773c471..7a24b6c9b 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -221,6 +221,7 @@ def test_a_src_layout_package_resolves_when_both_files_are_listed(tmp_path): assert result["pass"] is True, result["errors"] +@pytest.mark.skipif(sys.platform == "win32", reason="a directory named src:lib cannot be created on Windows") @pytest.mark.parametrize("sep", [",", ":"]) def test_a_source_root_with_a_mypy_path_delimiter_is_a_task_error(tmp_path, sep): files = {f"src{sep}lib/pkg/__init__.py": "", f"src{sep}lib/pkg/x.py": "val: int = 1\n", @@ -393,6 +394,7 @@ def test_the_oracle_refuses_an_empty_reference_directory_created_through_a_link( assert not (outside / "sub").exists() +@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not keep the 0700 mode this checks") def test_the_oracle_refuses_a_linked_workspace_even_with_an_empty_reference(tmp_path): # no entry in reference/, so the walk checks nothing; copytree still copies the mode onto the linked root outside = tmp_path / "outside" @@ -541,6 +543,26 @@ def test_a_workspace_module_named_as_a_standard_library_one_fails_the_grade(tmp_ assert result["n_suppressions"] == 1 +@pytest.mark.parametrize("shadow", [ + {"mypy.py": "def __getattr__(name: str) -> int: ...\n"}, + {"mypy/__init__.py": "def __getattr__(name: str) -> int: ...\n"}, +]) +def test_a_workspace_module_named_as_an_installed_package_fails_the_grade(tmp_path, shadow): + # mypy is in the image's site-packages: a workspace mypy.py would stand in for it, as json.py does for json + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "import mypy\n\nprint(mypy.nosuch)\n", **shadow}, + _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_image_provides_finds_an_installed_module_and_not_a_workspace_name(): + assert typecheck._image_provides("mypy") is True + assert typecheck._image_provides("pkg") is False + assert typecheck._image_provides("not-an-identifier") is False + assert typecheck._image_provides("mypy.version") is False # a dotted name is not looked up + + def test_a_standard_library_name_on_a_source_root_fails_the_grade(tmp_path): files = {"src/pkg/__init__.py": "", "src/pkg/api.py": "import json\n\nprint(json.nosuch())\n", "src/json.py": "def nosuch() -> None: ...\n"}