diff --git a/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md new file mode 100644 index 000000000..ab84b88e1 --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_repo-typecheck.md @@ -0,0 +1,15 @@ +### Added + +- **`dokimi:typecheck` built-in grader and `repo scan --typecheck` (Python fences).** The grader type-checks the + task's listed files with the image's pinned mypy and passes when none has an error (`tool`, `version`, `paths` + in `task.json`; a mismatched version is a grader failure). `repo scan --typecheck` (with `--typecheck-tool mypy|basedpyright`) checks + each Python fence of the guidance against `--python-env`'s site-packages: a fence with an error is a `fail` + stale-API candidate, a fence that does not parse or has no interpreter is `unresolved`. The report gains a + `typecheck` block, `null` without the flag. See `docs/repo-evaluation-contract.md` §4.5 and §8.2. +- A `dokimi:typecheck` task needs no `reference/` directory: with one, its files are the repair the grade + reads; without one, the listed files are graded as they are and the self-check runs the null step only, + reporting a listed path that is missing or a suppression the workspace already has (no run could pass). +- A tool report without its diagnostic fields (for example `{}`) is a tool failure, not a clean run. +- A fence that imports a module the environment lacks is `unresolved`, not `fail`. A fence whose only errors are + a statement that needs a function or loop around it (`break`, `continue`, `yield from`, `nonlocal`) is unresolved too. +- The grader refuses a workspace module named as one the image's site-packages provides, as it does a standard-library name. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 9f226c781..2e828f25c 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -12152,6 +12152,22 @@ "required": false, "type": null }, + { + "action": "_StoreAction", + "choices": null, + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--python-env", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, @@ -12167,6 +12183,41 @@ "position": null, "required": false, "type": null + }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--typecheck", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, + { + "action": "_StoreAction", + "choices": [ + "'basedpyright'", + "'mypy'" + ], + "const": null, + "default": null, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--typecheck-tool", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null } ], "path": "dokimi repo scan", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 1c106e8b9..051b269d5 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -552,7 +552,8 @@ measure the patch's text, such as a guidance line, not the tool. For every task, before any paid run: - the null workspace (prepared, untouched) must fail the gate -- the workspace after `oracle` must pass the gate +- the workspace after `oracle` must pass the gate (a `dokimi:typecheck` task with no `reference/` + has no oracle step: its null step alone must fail, §4.5) - for score-only tasks, the oracle must reach `score.max` on `score.metric` - the agent container must not be able to read any file under `tasks` or `{source_dir}` (§1.1) @@ -601,7 +602,8 @@ repository's checks with its own without knowing their rules. A task whose `grader` is `dokimi:` is graded by the framework, not by the repository's `grade` and `oracle` entry points; any other name goes to those entry points. Such a task is data only: its directory holds -`task.json`, the patch and the JSON files the grader reads. The `dokimi:` +`task.json`, the patch and the JSON files the grader reads (and, for +`dokimi:typecheck`, an optional `reference/` directory, §4.5 below). The `dokimi:` prefix is the framework's: an unknown `dokimi:` name fails `tasks check` and the run's grade, and is never handed to the repository. @@ -611,7 +613,9 @@ framework's grader code is mounted read-only at `/eval/graders` (never into the agent's container) and runs as `python3 -I -B /eval/graders/main.py grade|oracle /eval/run`, where `` is the task's directory under `{eval_dir}`. It uses the -standard library only and the image's `python3` (3.8 or newer). `-I` keeps +standard library only and the image's `python3` (3.8 or newer), except +`dokimi:typecheck`, which imports the image's mypy (the pinned one, on Python +3.10 or newer). `-I` keeps `PYTHONPATH` from `image.env` and the workspace off `sys.path`; `-B` prevents writing `.pyc` files to the read-only mount. An image with no `python3` on `PATH` is a grader failure that says so: the grader exits 127 and its @@ -625,7 +629,8 @@ other exit 127 is reported as a plain `grade exited 127`. `pass: null` with `errors`. A metric the task's `gate` names that the grader did not produce is `null`. - `oracle` writes the reference answer to `answer.txt` and leaves the - workspace alone. + workspace alone, except `dokimi:typecheck`, whose `oracle` copies `reference/` + over the workspace (see its paragraph below). A review task with `pass: null` is scored, not gated: give it `score`. A `gate` on its metrics is not applied (`success` is null), and without `score` the run @@ -654,6 +659,28 @@ failed the task, and on a task with `score` that is a failed run with score |---|---|---|---|---| | `dokimi:review_seeds` | `manifest.json` (`manifest`); oracle answer `oracle.json` (`oracle_answer`), a list of findings | `null`; for a manifest with no seeds (a clean-diff control), `true` when no finding is `blocker` or `major` | `n_findings`, `n_severe`, `n_decoys`, `unmatched`, `recall` (`null` with no seeds) | `seeds_hit`, `decoys_flagged`, `recall_by_class` (`"hit/total"`) | | `dokimi:review_replay` | `gold.json` (`gold`); `task.json` `window`, default 5 | `null` (score-only) | `n_findings`, `n_severe`, `n_threads`, `recall` (= `recall_all`), `recall_addressed`, `precision_proxy` | `threads_hit`, `matches`, `recall_by_kind`, `decoys_flagged` (always `[]`) | +| `dokimi:typecheck` | none: the task's own fields `tool` (`mypy`), `version` (the image's mypy, exactly), `paths` (workspace-relative files) | `true` when no error is reported on a listed file, every listed file exists, and nothing silences mypy, else `false` | `n_errors`, `n_missing`, `n_suppressions`, `n_files` | `errors` (first 50, `file:line: message [code]`, and each silenced spot), `missing`, `tool`, `tool_version` | + +`dokimi:typecheck` checks the listed files only. Its pass is "no error on a +listed file" (an error in a module a listed file imports belongs to that module, +not to the listed file), and a listed file that is missing fails. Silencing mypy fails too: a `# type: ignore` or `# mypy:` comment or the name `no_type_check` in a listed file, or a `.pyi` anywhere in the workspace (it would replace the module it names), or a top-level module or package named as a standard-library one or one the image's site-packages provides (`json.py`, `requests/__init__.py`, the same replacement), is a suppression, and the grade counts it in `n_suppressions`. A listed path +reached through a symbolic link is missing, not read. A symbolic link in the workspace whose target +lies outside it refuses the whole grade (a grader error, not a pass), listed or not, whenever mypy runs, that is +when some listed file is present: mypy follows an import through a link, so an unlisted module could supply what a +listed file imports. When every listed file is missing, mypy does not run, and the grade is a failure with +`n_missing` set, not a grader error, even if a link lies outside. The exception +is a blocking error (a syntax error, a duplicate module) that stops mypy before +it reaches the listed files: then that error is the grade's, and the grade fails. +`reference/` is optional. The task's `oracle` copies its `reference/` directory, when it has one, over the +workspace, so a reference repair is what gets graded; its answer is empty. Without `reference/` the grade +reads the listed files as they are, and the self-check (§4.3) runs the null step only: the untouched +listed files must fail, since there is no reference repair for the oracle step to pass. +It runs mypy in-process in the grading container, so the image's `python3` +must import the pinned mypy: a `version` other than the installed one is a +grader failure, not a fail. It is not "no new errors against the base": the +grader is stdlib-only and cannot run `git` to read the base, and it does not +start basedpyright (the L0 check, §8.2, runs mypy or basedpyright on the host, as `--typecheck-tool` selects). A repository that +needs that comparison writes it in its own `grade` entry point. `manifest.json` is `{"seeds": [{id, class, file, lines, kw}], "decoys": [{id, file, lines, kw}]}`; `file` `"*"` matches any file, and `kw` is a @@ -1507,6 +1534,55 @@ oneDAL they tracked agent cost or error over 39 modules only, which cannot separate structure from question design. Whether the index is current is not checked: `not_observed` says so. +### 8.2 Python fences (`repo scan --typecheck`) + +`--typecheck` type-checks every fence tagged +`python`, `py` or `python3` in the guidance files, with `--typecheck-tool mypy|basedpyright` +(default mypy). Each fence that parses is one +`python_fence` claim, so the report and `--fail-on-fail` treat it like any other +claim: + +- `ok`: the tool reports no error in the fence. +- `fail`: a stale-API candidate. The claim's line is the error's line in the + guidance file, and its `why` names the first error (`mypy: line N: message`). +- `unresolved`: the fence is not valid Python (a placeholder such as `f()`), + is too deeply nested for the parser, or no `--python-env` was given, so nothing + was checked. It is also unresolved when its only errors are missing context (any + undefined name, as from an earlier fence or an omitted import, so a removed builtin such as + `unicode` is unresolved too; `await`/`return`/`yield`/`yield from`, `break`/`continue` outside a loop, + `async with`/`async for` outside a function, `nonlocal` at module level), or a module the + environment does not have (see below), or when the + repository is older than the tool checks (mypy: Python 3.10). The reason says which. + +`--python-env PYTHON` names the repository's interpreter, a virtualenv with its +pinned dependencies installed (and the repository itself, if its fences import +it). The fences are checked against that site-packages with imports followed +silently (`--follow-imports=silent`), so only errors in the fence count. A +module the environment does not have is `unresolved`, not `fail`: the fence's own +import fails, so nothing it says about that module can be checked, and a fence that +imports an uninstalled package is not a stale-API candidate. A real error in the same fence still +fails it. `pip install +dokimi` does not install mypy: the tested pin is `requirements-test.txt` +(`mypy==2.4.0`), and a missing mypy exits 2. basedpyright must be on `PATH` and +is recorded with the version it reports, not pinned by dokimi. The report's +`typecheck` block names the tool, its version, the interpreter (`null`: unknown) and the counts +(`fences`, `checked`, `fail`, `unknown`, `syntax`); it is `null` without the +flag. `--python-env` and `--typecheck-tool` need `--typecheck`. The fences are parsed with the +grammar of the repository interpreter's version, and mypy runs with that +`--python-version` and an explicit empty config, so the subject's and the host's +mypy settings do not apply. basedpyright gets the same version as `pythonVersion` +in its explicit project file, since `--pythonpath` alone sets the import paths. The tools run on the host through +`dokimi.process`; one run covers every fence of the repository. When mypy stops on a blocking error, it runs +again on the fences that error did not name, so one fence cannot leave the others unchecked. + +```bash +python cli.py repo scan --manifest --typecheck --python-env /bin/python +``` + +A guidance note under test can be scored the same way as any other content: +feed the note as a `patch:` arm (see §3) and grade the arm's workspace with +the `dokimi:typecheck` grader above. + ## 9. Running `repo tasks check` and `repo run` need docker. Both export `eval_dir` from the diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index 33b49321b..4498f8d0e 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -14,6 +14,7 @@ from typing import Any from dokimi.exit_codes import EXIT_USAGE +from dokimi.commands.repo_scan import _print_scan_text, _typecheck_or_exit def _normalize_whitespace(text: str, max_len: int | None = 100) -> str: @@ -126,11 +127,17 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: """L0: check every mechanically checkable claim in the guidance against the same tree.""" from dokimi.artifacts import save_artifact, stamp from dokimi.repos.l0 import scan_repo + from dokimi.repos.l0.typecheck import TypecheckError _, tree = _source_tree(args) manifest = _manifest_or_exit(tree, args, guidance_default=True, also_usage=(OSError,)) static = _read_static(args.static) if args.static else None - payload = scan_repo(tree, manifest, static) + check = _typecheck_or_exit(args) + try: + payload = scan_repo(tree, manifest, static, typecheck=check) + except TypecheckError as exc: + print(f"✗ repo scan --typecheck: {exc}", file=sys.stderr) # the error says whether the tool or the interpreter + sys.exit(EXIT_USAGE) if args.out: try: save_artifact("repo_readiness", payload, Path(args.out)) @@ -144,25 +151,7 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: if args.json: print(json.dumps(stamp("repo_readiness", dict(payload)), indent=2)) else: - c = payload["counts"] - detected = " (detected)" if payload["adapters_detected"] else "" - print(f"{manifest.id}: {len(payload['files'])} guidance files in {payload['repo']['tree']}; " - f"adapters: {', '.join(payload['adapters']) or 'none'}{detected}") - print(f" claims: {c['total']} (ok {c['ok']}, unresolved {c['unresolved']}, fail {c['fail']})") - for warning in payload["warnings"]: - print(f" warning: {warning}") - if payload["incomplete"]: - print(f" incomplete: {'; '.join(payload['incomplete_reasons'])} (repo scan --fail-on-fail exits 1)") - if static_info is not None and not static_errors: - r = payload["by_source"]["repository"] - print(f" repository static: {r['total']} claims, {r['fail']} fail; " - f"{static_info['superseded']} framework claims replaced") - for claim in payload["claims"]: - if claim["status"].split("_", 1)[0] == "fail": - print(f" FAIL {claim['file']}:{claim['line']} [{claim.get('kind')}] " - f"{str(claim.get('claim'))[:90]} -- {claim.get('why')}") - if args.out: - print(f"wrote {args.out}") + _print_scan_text(manifest, payload, args.out) _scan_gate(payload, args.fail_on_fail) # runs first: an invalid --static still gets the incomplete reason if static_errors: sys.exit(1) @@ -665,6 +654,17 @@ def register(sub, positive_int) -> None: "the report, its claims replacing the framework's on the same file, line and text. " "A file that cannot be read or is not JSON exits 2; a schema-invalid document is " "reported and makes the command exit 1") + scan_p.add_argument("--typecheck", action="store_true", default=False, + help="Type-check each Python fence of the guidance (§8.2): one claim per fence, fail when the " + "tool reports an error. The tool is mypy unless --typecheck-tool names basedpyright (on PATH). " + "mypy is not installed by pip install dokimi: the tested pin is requirements-test.txt " + "(mypy==2.4.0); a missing mypy exits 2. Without --python-env the fences are unknown (unresolved), " + "not checked") + scan_p.add_argument("--typecheck-tool", default=None, dest="typecheck_tool", choices=("mypy", "basedpyright"), + help="The tool --typecheck runs (default mypy). Needs --typecheck") + scan_p.add_argument("--python-env", default=None, dest="python_env", metavar="PYTHON", + help="The repository's interpreter (a venv with its pinned dependencies installed); the fences " + "are checked against its site-packages. Needs --typecheck") scan_p.add_argument("--out", default=None, help="Write the repo_readiness.v1 report to this file") scan_p.add_argument("--json", action="store_true", help="Print the full report as JSON") scan_p.add_argument("--fail-on-fail", action="store_true", dest="fail_on_fail", diff --git a/dokimi/dokimi/commands/repo_scan.py b/dokimi/dokimi/commands/repo_scan.py new file mode 100644 index 000000000..9078e3a26 --- /dev/null +++ b/dokimi/dokimi/commands/repo_scan.py @@ -0,0 +1,66 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo scan` helpers kept out of commands/repo.py, which is at its size cap: the `--typecheck` check and the text report.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import sys + +from dokimi.exit_codes import EXIT_USAGE + + +def _typecheck_or_exit(args: argparse.Namespace): + """The ``--typecheck`` check, or None; ``--python-env`` needs it and must name an interpreter.""" + from dokimi.repos.l0 import Check + + if not args.typecheck: + if args.python_env is not None: + print("✗ --python-env needs --typecheck", file=sys.stderr) + sys.exit(EXIT_USAGE) + if args.typecheck_tool is not None: + print("✗ --typecheck-tool needs --typecheck", file=sys.stderr) + sys.exit(EXIT_USAGE) + return None + tool = args.typecheck_tool or "mypy" + # a directory is X_OK too: only a regular file can be the interpreter + if args.python_env is not None and not (os.path.isfile(args.python_env) and os.access(args.python_env, os.X_OK)): + print(f"✗ --python-env {args.python_env}: not an executable interpreter", file=sys.stderr) + sys.exit(EXIT_USAGE) + if tool != "mypy" and shutil.which(tool) is None: + print(f"✗ --typecheck-tool {tool}: not on PATH", file=sys.stderr) + sys.exit(EXIT_USAGE) + python = os.path.abspath(args.python_env) if args.python_env is not None else None # the tool runs elsewhere + return Check(tool, python) + + +def _print_scan_text(manifest, payload: dict, out: str | None) -> None: + """The text report of ``repo scan``: the counts, the warnings, the typecheck and static lines, the FAIL lines.""" + c = payload["counts"] + detected = " (detected)" if payload["adapters_detected"] else "" + print(f"{manifest.id}: {len(payload['files'])} guidance files in {payload['repo']['tree']}; " + f"adapters: {', '.join(payload['adapters']) or 'none'}{detected}") + print(f" claims: {c['total']} (ok {c['ok']}, unresolved {c['unresolved']}, fail {c['fail']})") + for warning in payload["warnings"]: + print(f" warning: {warning}") + if payload["incomplete"]: + print(f" incomplete: {'; '.join(payload['incomplete_reasons'])} (repo scan --fail-on-fail exits 1)") + tc = payload["typecheck"] + if tc is not None: + where = tc["python"] or "no --python-env: unknown" + print(f" typecheck {tc['tool']} {tc['tool_version'] or ''} ({where}): {tc['fences']} python fences, " + f"{tc['checked']} checked, {tc['fail']} fail, {tc['unknown']} unknown, {tc['syntax']} not Python") + static_info = payload["static"] + if static_info is not None and not static_info["errors"]: + r = payload["by_source"]["repository"] + print(f" repository static: {r['total']} claims, {r['fail']} fail; " + f"{static_info['superseded']} framework claims replaced") + for claim in payload["claims"]: + if claim["status"].split("_", 1)[0] == "fail": + print(f" FAIL {claim['file']}:{claim['line']} [{claim.get('kind')}] " + f"{str(claim.get('claim'))[:90]} -- {claim.get('why')}") + if out: + print(f"wrote {out}") diff --git a/dokimi/dokimi/repos/graders/__init__.py b/dokimi/dokimi/repos/graders/__init__.py index 373cc322f..5f0f800b6 100644 --- a/dokimi/dokimi/repos/graders/__init__.py +++ b/dokimi/dokimi/repos/graders/__init__.py @@ -15,12 +15,15 @@ from pathlib import Path -from .review import GRADERS +from . import typecheck +from .review import GRADERS as REVIEW_GRADERS PREFIX = "dokimi:" -NAMES = tuple(GRADERS) +NAMES = (*REVIEW_GRADERS, *typecheck.GRADERS) # the data file each built-in grader reads next to task.json: the task key that renames it, and its default DATA_FILES = {"review_seeds": ("manifest", "manifest.json"), "review_replay": ("gold", "gold.json")} +# graders whose options are the task's own fields, checked by ``tasks check`` instead of a data file +TASK_PROBLEMS = {"typecheck": typecheck.task_problems} HOST_DIR = Path(__file__).resolve().parent ENTRY = "main.py" diff --git a/dokimi/dokimi/repos/graders/main.py b/dokimi/dokimi/repos/graders/main.py index 7d5536444..e8d6d23fe 100644 --- a/dokimi/dokimi/repos/graders/main.py +++ b/dokimi/dokimi/repos/graders/main.py @@ -28,15 +28,16 @@ from pathlib import Path if __package__: - from . import review + from . import review, typecheck else: # a script under -I: its own directory is not on sys.path # By name, not `import review`: dependency hygiene (deptry) reads a bare import as a third-party package. sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) review = importlib.import_module("review") + typecheck = importlib.import_module("typecheck") -GRADERS = dict(review.GRADERS) -ORACLES = dict(review.ORACLES) -VERSIONS = {name: review.VERSION for name in review.GRADERS} +GRADERS = {**review.GRADERS, **typecheck.GRADERS} +ORACLES = {**review.ORACLES, **typecheck.ORACLES} +VERSIONS = {**{name: review.VERSION for name in review.GRADERS}, **{name: typecheck.VERSION for name in typecheck.GRADERS}} RUN_DIR = "/eval/run" @@ -76,7 +77,7 @@ def grade(name, task_dir, run_dir): try: answer = run_dir / "answer.txt" text = answer.read_text(encoding="utf-8") if answer.exists() else "" - g = GRADERS[name](t, task_dir, text) + g = GRADERS[name](t, task_dir, text, run_dir) out["metrics"], out["details"] = split_result(g) for k in gate_names(t): # a gate metric the grader did not reach is unmeasured, not failed out["metrics"].setdefault(k, None) @@ -92,7 +93,7 @@ def grade(name, task_dir, run_dir): def oracle(name, task_dir, run_dir): """Write the reference answer to ``run_dir/answer.txt``; returns it.""" - text = ORACLES[name](_read_task(run_dir), task_dir) + text = ORACLES[name](_read_task(run_dir), task_dir, run_dir) if text: (run_dir / "answer.txt").write_text(text, encoding="utf-8") return text diff --git a/dokimi/dokimi/repos/graders/review.py b/dokimi/dokimi/repos/graders/review.py index 7eeeac56f..d3d2d93ef 100644 --- a/dokimi/dokimi/repos/graders/review.py +++ b/dokimi/dokimi/repos/graders/review.py @@ -181,7 +181,7 @@ def _per_group(entries, key, hit): return {k: f"{a}/{b}" for k, (a, b) in counts.items()} -def grade_seeds(task, task_dir, answer): +def grade_seeds(task, task_dir, answer, run_dir=None): """``review_seeds``: the result dict (``pass`` and flat metrics, lists and objects for details).""" man = _read_json(task_dir, task.get("manifest", "manifest.json")) seeds, decoys = man.get("seeds", []), man.get("decoys", []) @@ -211,7 +211,7 @@ def grade_seeds(task, task_dir, answer): return out -def oracle_seeds(task, task_dir): +def oracle_seeds(task, task_dir, run_dir=None): """The reference answer: ``oracle.json`` (a list of findings) in a fence.""" with open(str(task_dir / task.get("oracle_answer", "oracle.json")), encoding="utf-8") as fh: # ``` can only sit in a JSON string: escape it as oracle_replay does, or it closes the fence @@ -250,7 +250,7 @@ def _match(g, i, f, distance): "message": _cut(f.get("message")), "distance": distance} -def grade_replay(task, task_dir, answer): +def grade_replay(task, task_dir, answer, run_dir=None): """``review_replay``: score-only, ``recall`` over the human threads.""" gold = _read_json(task_dir, task.get("gold", "gold.json")) findings = findings_of(answer) @@ -276,7 +276,7 @@ def grade_replay(task, task_dir, answer): "precision_proxy": _ratio(near, len(findings)), "decoys_flagged": []} -def oracle_replay(task, task_dir): +def oracle_replay(task, task_dir, run_dir=None): """The reference answer: one finding per gold thread, at its anchor.""" gold = _read_json(task_dir, task.get("gold", "gold.json")) ans = [{"file": g["file"], "line": g["line"], "severity": "minor", "rule": g.get("kind", ""), diff --git a/dokimi/dokimi/repos/graders/typecheck.py b/dokimi/dokimi/repos/graders/typecheck.py new file mode 100644 index 000000000..7cac0c445 --- /dev/null +++ b/dokimi/dokimi/repos/graders/typecheck.py @@ -0,0 +1,306 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""``dokimi:typecheck``: the listed files type-check clean under the image's pinned mypy (contract §4.5). + +task.json: ``"tool": "mypy"``, ``"version"`` (exactly the image's mypy version, else a grader +failure) and ``"paths"`` (workspace-relative files). ``pass`` is true when no error is reported +on a listed file, every listed file exists and nothing silences mypy. Errors in other modules are not counted. + +It runs mypy in-process: the grader cannot start a child (the import layers allow spawns only in +``dokimi.process``, which the container cannot import), so basedpyright is not offered here. The +host-side L0 check (``repo scan --typecheck``) runs one, as ``--typecheck-tool`` selects. Standard library only at import +time; mypy is imported by :func:`grade` and must be importable by the image's ``python3``. +""" + +from __future__ import annotations + +import importlib.util +import io +import os +import posixpath +import re +import shutil +import sys +import sysconfig +import tempfile +import tokenize +from pathlib import Path +from typing import Any + +# Bump when a result for the same inputs changes (a metric or ``pass``). +VERSION = "1" +TOOL = "mypy" +REFERENCE_DIR = "reference" # task dir: the files of a reference repair, copied over the workspace by oracle +MAX_ERRORS = 50 # reported in details; the count is in the metrics +_ERROR = re.compile(r"^(?P.+?)(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") +_TAIL = re.compile(r"^(?::(?P\d+))?(?::\d+)?: error: (?P.*?)(?:\s+\[(?P[\w-]+)\])?$") +# A comment that silences mypy: ``# type: ignore`` on a line, or a mypy directive comment (ignore-errors, ...). +# A listed file with one is not graded clean, whatever mypy then reports. +_SILENCE = re.compile(r"#\s*(?:type:\s*ignore|mypy:)") +_NO_TYPE_CHECK = "no_type_check" # typing's decorator: mypy skips the body it wraps, under any import alias + + +def _safe_path(path: object) -> bool: + """A workspace-relative path: no leading slash, no ``..`` segment, not empty, and one line: a line separator in + a name would split it in mypy's output, so the error on that file would not be matched to it.""" + return (isinstance(path, str) and bool(path) and not path.startswith("/") and ".." not in path.split("/") + and path.splitlines() == [path]) + + +def task_problems(raw: dict[str, Any]) -> list[str]: + """Problems with a ``dokimi:typecheck`` task's own fields (checked by ``tasks check``).""" + problems = [] + if raw.get("tool", TOOL) != TOOL: + problems.append(f"tool must be {TOOL!r}: basedpyright starts a process, which the grader does not " + f"(the L0 check runs it), not {raw.get('tool')!r}") + if not isinstance(raw.get("version"), str) or not raw.get("version"): + problems.append('version must be the pinned mypy version, e.g. "2.4.0"') + paths = raw.get("paths") + if not isinstance(paths, list) or not paths or not all(_safe_path(p) for p in paths): + problems.append("paths must be a non-empty list of workspace-relative file paths") + elif len({posixpath.normpath(p) for p in paths}) != len(paths): # "a.py" and "./a.py": one file, a duplicate module + problems.append("paths must be distinct after normalising (\"a.py\" and \"./a.py\" are one file)") + return problems + + +def _name(text: str, sep: str) -> str: + """A diagnostic's file name as the listed names are: POSIX separators, normalised. Only ``sep`` (the native + separator, which mypy prints on Windows) is converted, so a backslash in a POSIX name stays one.""" + return posixpath.normpath(text.replace(sep, "/")) + + +def _split(line: str, listed: set[str] | None, root: str, sep: str = os.sep) -> tuple[str, re.Match[str]] | None: + """The file a mypy line is about, and the match of the diagnostic after it; None for a line that is not one. + + With ``listed``, the file is a listed name, normalised as the names are: a name may hold ": error: " itself, so + the first separator in the line is not always the end of the name. Each separator is tried as the end of the + name, and the last one that gives a listed name is taken (the longest name).""" + if listed is None: + m = _ERROR.match(line) + if m is None: + return None + name = m.group("file") + return _name(name[len(root):] if name.startswith(root) else name, sep), m + text = line[len(root):] if line.startswith(root) else line + best: tuple[str, re.Match[str]] | None = None + for k, ch in enumerate(text): + if ch != ":": + continue + tail = _TAIL.match(text[k:]) + name = _name(text[:k], sep) + if tail is not None and name in listed: + best = (name, tail) + return best + + +def _errors(stdout: str, listed: set[str] | None, root: str, sep: str = os.sep) -> list[dict[str, Any]]: + """The error lines of mypy's output on a listed file (any file when ``listed`` is None), in output + order; ``root`` is stripped from the names, and ``sep`` (the separator mypy printed) is read as ``/``.""" + out = [] + for line in stdout.splitlines(): + hit = _split(line, listed, root, sep) + if hit is None: + continue + name, m = hit + if listed is None or name in listed: + out.append({"file": name, "line": int(m.group("line") or 0), + "code": m.group("code") or "", "message": m.group("message")}) + return out + + +def _package_roots(workspace: Path, paths: list[str]) -> list[Path]: + """For each listed file inside regular packages (``__init__.py``), the directory above its top package. + + A namespace directory (no ``__init__.py``) is not a package root: its files are named from the workspace, as + an import names them, and nothing above them is added.""" + roots: list[Path] = [] + for p in paths: + top = None + d = (workspace / p).parent + while d != workspace and (d / "__init__.py").is_file(): + top = d + d = d.parent + if top is not None and top.parent != workspace and top.parent not in roots: + roots.append(top.parent) + return roots + + +def _comments(text: str) -> list[tuple[int, str]]: + """The comments of a Python source, with their lines: a marker inside a string or a docstring is not one. + A source the tokenizer cannot read is taken line by line, so a marker in it still counts.""" + try: + tokens = tokenize.generate_tokens(io.StringIO(text).readline) + return [(tok.start[0], tok.string) for tok in tokens if tok.type == tokenize.COMMENT] + except (tokenize.TokenError, SyntaxError): + return list(enumerate(text.splitlines(), 1)) + + +def _no_type_check(text: str) -> list[int]: + """The lines where a source names ``no_type_check`` as code (not in a comment or a string); line by line for a + source the tokenizer cannot read.""" + try: + tokens = tokenize.generate_tokens(io.StringIO(text).readline) + return [tok.start[0] for tok in tokens if tok.type == tokenize.NAME and tok.string == _NO_TYPE_CHECK] + except (tokenize.TokenError, SyntaxError): + return [n for n, line in enumerate(text.splitlines(), 1) if _NO_TYPE_CHECK in line] + + +def _image_provides(name: str) -> bool: + """True when the image's site-packages provides the top-level module ``name``. The module is found, not imported; + a dotted name is not looked up (find_spec would import its parent).""" + if not name.isidentifier(): + return False + try: + spec = importlib.util.find_spec(name) + except (ImportError, ValueError): # a module in sys.modules without a spec + return False + if spec is None: + return False + found = [spec.origin or "", *(spec.submodule_search_locations or [])] + site = [Path(sysconfig.get_path(key)).resolve() for key in ("purelib", "platlib")] + return any(os.path.isabs(loc) and Path(loc).resolve().is_relative_to(d) for loc in found for d in site) + + +def _silenced(workspace: Path, listed: list[str]) -> list[str]: + """Where the workspace silences mypy: ``file:line`` for a comment, the path for a stub (``.pyi`` shadows a module) + or for a top-level module named as a standard-library one or one the image's site-packages provides + (``json.py``, ``requests/__init__.py``), on the workspace or a source root. + + Both are refused: mypy_path puts the workspace first, so ``__getattr__ -> Any`` hides every attribute.""" + found = [] + for p in listed: + text = (workspace / p).read_text(encoding="utf-8", errors="replace") + found += [f"{p}:{n}" for n, comment in _comments(text) if _SILENCE.search(comment)] + found += [f"{p}:{n}" for n in _no_type_check(text)] + for dirpath, _, filenames in os.walk(workspace): + found += [Path(dirpath, name).relative_to(workspace).as_posix() for name in filenames if name.endswith(".pyi")] + for base in (workspace, *_package_roots(workspace, listed)): + for entry in sorted(base.iterdir()) if base.is_dir() else []: + module = entry.stem if entry.suffix == ".py" else entry.name if (entry / "__init__.py").is_file() else "" + if module in sys.stdlib_module_names or _image_provides(module): + found.append(entry.relative_to(workspace).as_posix()) + return found + + +def _links_outside(workspace: Path) -> list[str]: + """The symbolic links under the (resolved) workspace, a link to a directory included, whose target lies outside + it. mypy follows an import through a link, so a link to a file this run cannot see (a hidden repair) would be read + as the agent's own code, unlisted modules included. A link whose target stays inside is read as the file it names.""" + found = [] + for dirpath, dirnames, filenames in os.walk(workspace): + for name in (*dirnames, *filenames): + path = Path(dirpath, name) + if path.is_symlink() and not path.resolve().is_relative_to(workspace): + found.append(path.relative_to(workspace).as_posix()) + return found + + +def grade(task: dict[str, Any], task_dir: Path, answer: str, run_dir: Path | None = None) -> dict[str, Any]: + """Type-check the listed files in the workspace of ``run_dir``; the grader's result dict.""" + del task_dir, answer # the answer is not read: the workspace is the subject + problems = task_problems(task) + if problems: + raise ValueError("; ".join(problems)) + import mypy.version + from mypy import api + + if mypy.version.__version__ != task["version"]: + raise ValueError(f"the image's mypy is {mypy.version.__version__}, the task pins {task['version']}") + linked_root = Path(run_dir or "/eval/run") / "workspace" # unresolved: a link at the root must be seen + workspace = linked_root.resolve() # absolute: mypy gets these paths + paths = [posixpath.normpath(p) for p in task["paths"]] + # is_file() and mypy both follow links, and the grading container mounts the hidden reference: a listed path + # with a link on the way (or a linked root) is missing, never read. + missing = [p for p in paths + if _linked_on_the_way(linked_root, linked_root / p) or not (workspace / p).is_file()] + present = [p for p in paths if p not in missing] + errors: list[dict[str, Any]] = [] + if present: + outside = _links_outside(workspace) + if outside: # refused, not missing: the import of an unlisted module through it would be graded + raise ValueError(f"{outside[0]} is a symbolic link to a file outside the workspace; mypy would follow it " + "to content this run cannot see") + cache = tempfile.mkdtemp(prefix="dokimi-mypy-") + neutral = tempfile.mkdtemp(prefix="dokimi-mypy-cfg-") + limit = sys.getrecursionlimit() # mypy's main raises it for the whole process + # mypy reads MYPYPATH ahead of mypy_path, so an externally supplied stub directory could stand in for a + # module the image lacks: it is out of the run, and restored afterwards. + mypypath = os.environ.pop("MYPYPATH", None) + try: + # Neutral config: the subject's mypy.ini, pyproject.toml and the host's user config are not read. + # mypy_path is the workspace and each regular package's source root (src/ for src/pkg/__init__.py). + roots = [*_package_roots(workspace, present), workspace] + bad = [str(r) for r in roots if re.search("[,:]", os.path.splitdrive(str(r))[1])] + if bad: # mypy splits mypy_path on , and :, so such a root would be read as two + raise ValueError(f"a source root has , or : in its name, which mypy_path cannot hold: {bad[0]}") + config = Path(neutral, "mypy.ini") + config.write_text(f"[mypy]\nmypy_path = {':'.join(str(r) for r in roots)}\n", encoding="utf-8") + # plain lines: the image's MYPY_FORCE_COLOR would put escapes around the file name and the marker + stdout, stderr, status = api.run(["--no-incremental", "--no-color-output", f"--cache-dir={cache}", + f"--config-file={config}", "--explicit-package-bases", + "--check-untyped-defs", # as L0: stripped annotations hide nothing + "--show-absolute-path", # cwd-independent names, matched to the listed ones + *[str(workspace / p) for p in present]]) + finally: + sys.setrecursionlimit(limit) + if mypypath is not None: + os.environ["MYPYPATH"] = mypypath + shutil.rmtree(cache, ignore_errors=True) + shutil.rmtree(neutral, ignore_errors=True) + # 2 is also what mypy gives for a blocking source error (a syntax error, a duplicate module): those + # are diagnostics like any other. A 2 with no diagnostic line is mypy itself failing. + if status not in (0, 1) and not any(": error: " in line for line in stdout.splitlines()): + raise RuntimeError(f"mypy exited {status}: {stderr[-300:]}") + root = str(workspace) + os.sep + errors = _errors(stdout, set(present), root) + if status == 2 and not errors: + # mypy stopped at a blocking error in a file it did not reach ("errors prevented further + # checking"): the listed files were not checked, so that error is the grade's + errors = _errors(stdout, None, root) + silenced = _silenced(workspace, present) + shown = [f"{e['file']}:{e['line']}: {e['message']}" + (f" [{e['code']}]" if e["code"] else "") for e in errors] + shown += [f"{s}: mypy is silenced here (fix the code instead)" for s in silenced] + return {"pass": not errors and not missing and not silenced, "n_errors": len(errors), "n_missing": len(missing), + "n_suppressions": len(silenced), "n_files": len(paths), "errors": shown[:MAX_ERRORS], "missing": missing, + "tool": TOOL, "tool_version": mypy.version.__version__} + + +def _linked_on_the_way(workspace: Path, dest: Path) -> bool: + """True when the workspace root, a directory on the way to ``dest``, or ``dest`` itself is a symbolic link: + a copy would then write to the link's target, which can lie outside the workspace.""" + if workspace.is_symlink(): + return True + current = workspace + for part in dest.relative_to(workspace).parts: + current = current / part + if current.is_symlink(): + return True + return False + + +def oracle(task: dict[str, Any], task_dir: Path, run_dir: Path | None = None) -> str: + """No reference answer: the grade reads the workspace only. A task with a ``reference/`` directory + has its files copied over the workspace first, so the reference repair is what gets graded. A destination + reached through a symbolic link in the workspace is refused before anything is copied. A symbolic link in the + reference is refused too: copytree would follow it, and the entries behind it are not walked here.""" + reference = task_dir / REFERENCE_DIR + if reference.is_dir() and run_dir is not None: + workspace = Path(run_dir) / "workspace" + # checked even for an empty reference: copytree still copies the mode onto a linked root + if _linked_on_the_way(workspace, workspace): + raise ValueError("the workspace is a symbolic link; the reference would be written through it") + for source in reference.rglob("*"): # directories too: copytree creates an empty one through a link + if source.is_symlink(): + raise ValueError(f"reference/{source.relative_to(reference).as_posix()} is a symbolic link; " + "the reference is copied without links") + dest = workspace / source.relative_to(reference) + if _linked_on_the_way(workspace, dest): + raise ValueError(f"{dest.relative_to(workspace).as_posix()} would be written through a " + "symbolic link in the workspace") + shutil.copytree(reference, workspace, dirs_exist_ok=True) + return "" + + +GRADERS = {"typecheck": grade} +ORACLES = {"typecheck": oracle} diff --git a/dokimi/dokimi/repos/l0/__init__.py b/dokimi/dokimi/repos/l0/__init__.py index 796eb510d..607d15184 100644 --- a/dokimi/dokimi/repos/l0/__init__.py +++ b/dokimi/dokimi/repos/l0/__init__.py @@ -19,8 +19,9 @@ from ..static import STATUS_CLASSES, check_static, status_class from .adapters import ADAPTERS, detect from .core import Claim, Scan +from .typecheck import Check, check_fences -__all__ = ["ADAPTERS", "Claim", "Scan", "guidance_files", "scan_repo"] +__all__ = ["ADAPTERS", "Check", "Claim", "Scan", "guidance_files", "scan_repo"] def guidance_files(tree: RepoTree, manifest: RepoManifest) -> list[str]: @@ -52,12 +53,14 @@ def _is_build_input(path: str) -> bool: or name in ("BUILD", "BUILD.bazel", "WORKSPACE", "WORKSPACE.bazel", "MODULE.bazel")) -def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | None = None) -> dict[str, Any]: +def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | None = None, + typecheck: Check | None = None) -> dict[str, Any]: """The ``repo_readiness.v1`` payload (without ``schema_version``) for ``tree``. ``static`` is the document the repository's ``static`` entry point wrote. Its claims replace the framework's claim on the same file, line and text, - because the repository knows its own rules better. + because the repository knows its own rules better. ``typecheck`` adds the + ``python_fence`` claims (§8.2); None leaves the Python fences unchecked. """ files = guidance_files(tree, manifest) scan = Scan(tree, frozenset(manifest.build_systems), repo_url=manifest.source_url, guidance=tuple(files)) @@ -68,9 +71,10 @@ def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | N skipped: dict[str, str] = {} scan.preload(files, skipped) aliases = _aliases(tree, files) - for f in files: - if f not in skipped and f not in aliases: - scan.check_file(f) + checked = [f for f in files if f not in skipped and f not in aliases] + for f in checked: + scan.check_file(f) + typecheck_block = None if typecheck is None else check_fences(scan, checked, typecheck) framework = [dict(c.to_dict(), source="framework") for c in scan.claims] static_info: dict[str, Any] | None = None @@ -108,6 +112,7 @@ def scan_repo(tree: RepoTree, manifest: RepoManifest, static: dict[str, Any] | N "by_kind": _grouped(claims, "kind"), "by_file": _grouped(claims, "file"), "static": static_info, + "typecheck": typecheck_block, "warnings": warnings, "incomplete": bool(reasons), "incomplete_reasons": reasons, diff --git a/dokimi/dokimi/repos/l0/typecheck.py b/dokimi/dokimi/repos/l0/typecheck.py new file mode 100644 index 000000000..d5bef0e87 --- /dev/null +++ b/dokimi/dokimi/repos/l0/typecheck.py @@ -0,0 +1,357 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Python fences of the guidance, type-checked against the repository's environment (contract §8.2). + +Each fence that parses becomes a ``python_fence`` claim: ``ok`` when the tool reports no error, +``fail`` (a stale-API candidate) when it does, and ``unresolved`` when it does not parse or no +repository interpreter was given, so nothing was checked. One tool run covers every fence (mypy runs again +without the fences a blocking error named); the interpreter's site-packages are the pinned dependencies, +and imports are not followed past the fence. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import tempfile +import textwrap +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from dokimi import process + +from .core import Scan +from .markdown import PYTHON_LANGS, Fence, split + +TOOLS = ("mypy", "basedpyright") +TIMEOUT = 600 # seconds for one run over every fence of a repository +MAX_WHY = 200 +# The oldest repository Python each tool accepts: the pinned mypy 2.x refuses --python-version 3.9 and exits 2. +MIN_PYTHON = {"mypy": (3, 10)} +# Fragment-only errors (a name an earlier fence defines, a statement that needs a function, loop or async body +# around it), in mypy's or basedpyright's wording; any other error is real. +_FRAGMENT = re.compile(r'^(?:Name "[^"]+"|"[^"]+") is not defined(?: \[reportUndefinedVariable\])?$' + r'|^"(?:await|return|yield|yield from|continue|break)" outside (?:function|loop)$' + r'|^"async (?:with|for)" outside async function$' + r'|^(?:Nonlocal|nonlocal) declaration not allowed at module level(?: \[\w+\])?$' + r'|^Use of "async" not allowed outside of async function(?: \[\w+\])?$' + r'|^"await" allowed only within async function(?: \[\w+\])?$' + r'|^"return" can be used only within a function(?: \[\w+\])?$' + r'|^"(?:break|continue)" can be used only within a loop(?: \[\w+\])?$' + r'|^"(?:yield|yield from)" not allowed outside of a function or lambda(?: \[\w+\])?$') +# A module the repository environment does not have: the fence's own import fails, so nothing it says about that +# module can be checked (in mypy's or basedpyright's wording). A missing module is not a stale call. +_MISSING_MODULE = re.compile(r'^Cannot find implementation or library stub for module named "[^"]+"$' + r'|^Import "[^"]+" could not be resolved(?: \[\w+\])?$') + + +class TypecheckError(RuntimeError): + """The tool could not run or exited with a failure of its own (not a type error).""" + + +@dataclass(frozen=True) +class Check: + tool: str # ``mypy`` (dokimi's own interpreter, pinned in requirements-test.txt) or ``basedpyright`` (on PATH) + python: str | None = None # the repository's interpreter; None means unknown, nothing is checked + + +def python_fences(scan: Scan, files: list[str]) -> list[tuple[str, Fence, str]]: + """Every fence tagged as Python in ``files``: (file, fence, dedented code).""" + out = [] + for f in files: + for fence in split(scan.text(f)).fences: + if fence.lang in PYTHON_LANGS: + out.append((f, fence, textwrap.dedent("\n".join(text for _, text in fence.lines)))) + return out + + +def check_fences(scan: Scan, files: list[str], check: Check) -> dict[str, Any]: + """Add a ``python_fence`` claim per fence to ``scan``; the report's ``typecheck`` block.""" + if check.tool not in TOOLS: + raise TypecheckError(f"unknown tool {check.tool!r}; one of {', '.join(TOOLS)}") + found = python_fences(scan, files) + block: dict[str, Any] = {"tool": check.tool, "tool_version": None, "python": check.python, + "fences": len(found), "checked": 0, "fail": 0, "unknown": 0, "syntax": 0} + # The repository's grammar and version are the target: a fence is parsed by that interpreter, and checked as + # that Python. No feature_version: it approximates an older grammar, the interpreter has the real one. + target = python_version(check) if check.python is not None and found else None + syntax = _syntax_errors(check, [code for _, _, code in found]) + runnable = [] + for (f, fence, code), why in zip(found, syntax, strict=True): + if why is not None: # `f()` is not Python, or too deep for the parser + block["syntax"] += 1 + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + f"not parsed by {check.python or 'dokimi'} ({why}): not type-checked") + continue + runnable.append((f, fence, code)) + if check.python is None: + for f, fence, code in runnable: + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + "unknown: no repository environment (--python-env), not type-checked") + block["unknown"] = len(runnable) + return block + # Nothing to run, no tool run: a version probe would fail on a repository without the tool, for no result. + block["tool_version"] = tool_version(check) if runnable else None + if _older_than_the_tool(scan, check, runnable, target): + block["unknown"] = len(runnable) + return block + found_errors = _run_past_blocking(check, [code for _, _, code in runnable], target) if runnable else {} + fails, unchecked = _claims(scan, check.tool, runnable, found_errors) + block["fail"] = fails + block["checked"] = len(runnable) - unchecked + block["unknown"] += unchecked # "unknown" is every fence that was not checked + return block + + +def _claims(scan: Scan, tool: str, runnable: list[tuple[str, Fence, str]], + found_errors: dict[int, list[tuple[int, str]]]) -> tuple[int, int]: + """One claim per runnable fence from the tool's errors; (failing fences, fences not checked).""" + fails = unchecked = 0 + for i, (f, fence, code) in enumerate(runnable): + errors = found_errors.get(i, []) + if not errors: + scan.add(f, fence.line, "python_fence", _label(code), "ok", f"{tool}: no error") + continue + if _is_unchecked(errors): + unchecked += 1 + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + f"{tool}: {errors[0][1]} ({_unchecked_reason(errors)}: not type-checked)"[:MAX_WHY]) + continue + fails += 1 + # the first real error names the claim: a fragment or missing-module error of the same fence is only counted + line, message = next(e for e in errors if not _missing_context(e[1])) + more = f" (+{len(errors) - 1} more)" if len(errors) > 1 else "" + where = f"line {line}: " if line else "" # 0: the tool gave no line, so the claim is the fence itself + scan.add(f, fence.line + line, "python_fence", _label(code), "fail", + f"{tool}: {where}{message}{more}"[:MAX_WHY]) + return fails, unchecked + + +def _older_than_the_tool(scan: Scan, check: Check, runnable: list[tuple[str, Fence, str]], + target: tuple[int, int] | None) -> bool: + """True, with each runnable fence unresolved, when the repository is older than the tool checks (no run).""" + floor = MIN_PYTHON.get(check.tool, (0, 0)) + if target is None or target >= floor: + return False + for f, fence, code in runnable: + scan.add(f, fence.line, "python_fence", _label(code), "unresolved", + f"{check.tool} checks Python {floor[0]}.{floor[1]} and later; the repository is " + f"{target[0]}.{target[1]}: not type-checked") + return True + + +def _missing_context(message: str) -> bool: + """True for an error the fence's missing context or a module the environment lacks causes (see the patterns).""" + return _FRAGMENT.match(message) is not None or _MISSING_MODULE.match(message) is not None + + +def _is_unchecked(errors: list[tuple[int, str]]) -> bool: + """True when every error of a fence is one of those (see ``_missing_context``): nothing real was found.""" + return all(_missing_context(message) for _, message in errors) + + +def _unchecked_reason(errors: list[tuple[int, str]]) -> str: + """Why the fence is not checked, for the claim: the fence is a fragment, or a module is missing (or both).""" + missing = [m for _, m in errors if _MISSING_MODULE.match(m)] + if len(missing) == len(errors): + return "a module the repository environment lacks" + if missing: + return "a fragment of a longer example, or a module the repository environment lacks" + return "a fragment of a longer example" + + +def tool_version(check: Check) -> str: + """The version the tool reports, e.g. ``2.4.0``. ``-I``: a module in the checkout cannot stand in for it.""" + argv = ([sys.executable, "-I", "-m", "mypy", "--version"] if check.tool == "mypy" + else [check.tool, "--version"]) + done = _run_tool(argv, tempfile.gettempdir()) + if done.timed_out or done.returncode != 0: # a missing module exits 1 with the reason on stderr + raise TypecheckError(f"{check.tool} --version exited {done.returncode}: {done.stderr[-300:]}") + parts = done.stdout.split() + return parts[1] if len(parts) > 1 else done.stdout.strip() + + +def python_version(check: Check) -> tuple[int, int]: + """The repository interpreter's (major, minor) version, read by the interpreter itself (``-I``).""" + done = _run_tool([str(check.python), "-I", "-c", "import sys; print(*sys.version_info[:2])"], + tempfile.gettempdir()) + if done.timed_out or done.returncode != 0: + raise TypecheckError(f"{check.python} did not report its version: {done.stderr[-300:]}") + try: + major, minor = done.stdout.split()[:2] + return int(major), int(minor) + except ValueError: # no output, or output that is not two integers: the path is not a Python + raise TypecheckError(f"{check.python} did not report its version: {done.stdout[-300:]!r}") from None + + +def _label(code: str) -> str: + """The fence's first non-blank line, for the claim's text.""" + first = next((ln.strip() for ln in code.splitlines() if ln.strip()), "") + return first[:80] + + +_PARSE = """\ +import ast, json, sys +def parse(code): + try: + ast.parse(code, type_comments=True) + except TypeError: # Python 3.7: ast.parse has no type_comments keyword (it came in 3.8) + ast.parse(code) +out = [] +for code in json.load(sys.stdin): + try: + parse(code) + out.append(None) + except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: + out.append((exc.msg or "invalid syntax") if isinstance(exc, SyntaxError) else type(exc).__name__) +json.dump(out, sys.stdout) +""" + + +def _syntax_errors(check: Check, codes: list[str]) -> list[str | None]: + """Per fence: None when it parses, else why not. The repository's interpreter parses it (its own grammar); + with none given, dokimi's own, ``-I`` and from the temporary directory, so the checkout cannot shadow it.""" + if not codes: + return [] + python = check.python or sys.executable + with tempfile.TemporaryDirectory(prefix="dokimi-typecheck-") as tmp: + done = _run_tool([python, "-I", "-c", _PARSE], tmp, stdin=json.dumps(codes)) + if done.timed_out or done.returncode != 0 or done.truncated: + raise TypecheckError(f"{python} did not parse the fences: {done.stderr[-300:]}") + try: + reasons = json.loads(done.stdout) + except ValueError: + reasons = None + if not isinstance(reasons, list) or len(reasons) != len(codes): + raise TypecheckError(f"{python} did not parse the fences: {done.stdout[-300:]!r}") + return reasons + + +def _run_past_blocking(check: Check, codes: list[str], + target: tuple[int, int] | None) -> dict[int, list[tuple[int, str]]]: + """Errors per fence index. A run mypy stopped on a blocking error is run again without the fences it named, + so that one fence cannot leave every other fence unchecked; each round drops at least one fence.""" + errors: dict[int, list[tuple[int, str]]] = {} + todo = list(range(len(codes))) + while todo: + found, blocked = _run(check, [codes[i] for i in todo], target) + errors.update((todo[k], found_errors) for k, found_errors in found.items()) + if not blocked: + break + todo = [i for k, i in enumerate(todo) if k not in found] + return errors + + +def _run(check: Check, codes: list[str], + target: tuple[int, int] | None = None) -> tuple[dict[int, list[tuple[int, str]]], bool]: + """Errors per fence index: (line in the fence, message), in tool order; and whether the tool stopped early. + ``target``: the repository's version.""" + with tempfile.TemporaryDirectory(prefix="dokimi-typecheck-") as tmp: + names = [] + for i, code in enumerate(codes): + # A hyphen makes the name unimportable: no ``import`` in one fence can resolve to another fence's file, + # so every fence is checked against the repository's environment alone. + Path(tmp, f"fence-{i}.py").write_text(code + "\n", encoding="utf-8") + names.append(f"fence-{i}.py") + if check.tool == "mypy": + # The config is explicit and empty: no subject or host mypy settings apply. -I: nothing in the + # run directory or PYTHON* variables can stand in for mypy or for the standard library. + config = os.path.join(tmp, "mypy.ini") + Path(config).write_text("[mypy]\n", encoding="utf-8") + version = [] if target is None else ["--python-version", f"{target[0]}.{target[1]}"] + argv = [sys.executable, "-I", "-m", "mypy", "--python-executable", str(check.python), *version, + "--config-file", config, "--check-untyped-defs", + "--follow-imports=silent", "--no-incremental", "--cache-dir", os.path.join(tmp, "cache"), + "--output", "json", *names] + else: + # The project is explicit and empty: a pyrightconfig.json above the temporary directory is not read. + # the target version is set here: --pythonpath gives the import paths, not the version checked against + settings = {} if target is None else {"pythonVersion": f"{target[0]}.{target[1]}"} + project = os.path.join(tmp, "pyrightconfig.json") + Path(project).write_text(json.dumps(settings) + "\n", encoding="utf-8") + argv = [check.tool, "--project", project, "--pythonpath", str(check.python), "--outputjson", *names] + done = _run_tool(argv, tmp) + # 1: the tool found errors. mypy's 2: it stopped on a blocking one ("errors prevented further checking"), + # which is a result only when it names a fence. Any other exit is the tool's own failure (pyright's 2 is fatal). + blocked = check.tool == "mypy" and done.returncode == 2 + if done.timed_out or (done.returncode not in (0, 1) and not blocked): + raise TypecheckError(f"{check.tool} exited {done.returncode}: {done.stderr[-300:]}") + if done.truncated: # the middle of the output was dropped, so an error there would be missed + raise TypecheckError(f"{check.tool} output was truncated; not every error is known") + if done.returncode == 1 and not done.stdout.strip(): # exit 1 with no report: the tool did not run + raise TypecheckError(f"{check.tool} exited 1 with no report: {done.stderr[-300:]}") + errors = _errors(check.tool, done.stdout) + if blocked: + if not errors: # a usage error, or a blocking one that names no fence + raise TypecheckError(f"{check.tool} exited 2 with no fence report: {done.stderr[-300:]}") + return errors, True + return errors, False + + +# What a tool run may inherit from the host. Not the whole environment: MYPYPATH, for one, adds modules to +# mypy's search path, so a fence could pass on a module the pinned environment does not have. +_TOOL_ENV = ("PATH", "LANG", "LC_ALL", "LC_CTYPE", "TMPDIR", "TEMP", "TMP", "SYSTEMROOT") + + +def _run_tool(argv: list[str], cwd: str, stdin: str | None = None): + try: + return process.run(argv, timeout=TIMEOUT, env=process.Env.only(_TOOL_ENV), cwd=cwd, input=stdin) + except OSError as exc: + raise TypecheckError(f"cannot run {argv[0]}: {exc}") from exc + + +def _errors(tool: str, stdout: str) -> dict[int, list[tuple[int, str]]]: + """Parse the tool's JSON output into errors keyed by fence index.""" + out: dict[int, list[tuple[int, str]]] = {} + # A report we cannot read is a tool failure, not a clean run: the fences it would have flagged pass otherwise. + diagnostics = _mypy_errors(stdout) if tool == "mypy" else _basedpyright_errors(stdout) + for name, line, message in diagnostics: + index = _fence_index(name) + if index is not None: + out.setdefault(index, []).append((line, message)) + return out + + +def _mypy_errors(stdout: str) -> list[tuple[str, int, str]]: + """The error diagnostics in mypy's report: one JSON object per line.""" + diagnostics = [] + for text in stdout.splitlines(): + if not text.strip(): + continue + try: + d = json.loads(text) + # A valid JSON object is not a diagnostic: without these fields "{}" would read as a clean run. + if not isinstance(d, dict) or not {"file", "line", "severity", "message"} <= d.keys(): + raise ValueError("not a diagnostic") + if d["severity"] == "error": + line = int(d.get("line") or 0) # mypy writes -1 for an error with no line + diagnostics.append((str(d.get("file")), max(line, 0), str(d.get("message")))) + except (ValueError, TypeError) as exc: + raise TypecheckError(f"mypy wrote a line that is not a diagnostic in its report: {text[:200]!r}") \ + from exc + return diagnostics + + +def _basedpyright_errors(stdout: str) -> list[tuple[str, int, str]]: + """The error diagnostics in basedpyright's report: one document whose ranges are zero-based.""" + try: + doc = json.loads(stdout) # empty output is not a report either + if not isinstance(doc, dict) or not isinstance(doc.get("generalDiagnostics"), list): + raise ValueError("no generalDiagnostics list") # a missing list is not an empty one + return [(Path(str(d.get("file"))).name, int(d["range"]["start"]["line"]) + 1, + str(d.get("message")) + (f" [{d['rule']}]" if d.get("rule") else "")) + for d in doc["generalDiagnostics"] if d.get("severity") == "error"] + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise TypecheckError(f"basedpyright wrote a report that is not in its JSON format: {exc}") from exc + + +def _fence_index(name: str) -> int | None: + """``3`` for ``fence-3.py``; None for any other file (a note about an import, say).""" + stem = Path(name).name + if stem.startswith("fence-") and stem.endswith(".py") and stem[6:-3].isdigit(): + return int(stem[6:-3]) + return None diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 57aa5ce6a..b138951f9 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -31,7 +31,7 @@ from dokimi.artifacts import schema_errors from dokimi.measure import SCORE_BOUND, as_float, is_finite -from .graders import DATA_FILES, NAMES as BUILTIN_NAMES, PREFIX as BUILTIN_PREFIX +from .graders import DATA_FILES, NAMES as BUILTIN_NAMES, PREFIX as BUILTIN_PREFIX, TASK_PROBLEMS from .host_spec import HostSpecError, parse_host from .process_metrics import Counter, parse_counter from .source import RepoTree @@ -596,6 +596,8 @@ def _builtin_problems(tree: RepoTree, task_dir: str, raw: dict[str, Any], name: if name not in BUILTIN_NAMES: return [f"unknown built-in grader {BUILTIN_PREFIX + name!r}; known: " + ", ".join(BUILTIN_PREFIX + n for n in BUILTIN_NAMES)] + if name in TASK_PROBLEMS: + return TASK_PROBLEMS[name](raw) key, default = DATA_FILES[name] path = raw.get(key, default) if not isinstance(path, str): diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index 3e3a1a782..8aff71098 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -31,7 +31,9 @@ from typing import Any from .container import CONTENT_DIR, RUN_DIR, SOURCE_DIR, ContainerError, ContainerSpec, docker_network -from .grading import EvalContext, outcome, run_entry +from .graders import builtin_name +from .graders.typecheck import REFERENCE_DIR +from .grading import EvalContext, outcome, read_grade_json, run_entry from .host import probe_host, shortfall_kinds, unmet, validate_tasks from .runner import agent_layout, container_name from .workspace import PrepError, Task, prepare_workspace @@ -82,7 +84,8 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, reason, unsupported_by, facts = _host_reason(ctx, task, work_dir, host) # Only what the hardware decides is skipped: the null check (it passes where the bug cannot show) and, # unless the miss is CPUs alone, the oracle. Preparing the workspace and isolation hold anywhere. - run_oracle = not reason or (unsupported_by == "host" and shortfall_kinds(task, facts) <= {"cpus"}) + run_oracle = _oracle_applies(task) and ( + not reason or (unsupported_by == "host" and shortfall_kinds(task, facts) <= {"cpus"})) for kind, run_it in (("null", not reason), ("oracle", run_oracle)): _check_step(ctx, task, work_dir / f"{task.id}__{kind}", kind, run_it, net, res) if not res["problems"]: @@ -95,6 +98,15 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, return res +def _oracle_applies(task: Task) -> bool: + """False for a built-in ``dokimi:typecheck`` task with no ``reference/``: its oracle has no repair to apply + (the grade reads the workspace as it is), so the self-check runs the null step only, and the untouched + listed files must fail the gate.""" + if builtin_name(task.raw.get("grader")) != "typecheck": + return True + return (task.dir / REFERENCE_DIR).is_dir() + + def _host_reason(ctx: EvalContext, task: Task, work_dir: Path, host: dict[str, Any] | None) -> tuple[str | None, str | None, dict[str, Any] | None]: """``(reason, unsupported_by, facts)``; reason and unsupported_by are ``None`` when the host suffices.""" @@ -136,6 +148,22 @@ def _prepare(ctx: EvalContext, task: Task, rd: Path) -> None: (rd / "events.jsonl").write_text("", encoding="utf-8") +def _missing_listed(rd: Path) -> list[str]: + """For a typecheck task with no reference the null grade is the only step: a listed path it reports missing, + or a suppression already in the workspace, fails that step, so no run that repairs the code could pass.""" + doc, _ = read_grade_json(rd / "grade.json") + details = doc.get("details") if isinstance(doc, dict) else None # the grader's lists are under details + metrics = doc.get("metrics") if isinstance(doc, dict) else None + missing = details.get("missing") if isinstance(details, dict) else None + out = [] + if isinstance(missing, list) and missing: + out.append(f"null: listed path {', '.join(str(p) for p in missing)} is missing; with no reference no run " + "can pass") + if isinstance(metrics, dict) and isinstance(metrics.get("n_suppressions"), int) and metrics["n_suppressions"] > 0: + out.append("null: the untouched workspace already silences mypy; with no reference a code repair cannot pass") + return out + + def _verdict_problems(task: Task, kind: str, grade: dict[str, Any], rd: Path) -> list[str]: if grade["status"] == "grader_failure": return [f"{kind}: grader failure ({'; '.join(grade['reasons'])}); see {rd / 'grade.log'}"] @@ -143,7 +171,8 @@ def _verdict_problems(task: Task, kind: str, grade: dict[str, Any], rd: Path) -> reached = top is not None and grade["score"] is not None and grade["score"] >= top if kind == "null": out = ["null: the untouched workspace passes the gate"] if grade["success"] else [] - return out + ([f"null: the untouched workspace reaches score.max {top}"] if reached else []) + out += [f"null: the untouched workspace reaches score.max {top}"] if reached else [] + return out + ([] if _oracle_applies(task) else _missing_listed(rd)) if grade["score_only"] and task.score: # outcome() fails pass: null when the task has no `score` return [] if reached else [f"oracle: {task.score['metric']} = {grade['score']}, below score.max {top}"] out = [] if grade["success"] else ["oracle: the reference solution does not pass the gate"] diff --git a/dokimi/dokimi/schemas/repo_readiness.v1.json b/dokimi/dokimi/schemas/repo_readiness.v1.json index 6340840af..8b9bca3a3 100644 --- a/dokimi/dokimi/schemas/repo_readiness.v1.json +++ b/dokimi/dokimi/schemas/repo_readiness.v1.json @@ -65,6 +65,7 @@ } ] }, + "typecheck": {"oneOf": [{"type": "null"}, {"type": "object", "description": "The Python-fence check (§8.2): tool and its version, the interpreter (null: unknown), fence counts."}], "description": "null when the check was not requested."}, "warnings": {"type": "array", "items": {"type": "string"}, "description": "Guidance the manifest names but the tree lacks: a glob with no match, a missing also_scan file, no guidance at all."}, "incomplete": {"type": "boolean", "description": "True when an also_scan file is missing, a guidance file or Bazel build input cannot be read, or no guidance file was scanned; `repo scan --fail-on-fail` then exits 1."}, "incomplete_reasons": {"type": "array", "items": {"type": "string"}, "description": "The warnings that make the scan incomplete, as they appear in `warnings` and in its order; empty when the scan is complete. Absent from reports written before it was added."}, diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index c645cedb0..19aa99b91 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -142,6 +142,9 @@ optional_dependencies_dev_groups = ["dev"] DEP001 = ["audit_mocks"] # Declared only to cap the version ragas resolves (see the comment in the ragas extra). DEP002 = ["langchain-community"] +# repos/graders/typecheck.py imports mypy inside the grading image, where it is installed +# for the grader; the wheel does not ship it. The host never imports it (the L0 check runs it as a tool). +DEP004 = ["mypy"] [tool.ruff] # The width `ruff format` wraps to, if someone runs it: neither CI nor diff --git a/dokimi/requirements-test.txt b/dokimi/requirements-test.txt index ae449e15d..219b74f19 100644 --- a/dokimi/requirements-test.txt +++ b/dokimi/requirements-test.txt @@ -11,6 +11,8 @@ pytest-socket>=0.7 pytest-timeout>=2.2 pytest-randomly>=3.15 pytest-xdist>=3.6 +# The typecheck tests run mypy (tests/test_repo_typecheck_grader.py, test_repo_l0_typecheck.py) on this interpreter. +mypy==2.4.0 diff-cover>=10.0 httpx>=0.27.0 numpy>=1.24.0 diff --git a/dokimi/requirements.lock.txt b/dokimi/requirements.lock.txt index 9924f2207..eebd6db5d 100644 --- a/dokimi/requirements.lock.txt +++ b/dokimi/requirements.lock.txt @@ -183,6 +183,65 @@ anyio==4.15.1 \ # via # httpx # openai +ast-serialize==0.13.0 \ + --hash=sha256:017ddd4f22e727ef93e66df2d53340a6ff809b7e34cc2218f67918ae6239aad0 \ + --hash=sha256:12441bc7e41e495db5634c98adc8f8886b619ce2f1e68effe3f792a2adca9f47 \ + --hash=sha256:192aed400b2b92ebe41da17e856b9b6e17dbcebe0011ce4c6370d4a8a0486233 \ + --hash=sha256:19b1e8f4c088ce91053df310b444fceeddb39f728ca7d04272c983646e36314b \ + --hash=sha256:220a993dfc8b173e7062f690f9e00f4ebefe56718171bf35dccd73a9f8cea100 \ + --hash=sha256:359fcebc49f855bd189cf568235dc84eabf521e00d03fce43135cad6484906bc \ + --hash=sha256:3a9469e4b93d87e4ee8f5c7e9462f973032793a24b9ae37ffee860220f17586a \ + --hash=sha256:3bb8dd779c0a25478fe1db1a8b06dd4dd5e66077d6d0afe354acadb6d9aee7d0 \ + --hash=sha256:3e20e9ca3952196b91798f77ef267c36c7b3470021f950aa361d9be003fb655f \ + --hash=sha256:3f5d4a7fcc916026010bae2a154e5be2c05040e67ef5c494c66a5cf315c41e30 \ + --hash=sha256:446de6067d853f61b4bde8741762c83d06b57ea81f96dd47977714d9f31837fa \ + --hash=sha256:452fdaf5ff0b791870bb332254e083107d7abe29ef43251411c265c5b138f9a2 \ + --hash=sha256:4b2ee61692de03009e6a8f372f24fbc2d4b768a2f51ecd426bb84acdfd6da3d1 \ + --hash=sha256:4d1e15da4b6afc6fe80b87704be452aa0639df93d019a516e9ac9540357fc9b2 \ + --hash=sha256:4dc7a24c734aded0557ef90bfabd2278b37502aacde84f49b53b4cb6a0711ea9 \ + --hash=sha256:4e0bc018a457052d4638b469f90674e6ec0e32d86ac4a7bfa1f7d1c71a961426 \ + --hash=sha256:4f55668338bcb871e83ee21ba865fb08b7b4b13af9312742f9878c39f9d84ee3 \ + --hash=sha256:5e88733df5ffff9062b5ff2779cf402e0aa1a61ca3f7f84b577a1af2b7e09676 \ + --hash=sha256:619050b18705310e19e254cdb7554289fe14da374cbfbb1362cd84635896fb7f \ + --hash=sha256:684e191dd41b08b0b92692a181380a70cd76e3b6469606a40aae1ca6dab39e7d \ + --hash=sha256:6a406251363eeb5c7b85a405eddd123e627a531bd150dc673a7f5dd087743b5c \ + --hash=sha256:6a49f2a01a6df3150e022087cec0bf0ad580fec8f38a17f124d07dbb115106d1 \ + --hash=sha256:6bab08a6f287cd620578084f9974cfaf3bef71959105f62af85fa70298b24851 \ + --hash=sha256:6c95c04f1781cbefe89d512ce30e051d10823827ae542d9f18d5c2e7ba0fad08 \ + --hash=sha256:6cbfa6dae34d5686056ef7c40ce1d3e7e1de48555e3a2fed985aef2d1f869d9a \ + --hash=sha256:771cf5ee8329ee8472dd7a3d8bea7dbc480032ed8ddb4d37d40b57b95ee19ef2 \ + --hash=sha256:77efef815ecae1195ac9889f616bd518d53b2173e87157043253b864af1c81c5 \ + --hash=sha256:7ce1b50c5a68233e890926405afc308a5f10f6f49ec3a094d3dfa8b6733e4496 \ + --hash=sha256:7d7376c611055f5ee44e5e13a2620dbc6846f47c583952c1704c8805154c2d2f \ + --hash=sha256:807875ed8c5de739c8c55a45944336fcb9b8601d77fcee384a743cd0497211d6 \ + --hash=sha256:841262622499585f0610a927434db526578553d8dae270b90d4c419a385e46b7 \ + --hash=sha256:84cd9efdd3cd780b1f5361049becd91e0bcb0f16c2c216f41ee82e728a98390b \ + --hash=sha256:8aff1682f9fa3e119a1cf8ef47504d38f7019b22b79e0f2b5c2135b9532d14db \ + --hash=sha256:8e7c6fec7fe03cb8f40c4af81d742aa0cf690cf0b7bded47508a8a392093f414 \ + --hash=sha256:8f672c8e6d3b9ef6e365a5543aee2012247d1d58d948ceddb75b33a6679609ca \ + --hash=sha256:98edcd24240fa217d903c8f221bc05575baf38a87a207264ee7c800d21eef5a4 \ + --hash=sha256:9eaa20714acf43ef0a0c82850a2ec8097f834648f527c38f1483e2fd9a52cd3e \ + --hash=sha256:a0bdcef01e643e0810d2dedfb64d924bcfe079a15dc20d1c067870cc01d5c5e6 \ + --hash=sha256:a1714ee591a8e19833c0530a89e5a9faa7f62a11fc88f62fc5b722c7425dcc1f \ + --hash=sha256:a918572608ceb20fba8c2b83560f3d20be90effa4614589477b46d81672f0c3d \ + --hash=sha256:b004c3bdab0beb45194cd66c0b8feea40d676e461d26296f9c791c8e3e1b7061 \ + --hash=sha256:bd89da715b857a27c33fad713ea0561912c56f773ebd0fed2760cedd99724dc6 \ + --hash=sha256:be6b1a4ee49866c77eb8a50e9cbc845370e6c15230a126d0a71aeab35f31c78c \ + --hash=sha256:c51c855d8b7d5403925599acd3c6fc91b321eba3bf46dcc9fe88fd2d6619dac7 \ + --hash=sha256:c77e5b62dfbfdfc1b025105f5038114ae988a0e616d48a845e0e57173f3f37c8 \ + --hash=sha256:cdd8fd066858b57ea2761b3d3989c90ea23913825bbb5453cc684c28bba3fb19 \ + --hash=sha256:d47c8f0eedf0a41681c10a7c497fef3691c4a6b4af4de6c93a4916bb29712554 \ + --hash=sha256:dffcffa543c8fcfb1ca941038eeae23e7f97ad8994e4d6f81fbd658cfa8cb440 \ + --hash=sha256:e241f68cf5060bff9b161b202b60d6d52161ff3777fe56eb6a9a6764fdb7fdd9 \ + --hash=sha256:e611937e6e77448496489627ae2558b6f6143449b1fb33f8a495665212eee58a \ + --hash=sha256:ee58f0db40f121ff0820242b286702700bc0ec58a53b6ac43ce4f43714d42e0d \ + --hash=sha256:efaab6400de8ee2d0e38695feccb0758acf11c1d0f8bc58a7090c566dd3ae88e \ + --hash=sha256:f0cc1f94fcd3b67005a32ee3e4c6b27cdc41659f697840d00fbb1e815ec27044 \ + --hash=sha256:f30f0e59be30c0c9540e8908b14874bc8d3c1d52a4562a4cfb426b003bd6c28b \ + --hash=sha256:f8da1a31e941adbea886a85fb25efc6f09353d58c665fdbc523a914e3d2e49fe \ + --hash=sha256:fadba24386498ed745c848b0d45e4939a506694bd2b474c57576e9640f3defe2 \ + --hash=sha256:fe2a3c8480e8e5eb41eaa958279b8c530b77b45065423f4ebd5223e118293055 + # via mypy attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 @@ -1269,6 +1328,145 @@ jsonschema-specifications==2025.9.1 \ --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d # via jsonschema +librt==0.16.0 ; platform_python_implementation != 'PyPy' \ + --hash=sha256:001bfd59a7d45b17e3e75f2a8c6405280b35e7b84471792778e718c4f368950e \ + --hash=sha256:0058f9d68721094105917254c72ac0569117bb7b13b9769cf45d26d89f9d21cd \ + --hash=sha256:02118f56a9c36ddd07dfd9b919d9ecc117ba20a90987d56aa4c429fa34509188 \ + --hash=sha256:0253721561787b8df8443eb347b7a6461015354e5bdd37ee38a41fef220d2bb0 \ + --hash=sha256:02d89c813d5ff74b17df72d3a34819d132cd168e56b81bf755b809bd9e46b8c4 \ + --hash=sha256:0314058469f4d2fd279ce7c62ac274ac82c3918ef7db62ef0697c4c359370155 \ + --hash=sha256:0dbe4096a7ecc00fa835d24510ad8545a4efef738dac96e0e63516783ccde905 \ + --hash=sha256:0ead24d2562a49473dddd9efef8581f020007eb0054389c3ee3ffad38b1ca4c9 \ + --hash=sha256:13b4e8aba90b0b1c82474e9844aa9ffe7ad3faa484350e1da64cb8188d903134 \ + --hash=sha256:14ed6ebe3e4f85f326d7920011ad30ff49ed9334e62cf88caef9ba973d9e3a92 \ + --hash=sha256:17bac7f7a16b328fff77e440287693eb017abde913595b5827ebccbc21ecd8a6 \ + --hash=sha256:1b384b90ab79a7bc30b566895809a636e0666f21f3cf12b54823d025b7e83839 \ + --hash=sha256:1bc17e54e5305f8d40b7ca203671ff5a9e59c1d0f8ea0f625dcca53a3984de11 \ + --hash=sha256:1d28ae980ae2218f9c5b95d191e947296f918c9bf0b400d467a9430275bbe678 \ + --hash=sha256:1e511762a074005bb0aa569166779834e75e438370226930d0ce1866d4b6a33b \ + --hash=sha256:20fe0bf9053885e21c62b3e091fb5e73e1c54d1daf2e70eb388d70763bcd4220 \ + --hash=sha256:242e00b3d4fa37c3d3c1ca5f5c9adb7d909ddb1eac9c41f2787320d00caa0af2 \ + --hash=sha256:25a58a19ea8d83b68209f04912df765e9260635ef77646542ed4b4abe6bc7940 \ + --hash=sha256:273d00be33792a15189331df10f1f1331621b043881e66c6c4377f7f776e1291 \ + --hash=sha256:28e038895b998d7a0c7798922ce8a1dc157675df5cf1c9ef0aca809ed804b7a1 \ + --hash=sha256:2bec3818c7da7c96ceae0ef5915a3d16c52dd08f3ea913bf1fe8568c447c7978 \ + --hash=sha256:2c4aa329c17bd1aaea4f6e89335d8ccd494b3a5830b6654462273e50e11023f0 \ + --hash=sha256:300c3ffdc459f4a779a8411ecb188e3ac0b1ff3a3a7b099642555dedae06c69b \ + --hash=sha256:30b7beaf3f4487b7d8adef1f158b49067cb4d5a19fa7a3bf31a4e7a820e435c5 \ + --hash=sha256:314e703f0c19320dc8094e7a784b9cf29e1b67402515abf580069a6363c0b4f1 \ + --hash=sha256:33f41443a1f4e1f099331b3d8120e409fbff84b9760bc1cc9ea496f37ddaa5cc \ + --hash=sha256:349c0bcb87ebd07481b6ff781e25cdc699723dbe2212e57dabb27f7a13b7b87d \ + --hash=sha256:36e53948e99bbe3ffea257124cfcae1cfb01831555c9a9c903c9f9a72db7fd07 \ + --hash=sha256:375bfe6b572a8f6cfc398709356046173bf27e64c4c5edaf5f7062f051fb4bf9 \ + --hash=sha256:378dfaffb38e59c24a87cde5713cd865d51ff7383fa12947f3907f306ea1ca55 \ + --hash=sha256:3931f7a3db322e7f44e02a280e3949326ce9579ad388ee8d691dc7c76da9fb70 \ + --hash=sha256:39ca4f2f2fe05de8e63493da592d84311adabe5bef52b193851981da9816b302 \ + --hash=sha256:39ec1d5a14e37baf1450a6cabf03fe552340808bf1ad9d71824ab90117716459 \ + --hash=sha256:3ddeb3c9dedb461bb457c6c7d9aa7fbf35329da313d1a7543d00c8d0f3473c96 \ + --hash=sha256:3e0c39bdc85370422e8b637be76eb1fd07d30967551b03e62267dd156f553152 \ + --hash=sha256:3e483a8d69ede8067db70c0e83007423b6925de6fd53afed01d66160f2e9398c \ + --hash=sha256:3f0b8114c44b2ac06ff5dacd08e07e8e807ff4f46083f2a1602685122559be41 \ + --hash=sha256:3ff4b2367926b69c6215635902cccb04048e73094e9862900d27cb2c6bbff143 \ + --hash=sha256:4323193ac0cd025f85af531df8ba91bf24d1973b401697347a6282e8fd3fcf5e \ + --hash=sha256:468df902df016a06eb0e40b0747dc8d14e47d7a38b18b63b1fb167d85cb94d63 \ + --hash=sha256:473eebc7866bb0a0c8849a292b5e7157c1aba5d14d0f0f610c52158d6d964262 \ + --hash=sha256:47ada6ea32636492c61aa8ad27ae3b9404bfe7a97e3ba946d1984236cc741da0 \ + --hash=sha256:4b6183e2e2e0ee00aac2ec07c7f7d151c97e85666b304f574b71cff0f9fccc4e \ + --hash=sha256:4e29522c62e28595ff7e324c6834ade51127707f0e255b18d1c1cf03d39c1048 \ + --hash=sha256:4eb1313a19847089ee81e88742abedf285c60538816640b99742d8534b81d26a \ + --hash=sha256:52327da75a94012e7f932f913d20d3876bed3c102be00e6c3e8600ff7bdd58a7 \ + --hash=sha256:54d11f726aae9df5a6ffbbf0a03a52449bbac84a53ef03669cb41cdfd4ae41bf \ + --hash=sha256:5696d7f52e7b37217cb3a8f92c744fe835942602fdd4c1a8bc4741d3bfdce15e \ + --hash=sha256:5750a105b42a416f930edc59054927a406effb2550cd5bab92ad7a5842ed5d05 \ + --hash=sha256:5810ba811297fdf37a1531a57667cb8ace0842013ca8606bf9eb7c24cf4be154 \ + --hash=sha256:5981c011b306781ce561e18e14230a14524a3d8109b97553666c942c18f31a96 \ + --hash=sha256:5a269c46ae327d8e6f8c1f85f7516cb52c0fa48127565a1105a4f4a05ff2a0b4 \ + --hash=sha256:5b976054553670829985ed767feb78fb6bcede0175327c4844dd5c281c1be659 \ + --hash=sha256:5bcc2c4726ced915b00de0c9856a4eeabfb3fddb93e10e0b8f735b7709358b6d \ + --hash=sha256:5cd5b092441053364af968ea12084692cb9d4a22f3ce9524e377880bf028761e \ + --hash=sha256:5f49cff01bd608ef7d97104cb035c75455e79c2d70bf4a506cf773338ac1860d \ + --hash=sha256:6072e92dd876ff6ceeb6cf371e35e51f479349837391341f479b08df4564242b \ + --hash=sha256:64c79520414a3fdfc6aabd7593e6169afa14d5f8d9908d4b498db068868b08dd \ + --hash=sha256:67e718c7a43f8db325abbbf1404e2d535f12f8f7a1a82259568385cc5274b82a \ + --hash=sha256:69ba927445cfaaffb4081003ef5224c55a5c2ab67ef956f416ef744916e44121 \ + --hash=sha256:6a63610fa76524edfa605b5b259a603915c7a6e10e54f003e5503030506e81de \ + --hash=sha256:6c5da27e8056439f927ea896735da60e616c477a8293feaa3233d4e7781a6726 \ + --hash=sha256:6c8893eae2fd13c5488d94056f3e6e5cf3142bfb1c4acaf136cb33d760c5964b \ + --hash=sha256:6d4a64283ee61824b5790de882bc68e2d9d7a5143537cb7a966f7354f71646d4 \ + --hash=sha256:6fe436af2eaf630474f491af5d032cbe45f93fcff5c3b9fe4ab194a7255b20ff \ + --hash=sha256:71b93b42784e25b975079573c642a8fedb049a7bb31d70a51721b1666b3b2ced \ + --hash=sha256:7393c9a48dcce4817dbd4b0d8ff6237efe9b0a0609f5b0adaef315f8541726b5 \ + --hash=sha256:77c7a2b4fe2c1369e0d5aa1cade26740a7b14be32fbc9a5535d617d20065c39d \ + --hash=sha256:7a1d272724b581bb6bc769dfdafed6da2ecc9886ba2450311de55a4ac2e1e9cd \ + --hash=sha256:7cc365f006891afb006b52d5ee5ee74c09306ffa20e2f8705a32a4450af2f3ba \ + --hash=sha256:7e510b7770bee609617a3374a96548eb114cae048023e3f049ee449e7ff2db32 \ + --hash=sha256:80039ba9b6a7d5f1a0175a4cca6bbefead87bd854c80abad1cb30afe47a830db \ + --hash=sha256:83d4041a3d9b2fd053a8a4e1f22878b3e5833e2712956382d5c048d791454e91 \ + --hash=sha256:845a511b60ca43b9880dcc84a9784c891d6a2098c829130b320846c69c9c0c68 \ + --hash=sha256:877698bf6bca5721d8be345f2fe09778e40ecadea8b58c73075f2b1a53666bf2 \ + --hash=sha256:8caf96a4ef8fb27d0ac0d1ad8337d26a240acd4a02fe4345d0a8f264753e8f99 \ + --hash=sha256:8ceafb70f2a4f0826f11031942e59c0728fd98da112dc346d4352bde1e486866 \ + --hash=sha256:8f36c58e33b304b525c6c9c5076399c6ebf1109e17b9051a05a407b091b9215b \ + --hash=sha256:8ff5d26c529336be9bd7ae04483235d77778ee7d6444a95353102b542601ce81 \ + --hash=sha256:909d8e3c1faee44cb762b1c519ff8613dcc5ceae5c99987a00917b5a31fd1d6a \ + --hash=sha256:92caf82ebef5e12d21c72242b70d1e92536f1711cf2a727a4c276de4b4469087 \ + --hash=sha256:931a0bb0fcac88f263e269e46eb30ba8e21402cd3c62ca40cb97034c0693fab1 \ + --hash=sha256:943c6bbecbdf7fa575a4f2952fcfd848c88ef95507c3fca411e89d4ac3ff8143 \ + --hash=sha256:94aed6a8308818b91677957d1bd03188869cd7aeb23c5dba7912a6c0402f7602 \ + --hash=sha256:94be5cb7bca4df6201f4183e9e4fa2086c655283d20b38cd84500a69057575a7 \ + --hash=sha256:953107e2f68d0f3512c48f898b0dbf0ce5cc52bba0f318d847c985dc555ee4cc \ + --hash=sha256:96f576f2711f8519152ec76d0e599243555c1f07679fa73606ca8c8c868c0be6 \ + --hash=sha256:a33e0dae1f8592146a4764d54ce842b278732d21a84e17c3bbe6b1bc158a2248 \ + --hash=sha256:a4aaefb4ba6c07e1aeebb2795c8958148f1d6f9af3b555b53d23d766edb6d67a \ + --hash=sha256:a8afb6557920860b7a3a596eb804cf37e09e7cf8a803db2478c202acc72d8c2e \ + --hash=sha256:aa9357a1b4d4fc787bb718a59cb1112c28c8a976d6bfa268b71cc0a4ab8f3a94 \ + --hash=sha256:ac38d6d8d66bf3d744148dbbc0b8e193e195a51e364ed55e224631f5721891fc \ + --hash=sha256:ad37d5b9abd49c9a655dcda7ea52a8a752884062ef1ee71ae17c2f2a0f81fe6a \ + --hash=sha256:aea7b1f2b125dad5de85f049136651bff256c883c65e6b9209b2da0a1ac3cdef \ + --hash=sha256:afced3dfc17cd805ecf7a3d77996a71cf5f2c75aa66eb0c21a9930f4fc992f86 \ + --hash=sha256:b0e3e721c75d2e79a76d4422c79d7ba705fe1bbafec907037fe7a657a480a0e3 \ + --hash=sha256:b6d085d70bce51d43c5c7c36d63490770180d8779e71c49305c87b4213918de7 \ + --hash=sha256:b95d5d92ab83d39e760a52091bb1baba664f3a2351e39b1e16801e5747c2f0e9 \ + --hash=sha256:b9d6d4b14e92d876f8026b54c20c445f36425214c1081dc76f74e40db386b82b \ + --hash=sha256:bc02954b1295de798bbdb0b4e2d8a28c2117de8b5c73dcbeb27dc32572dfb971 \ + --hash=sha256:bd3150023d3dc2bc70f3784e59ffa1140d56ddba3d8125b3d6f9f85221279bfc \ + --hash=sha256:be56ba9c884143495b517f23fe794ae367d58cd89ea0fdd6d437e3c024a87f9f \ + --hash=sha256:c17194318e4c0c0348b36f36c2ec7534436fe0a4c15582403162a4f08c80797a \ + --hash=sha256:c3d1bb7841a816ace6449bb26d3f9560dbfa20e71c568d23f0f62bf1e68f50b1 \ + --hash=sha256:c43bd6e642d8a248c114327f98dd25ac5a7cb5aa168ef02f0559b91874df16b8 \ + --hash=sha256:c5db585d43449a5f54303d4b2774e45e1babd975cfe1630a3d708c0b80c3e560 \ + --hash=sha256:c5e6144e68b577f157519f2ba88ca20e3ed61c29b00e5cdfa76cd2d45acf059a \ + --hash=sha256:c6f1b27bf1632a7e016af9f145f82be95e1edd7721a646505c21059257cb5a04 \ + --hash=sha256:c71d1b76210a36729fedfc5115069b50a3d8619054745f8758fe5d6f19e86671 \ + --hash=sha256:c72c5295a84bd249526da9bdca38f2e176d15c31c13bb0063c5053f4ca023421 \ + --hash=sha256:ca8052401c55d7511dda6760719fda7618067e83535d7d0010096d216c34b667 \ + --hash=sha256:d1aabe3925cbb4a08d15b7b20ba4011b53019da0c4173a25155139b7b1baed65 \ + --hash=sha256:d3c94211ee0c4f8d649ec06b7c115c0ec4eadb873a0e3154ca15cef3f814b071 \ + --hash=sha256:d46ca272b251d033dd4527b0dec5f261a28a52bd5fa0f99c117b0a1f8588cc2d \ + --hash=sha256:d608f0bf3b8cbddd0067fe02cb8cab7d13e8eb9386b23a1843d4363044fd9e22 \ + --hash=sha256:d6a365f2ab45a984d0e00eee0dd17f599ceab8cadab6ea07b6111c8132fc0e42 \ + --hash=sha256:d92db7a0f6aee44f1baee94750457e8d2d1c6ccea41842de6268d34e8dc7eddd \ + --hash=sha256:df183721229ae51eef90108c115b43e98cb169b6155d34480338e5fc6616df00 \ + --hash=sha256:e05108e0849966f53a8d2d3112a7af881d0efaa479bc735bba91108f9f2350a7 \ + --hash=sha256:e1967e36ac4cae0c7e9615ad32e1a513cdacff79f9e8afb28bedc91caf48b4f3 \ + --hash=sha256:e42f8e098b9c5396fefa05fb1cc7e33b0e08fc51da106b5de4a45fd22aac6743 \ + --hash=sha256:e56aaf8c167548dc8e5d6f3bd0f48dcdd299a23c73be3f744aab79d99e9c7f5d \ + --hash=sha256:e9ce0bc440e7fd09b5f51f372f0f6640658f854b8fb05260f819cc669c93c42d \ + --hash=sha256:ebefd60b42e2a82b32d136bb5f7c94eadfcd29f772b547df6f3291d1ed855a1c \ + --hash=sha256:ef46c1a29ffb8c72e882e22618ec618778eacd0578fb22c6e7cf9c11d15f357b \ + --hash=sha256:efc49c462d4516b8a58b00b490078fa64689fd1fe66970cc190131d7afb8027e \ + --hash=sha256:f01f3805f2dae4781c0c34b440e31740d082950bdaf89a6f601ad589a28af57a \ + --hash=sha256:f06c689cb14afd9b612727553a5ec5a40febf113ca41c4413a2b0b334285884b \ + --hash=sha256:f1e8591bd8a5a628cd7f07954c6a1592359a878bf032957a8e9057a41d644311 \ + --hash=sha256:f4462528b6000afe8f16907b5c7c2553abf1df005ba5140e6eb394541c3624c3 \ + --hash=sha256:f7be7cf555bc30ec12622e9447299cc4a9b8ff307548b634794353db0c2065dc \ + --hash=sha256:f81b5b19ce748ef68d4746656b7929762eb2fe99269b266e4be07e2ee4de7144 \ + --hash=sha256:f9807485a908f00355820f18e91e045ffdcdc5adb68aaec40a1e2b88c5f7bba1 \ + --hash=sha256:fbe4fb8c5445f7496d7f7f6bb0807875d09d47e6771ffa175fb2df2895fb86ba \ + --hash=sha256:fe4372c52d4849096c6cc1cda2817d293ec51440c890474ed59ef38d46556f18 \ + --hash=sha256:fe52bf4641069e7978a14253b036cb9002def1926317e710f2e249f8a8c47742 \ + --hash=sha256:ff7baa55f8e7c69851419e50a666015d02a74198716fd45c0125a2112e0a389f + # via mypy litellm==1.106.4 \ --hash=sha256:1c2b7d072a47c3f04032764d8143ed6ade8c354a331f4ad5ab9b9d5b1ba2dd4c \ --hash=sha256:1dc78af5a710715b1807a73d1b378d4d1fd82a4f4a13e45179b76287cbbabba9 \ @@ -1635,6 +1833,64 @@ multiprocess==0.70.19 \ --hash=sha256:e6c0674d34b8adac22533f6786576b3de4e396aaeda9e0c15378af9b8ada2702 \ --hash=sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f # via datasets +mypy==2.4.0 \ + --hash=sha256:058165f564ccf559c68c70fec2091fca5891110480210c22594635e3f6683437 \ + --hash=sha256:0bb95cf34899e4619c61ab0a8667804e139e580b30d5df12af2102dfe44d0c97 \ + --hash=sha256:13fa24f439c0e48a290a3922fa14ccd22f2762bae99d2142931b3e40a9055080 \ + --hash=sha256:172e30b8fea631fe310f0c665477f52d9ea40bb4e99e0c81dc30118563b13710 \ + --hash=sha256:1dc0f64b0a92ae27a49d2175f0bacfa56e15bdc5b420cf92e0c1f79292219cbb \ + --hash=sha256:20e9a5cd875837520c43db98dea0b6d0c2197833d95c30127d8f570fb9b1f00b \ + --hash=sha256:2106b55105ba5ea9be4f53a24517fc5fa927ff1585edc9bc1a975abb72caef89 \ + --hash=sha256:236e0d68f6941992b0811128e652590f590db444ab29ad8f1324765b9298b946 \ + --hash=sha256:29243242cf72582b65f9582ad9e56e8cb281566ed3519f4cd70bb8b9f2977e90 \ + --hash=sha256:295ecf2e57542cd836ca537486951289678c8c7d1ee6ad74ebe29b2168a003cf \ + --hash=sha256:29eb0b9427a6b11b992e452f6cceb8af724f4dceb47e779d0b35e405e996ea5e \ + --hash=sha256:3011537be6cf1de4511c0255a324362a812b58184bbe61e15f59c8b31033bd74 \ + --hash=sha256:3adef556a19eb3b630bf86a79c29d7da3d61e541472d0d897ca01b171c0abf8f \ + --hash=sha256:3bd0e340f0ebe65c548210f53be3fd8192e83964760caf0c28bef368e68b0d37 \ + --hash=sha256:4209da39d85cf240f762af622d8180fcdfcb4727d021f44ade62d613a1a43324 \ + --hash=sha256:4a378fc15fb33e321f04652c166ce73eeb8833a97c3d218132844e938cd93220 \ + --hash=sha256:502b94b0b331f7dafe32fd6b151797ddbb4f32385b362e722c783a025e5954a3 \ + --hash=sha256:528c8744b8b5e3ecb8774f86af38d2376216816e9908317ad055f3c9c2d74799 \ + --hash=sha256:5786ef987b3767e51aaa53f20aec104c0252b42ecda7aef8e8b4cbae279b05c5 \ + --hash=sha256:5d20e6c7c35fcbf2a0ebdd0eaeacfbc243009dfd33ab7822d54e213912e6dbbd \ + --hash=sha256:615b03922d40e186fd1df73156473db0ba525c639bb4fd1cf28e1694879c9b23 \ + --hash=sha256:6306086b87cf7f8a29aa618d9fd9bffb56c59247166b9660fdb54d86d7714ecd \ + --hash=sha256:6be721bd4bd57576193653b75b4af3461c9d0bf7dd8b528f782e9be210dc75bb \ + --hash=sha256:720434d48542ecfe84d32d287b727569d3fc8f5769acd39051130e490a5c295c \ + --hash=sha256:77bdaebd452f43fcfc4cc3ba94352a3ea537cd01e3f2d0879f48673d2ec00d6e \ + --hash=sha256:7c4f8f8d1d1c0e2832d8ee7113dd08f6df6c7aad9e863fcbed9f25832be0b8c4 \ + --hash=sha256:7da85fbcff6dac1abcc636707bed38b45598131fb7a605d9719c70b5cc733af8 \ + --hash=sha256:7f38f57d344f8b6accb40e01c3d83cfc590498231724d16c07ffb7940f157818 \ + --hash=sha256:82d0f94c8587ccb472622ee7795280aaa38a06640d5f45b3f16909d6dd86a989 \ + --hash=sha256:86d616fe84c6eab8026f8c50ab5bcb90db780d2ccd233d971e34e92bede9b359 \ + --hash=sha256:9279488933040b638c0ab739084c0ca100efeea6db581bf5d7628d8e89de53fe \ + --hash=sha256:970b221ed5842213d98e3c480c08f795ace4b1f81fb21e1b126bd0476bce1c34 \ + --hash=sha256:9f03a7828cca2b0adcd6662aee8f2711ff8830e1027641fdea3ab0b787483566 \ + --hash=sha256:9f459f0b4f0596d9d51fe7716b404b35287b99e77da98a7af90a65dd5fd61141 \ + --hash=sha256:9fa247e02b505a45a2775f69df38d360d197e3790bc60f717595db9eda358b6e \ + --hash=sha256:a3f86fd1313dd69d013e265f1fdcd12ea7a9d606f9875b2a3db946cd334555f3 \ + --hash=sha256:a6e851b82c0661f69f1630fc16172c68787a6a9cf0991e7c6437d60976cdcd76 \ + --hash=sha256:a96b07a49b7b1d025ce59c1b3acbcf24bead9a83da4523c4a6bde1bb94e7a0e1 \ + --hash=sha256:afa89837d9be67e0cadfa33bca3bb7efdda98c3b07e74dc3b635ebfb1c8a926a \ + --hash=sha256:ba05652540bf12828e52abae807b024b09ca144ff4f75e2450a81d69c376425b \ + --hash=sha256:bc378bdad4e9f12b5bd96466083d1e71acf00594ec9c7b2bdb5e02816f77f303 \ + --hash=sha256:c9de622fd397495695d0598ddc789222bfcfec9d7c9ec3a1e385c855e3bc5e01 \ + --hash=sha256:cb734b2668c1f40d07ce093bbeb4407e9527c67901627b0e1679825be3f09975 \ + --hash=sha256:d01c5d26a352acc6d5cf3128225477e1e8465e8d3029d4c345807fbf7f3cf093 \ + --hash=sha256:e05ff2925d8b37ad26c80c1b9dc43ae5d455da2df1e23c24c095a6425917c57e \ + --hash=sha256:e1fde197ae65be856a034a91b70ed747a16562ca69577785f06c661548424bf1 \ + --hash=sha256:e3ebe2f72a2a1156065a9851570ffbf50c0a93cdccadef9c6e05c508a4fd10b1 \ + --hash=sha256:e76172710bd4e5eeae061abfd68347e5264632e02778be61784671ae3a2132f5 \ + --hash=sha256:f83353e47ab520bf6fd4df8f5897d9fe081211f2fbc4b7d37736a3e3c166cbcf \ + --hash=sha256:f9b028548b3af480e2b1ed8df14ccaac86f99c9f600d1580770ab7ba3dcd40f0 \ + --hash=sha256:fb443e81057896132d3642d6be219e6efd158691ac7883e3ba8fcb469865f05d \ + --hash=sha256:ffda5244fd1ad71a1e54405e35f50d09b80c3978efa120f58bd1252ae32c62d2 + # via -r requirements-test.txt +mypy-extensions==1.1.0 \ + --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ + --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 + # via mypy numpy==2.4.6 ; python_full_version < '3.12' \ --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ @@ -1861,6 +2117,10 @@ pandas==3.0.6 \ --hash=sha256:fb625f426b375bcc96e3a04c5d5d266cd7be6ae5d6866e0e703382ab5164068c \ --hash=sha256:ff51a4459ed036e93d1eb1bb5e6e7b28685d3cb6b7c12b91c05b31024e234729 # via datasets +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + # via mypy pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 2e460ae9f..ae31d9bf7 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -22,7 +22,6 @@ "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 30, "rationale": "pre-existing when the gate was introduced"}, "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13, "rationale": "W8 adds the --repeats guard after main's no_queries handling raised the file; the command is split with the rest of novelty.py in a separate PR"}, - "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 12}, "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "went over the cap on main in #120 (built-in review graders) after #93 seeded this baseline; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, "dokimi/commands/tools.py::cmd_tools_preference": {"complexity": 11}, diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index a7b476183..1936cac0c 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -154,6 +154,7 @@ --product --prompt-file # repo tasks replay (#265) adds it: no registry concept yet; a F1b migration names it --provider +--python-env # repo scan --typecheck: the interpreter whose site-packages the fences resolve against; no registry concept yet --queries --queries-by-skill # novelty routing (#172) adds it: no registry concept yet; a F1b migration names it --queries-from-headings @@ -211,6 +212,8 @@ --trials # catalog trigger-probe (#217) adds it: a banned synonym of reps; renaming it changes the CLI, so it waits for F1b --trigger-set --turn-reminder +--typecheck # repo scan's Python-fence check: no registry concept yet +--typecheck-tool # repo scan's checker for --typecheck (mypy or basedpyright): no registry concept yet --typed-max-len --typed-threads --validate diff --git a/dokimi/tests/test_repo_l0_typecheck.py b/dokimi/tests/test_repo_l0_typecheck.py new file mode 100644 index 000000000..0e44bb39e --- /dev/null +++ b/dokimi/tests/test_repo_l0_typecheck.py @@ -0,0 +1,844 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""L0 Python-fence check (contract §8.2): fences type-checked against an interpreter, or unknown without one.""" + +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos import parse_manifest +from dokimi.repos.l0 import Check, scan_repo +from dokimi.repos.source import WorkTree +from dokimi.repos.l0.typecheck import _fence_index + +MANIFEST = { + "schema": "repo_manifest.v1", + "id": "toy", + "source": {"url": "https://github.com/acme/toy"}, + "eval_dir": "dev/evals", + "guidance": {"globs": ["**/AGENTS.md"]}, + "entry_points": {"grade": "true"}, + "tasks": "dev/evals/tasks", + "build_systems": ["make"], +} + +FENCE = "```" +AGENTS = f"""# Toy + +{FENCE}python +import json + +print(json.dumps({{"a": 1}})) +{FENCE} + +{FENCE}python +import json + +print(json.dumps(1).nosuch()) +{FENCE} + +{FENCE}python +f() +{FENCE} +""" + + +def _tree(tmp_path: Path, agents: str = AGENTS) -> WorkTree: + root = tmp_path / "r" + root.mkdir() + (root / "AGENTS.md").write_text(agents, encoding="utf-8") + return WorkTree(root) + + +def _fence_claims(payload: dict) -> list[dict]: + return [c for c in payload["claims"] if c["kind"] == "python_fence"] + + +def test_without_the_check_the_python_fences_are_not_claimed(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST)) + assert payload["typecheck"] is None + assert _fence_claims(payload) == [] + + +def test_mypy_against_an_interpreter_passes_the_good_fence_and_fails_the_stale_one(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + claims = {c["line"]: c for c in _fence_claims(payload)} + assert claims[3]["status"] == "ok" # the good fence opens on line 3 + stale = claims[12] # the stale fence opens on line 9; its error is on code line 3, so the claim is line 12 + assert stale["status"] == "fail" + assert "nosuch" in stale["why"] + assert payload["typecheck"]["fail"] == 1 + assert payload["typecheck"]["checked"] == 2 + + +def test_a_fence_that_is_not_python_is_unresolved_not_failed(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + placeholder = {c["line"]: c for c in _fence_claims(payload)}[15] # opens on line 15 + assert placeholder["status"] == "unresolved" + assert "not parsed by" in placeholder["why"] and "invalid syntax" in placeholder["why"] + assert payload["typecheck"]["syntax"] == 1 + + +def test_without_an_interpreter_every_checkable_fence_is_unknown(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", None)) + fences = _fence_claims(payload) + assert {c["status"] for c in fences} == {"unresolved"} + assert all("unknown" in c["why"] for c in fences if "not parsed by" not in c["why"]) + assert payload["typecheck"] == {"tool": "mypy", "tool_version": None, "python": None, "fences": 3, + "checked": 0, "fail": 0, "unknown": 2, "syntax": 1} + + +def test_the_fence_line_of_a_failure_is_the_guidance_line(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + failed = [c for c in _fence_claims(payload) if c["status"] == "fail"] + assert [c["line"] for c in failed] == [12] + + +def test_the_report_names_the_tool_and_its_version(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + tc = payload["typecheck"] + assert tc["tool"] == "mypy" + assert tc["tool_version"] and tc["tool_version"][0].isdigit() + + +@pytest.mark.skipif(shutil.which("basedpyright") is None, reason="basedpyright is not installed") +def test_basedpyright_fails_the_same_stale_fence(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), + typecheck=Check("basedpyright", sys.executable)) + failed = [c for c in _fence_claims(payload) if c["status"] == "fail"] + assert [c["line"] for c in failed] == [12] + + +def test_fence_file_names_map_back_to_their_index_only(): + assert _fence_index("fence-3.py") == 3 + assert _fence_index("elsewhere/x/fence-12.py") == 12 + assert _fence_index("fence-x.py") is None + assert _fence_index("fence_3.py") is None # the name an import could reach: not a fence file + assert _fence_index("pkg/other.py") is None + + +def test_the_scan_command_refuses_python_env_without_typecheck(tmp_path, capsys): + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--python-env", sys.executable]) + from dokimi.commands.repo import _typecheck_or_exit + + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "needs --typecheck" in capsys.readouterr().err + + +def test_the_scan_command_refuses_a_python_env_that_is_not_executable(tmp_path, capsys): + missing = tmp_path / "no-such-python" + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", str(missing)]) + from dokimi.commands.repo import _typecheck_or_exit + + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "not an executable interpreter" in capsys.readouterr().err + + +def test_bare_typecheck_flag_means_mypy(tmp_path): + from dokimi.commands.repo import _typecheck_or_exit + + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck"]) + assert args.typecheck is True + assert args.python_env is None + assert _typecheck_or_exit(args).tool == "mypy" + + +def test_the_typecheck_flag_takes_no_value_so_a_source_after_it_is_not_its_checker(tmp_path): + # `repo scan --typecheck `: the source must not be read as the checker name + args = build_parser().parse_args(["repo", "scan", "--typecheck", str(tmp_path)]) + assert str(tmp_path) in vars(args).values() + assert args.typecheck is True + + +def test_the_checker_is_chosen_by_typecheck_tool(tmp_path): + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--typecheck-tool", "basedpyright"]) + assert args.typecheck_tool == "basedpyright" + + +def test_typecheck_tool_without_typecheck_is_refused(tmp_path, capsys): + from dokimi.commands.repo import _typecheck_or_exit + + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck-tool", "mypy"]) + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "needs --typecheck" in capsys.readouterr().err + + +def test_the_payload_is_json_serialisable(tmp_path): + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + json.dumps(payload) + + +def _scan_args(tmp_path: Path, *extra: str): + """Parsed ``repo scan`` arguments for a plain tree whose AGENTS.md is the fixture.""" + tree = tmp_path / "tree" + tree.mkdir() + (tree / "AGENTS.md").write_text(AGENTS, encoding="utf-8") + manifest_file = tmp_path / "m.json" + manifest_file.write_text(json.dumps(MANIFEST), encoding="utf-8") + return build_parser().parse_args(["repo", "scan", str(tree), "--manifest-file", str(manifest_file), *extra]) + + +def test_the_scan_text_report_names_the_tool_and_the_interpreter(tmp_path, capsys): + args = _scan_args(tmp_path, "--typecheck", "--python-env", sys.executable) + args.func(args) + out = capsys.readouterr().out + assert f"({sys.executable}): 3 python fences, 2 checked, 1 fail, 0 unknown, 1 not Python" in out + assert "FAIL AGENTS.md:12 [python_fence]" in out + + +def test_the_scan_text_report_says_unknown_without_an_interpreter(tmp_path, capsys): + args = _scan_args(tmp_path, "--typecheck") + args.func(args) + assert "(no --python-env: unknown)" in capsys.readouterr().out + + +def test_a_tool_failure_is_a_usage_error_not_a_traceback(tmp_path, capsys, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + def broken(check, codes, target=None): + raise l0_typecheck.TypecheckError("mypy exited 2: broken") + + monkeypatch.setattr(l0_typecheck, "_run", broken) + args = _scan_args(tmp_path, "--typecheck", "--python-env", sys.executable) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "repo scan --typecheck: mypy exited 2: broken" in capsys.readouterr().err + + +def test_an_interpreter_that_reports_no_version_is_not_labelled_as_the_tool(tmp_path, capsys, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + def no_version(check): + raise l0_typecheck.TypecheckError("/venv/bin/python did not report its version: ''") + + monkeypatch.setattr(l0_typecheck, "python_version", no_version) + args = _scan_args(tmp_path, "--typecheck", "--python-env", sys.executable) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + err = capsys.readouterr().err + assert "did not report its version" in err and "--typecheck-tool" not in err + + +def test_basedpyright_missing_from_path_is_a_usage_error(tmp_path, capsys, monkeypatch): + empty = tmp_path / "empty-path" + empty.mkdir() + monkeypatch.setenv("PATH", str(empty)) + args = _scan_args(tmp_path, "--typecheck", "--typecheck-tool", "basedpyright") + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "--typecheck-tool basedpyright: not on PATH" in capsys.readouterr().err + + +def test_an_interpreter_with_nothing_to_check_runs_no_tool(tmp_path): + payload = scan_repo(_tree(tmp_path, "# Toy\n\nNo code here.\n"), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + assert payload["typecheck"]["checked"] == 0 + assert payload["typecheck"]["fail"] == 0 + + +def _completed(**over): + from types import SimpleNamespace + + done = {"returncode": 0, "stdout": "", "stderr": "", "timed_out": False, "truncated": False} + done.update(over) + return SimpleNamespace(**done) + + +def test_a_tool_that_exits_one_with_no_report_is_an_error_not_a_pass(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd: _completed( + returncode=1, stderr="No module named mypy")) + with pytest.raises(l0_typecheck.TypecheckError, match="no report"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_truncated_tool_output_is_an_error_not_a_partial_result(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd: _completed(truncated=True)) + with pytest.raises(l0_typecheck.TypecheckError, match="truncated"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_a_relative_python_env_is_made_absolute_before_the_tool_runs(tmp_path, monkeypatch): + from dokimi.commands.repo import _typecheck_or_exit + + link = tmp_path / "py" + link.symlink_to(sys.executable) + monkeypatch.chdir(tmp_path) + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", "py"]) + assert Path(_typecheck_or_exit(args).python).is_absolute() + + +def test_a_fence_too_deep_for_the_parser_is_unresolved_not_a_crash(tmp_path): + deep = "x = " + "+".join(["1"] * 200000) # ast.parse raises RecursionError on this (CPython 3.14 among them) + agents = f"# Toy\n\n{FENCE}python\n{deep}\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", None)) + [claim] = _fence_claims(payload) + assert claim["status"] == "unresolved" + assert "RecursionError" in claim["why"] + assert payload["typecheck"]["syntax"] == 1 + + +def test_a_line_less_mypy_error_is_the_fence_line_not_the_line_above(tmp_path): + from dokimi.repos.l0.typecheck import _errors + + report = json.dumps({"file": "fence-0.py", "line": -1, "severity": "error", "message": "no line"}) + assert _errors("mypy", report) == {0: [(0, "no line")]} + + +def test_mypy_checks_untyped_function_bodies(tmp_path): + # Guidance examples are often unannotated functions; mypy skips their bodies unless told not to. + agents = f"{FENCE}python\nimport json\n\ndef example():\n json.dumps(1).nosuch()\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + [claim] = _fence_claims(payload) + assert claim["status"] == "fail" + assert "nosuch" in claim["why"] + + +def test_a_subject_or_host_mypy_config_cannot_suppress_a_fence(tmp_path, monkeypatch): + home = tmp_path / "home" + (home / ".config" / "mypy").mkdir(parents=True) + (home / ".config" / "mypy" / "config").write_text("[mypy]\nignore_errors = True\n", encoding="utf-8") + (home / ".mypy.ini").write_text("[mypy]\nignore_errors = True\n", encoding="utf-8") + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("XDG_CONFIG_HOME", str(home / ".config")) + tree = _tree(tmp_path, f"{FENCE}python\nimport json\nprint(json.dumps(1).nosuch())\n{FENCE}\n") + (tree.root / "mypy.ini").write_text("[mypy]\nignore_errors = True\n", encoding="utf-8") + [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + + +def test_a_host_mypypath_cannot_supply_a_module_the_pinned_environment_lacks(tmp_path, monkeypatch): + extra = tmp_path / "extra" + extra.mkdir() + (extra / "ghost_pkg_mod.py").write_text("def run() -> int:\n return 1\n", encoding="utf-8") + monkeypatch.setenv("MYPYPATH", str(extra)) + tree = _tree(tmp_path, f"{FENCE}python\nimport ghost_pkg_mod\nprint(ghost_pkg_mod.run())\n{FENCE}\n") + [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + # the module is missing from the pinned environment: unresolved, and never checked against the host's copy + assert claim["status"] == "unresolved" + + +@pytest.mark.skipif(sys.platform == "win32", reason="the stand-in interpreter is a POSIX shell script") +def test_a_python_env_that_reports_no_version_is_a_usage_error_not_a_traceback(tmp_path, capsys): + silent = tmp_path / "silent-python" + silent.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + silent.chmod(0o755) + args = _scan_args(tmp_path, "--typecheck", "--python-env", str(silent)) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "did not report its version" in capsys.readouterr().err + + +def test_the_version_probe_is_not_shadowed_by_a_module_in_the_checkout(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + import mypy.version + + checkout = tmp_path / "checkout" + checkout.mkdir() + (checkout / "mypy.py").write_text("print('mypy 9.9.9')\n", encoding="utf-8") + monkeypatch.chdir(checkout) + assert l0_typecheck.tool_version(Check("mypy", sys.executable)) == mypy.version.__version__ + + +def test_mypy_is_told_the_repository_interpreters_version(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd): + seen.append(argv) + return _completed(returncode=0, stdout="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("mypy", sys.executable), ["x.py"], (3, 11)) + argv = seen[0] + assert argv[argv.index("--python-version") + 1] == "3.11" + + +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="the fence needs a repository interpreter older than 3.12") +def test_a_fence_newer_than_the_repository_grammar_is_a_syntax_error(tmp_path): + # `type` is a 3.12 statement: the repository's interpreter (here, 3.11) is what parses the fence + tree = _tree(tmp_path, f"{FENCE}python\ntype Alias = int\n{FENCE}\n") + payload = scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + assert payload["typecheck"]["syntax"] == 1 + + +def _fake_tools(monkeypatch, mypy_result, seen: list): + """Stand-in for the tool runs: the version probes answer, the parse passes every fence, mypy gives ``mypy_result``.""" + from dokimi.repos.l0 import typecheck as l0_typecheck + + def fake(argv, cwd, stdin=None): + seen.append(argv) + if "ast" in argv[-1]: # the fence parse, run by the repository's interpreter + return _completed(returncode=0, stdout=json.dumps([None] * len(json.loads(stdin)))) + if "version_info" in argv[-1]: + return _completed(returncode=0, stdout="3 11\n") + if "--version" in argv: + return _completed(returncode=0, stdout="mypy 2.4.0 (compiled: no)\n") + return mypy_result + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + + +def test_a_tool_that_stops_on_a_blocking_error_is_run_again_on_the_fences_it_did_not_name(tmp_path, monkeypatch): + blocking = json.dumps({"file": "fence-0.py", "line": 1, "severity": "error", "message": "blocking"}) + stale = json.dumps({"file": "fence-0.py", "line": 1, "severity": "error", "message": "stale"}) + runs: list = [] + _fake_tools(monkeypatch, None, runs) + from dokimi.repos.l0 import typecheck as l0_typecheck + + fake = l0_typecheck._run_tool + + def per_run(argv, cwd, stdin=None): + done = fake(argv, cwd, stdin) + if done is not None: + return done + mypy_runs = [a for a in runs if "--output" in a] + report = blocking if len(mypy_runs) == 1 else stale # the second run has only the other fence, as fence-0 + return _completed(returncode=2 if len(mypy_runs) == 1 else 1, stdout=report + "\n") + + monkeypatch.setattr(l0_typecheck, "_run_tool", per_run) + agents = f"{FENCE}python\nx: int = 1\n{FENCE}\n\n{FENCE}python\ny: int = 2\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["fail", "fail"] + assert "blocking" in claims[0]["why"] and "stale" in claims[1]["why"] + assert [a[-1] for a in runs if "--output" in a] == ["fence-1.py", "fence-0.py"] + assert payload["typecheck"]["checked"] == 2 + + +def test_a_blocking_fragment_fence_does_not_hide_a_stale_fence(tmp_path): + # a top-level yield stops mypy and is itself a fragment: the stale fence must still be named, not unresolved + agents = (f"{FENCE}python\nconn = open('/dev/null')\nyield conn\n{FENCE}\n\n" + f"{FENCE}python\nimport json\njson.dumps(1).nosuch()\n{FENCE}\n") + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["unresolved", "fail"] + assert "nosuch" in claims[1]["why"] + assert payload["typecheck"]["fail"] == 1 + + +def test_a_tool_that_exits_two_with_no_fence_report_is_an_error_not_a_pass(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=2, stderr="usage: mypy [-h]")) + with pytest.raises(l0_typecheck.TypecheckError, match="exited 2 with no fence report"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_basedpyright_is_told_the_repository_interpreters_version(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = {} + + def fake(argv, cwd, stdin=None): + seen["config"] = json.loads(Path(argv[argv.index("--project") + 1]).read_text(encoding="utf-8")) + return _completed(returncode=0, stdout='{"generalDiagnostics": []}') + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("basedpyright", sys.executable), ["x: int = 1\n"], (3, 11)) + assert seen["config"] == {"pythonVersion": "3.11"} # not the interpreter's own version: the target + + +def test_basedpyright_is_given_an_empty_project_so_no_config_above_the_run_applies(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = {} + + def fake(argv, cwd, stdin=None): + seen["argv"] = argv + seen["config"] = Path(argv[argv.index("--project") + 1]).read_text(encoding="utf-8") # while it exists + return _completed(returncode=0, stdout='{"generalDiagnostics": []}') # basedpyright always writes its report + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("basedpyright", sys.executable), ["x: int = 1\n"]) + assert seen["argv"][0] == "basedpyright" + assert seen["config"] == "{}\n" + + +def test_a_fence_is_parsed_by_the_repository_interpreter_not_the_host(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + return _completed(returncode=0, stdout=json.dumps([None])) + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + assert l0_typecheck._syntax_errors(Check("mypy", "/opt/repo/python"), ["x = 1\n"]) == [None] + assert seen[0][:3] == ["/opt/repo/python", "-I", "-c"] + + +@pytest.mark.skipif(not Path("/bin/true").exists(), reason="needs /bin/true") +def test_a_true_python_env_is_a_usage_error_not_a_traceback(tmp_path, capsys): + # /bin/true exits 0 with no output: it reports no version, so the scan refuses it cleanly + args = _scan_args(tmp_path, "--typecheck", "--python-env", "/bin/true") + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 + assert "did not report its version" in capsys.readouterr().err + + +def test_a_blocking_type_comment_is_a_failing_fence_not_a_tool_failure(tmp_path): + # ast.parse reads no type comments by default, so this passes the parse; mypy then exits 2 on it + tree = _tree(tmp_path, f"{FENCE}python\nx = 1 # type: ???\n{FENCE}\n") + [claim] = _fence_claims(scan_repo(tree, parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + assert "type comment" in claim["why"] + + +def test_a_linked_guidance_file_has_its_fences_checked_and_counted_once(tmp_path): + root = tmp_path / "r" + root.mkdir() + (root / "AGENTS.md").write_text(f"{FENCE}python\nimport json\n\njson.nosuch()\n{FENCE}\n", encoding="utf-8") + (root / "CLAUDE.md").symlink_to("AGENTS.md") + manifest = parse_manifest(dict(MANIFEST, guidance={"globs": ["AGENTS.md", "CLAUDE.md"]})) + payload = scan_repo(WorkTree(root), manifest, typecheck=Check("mypy", sys.executable)) + assert len(_fence_claims(payload)) == 1 + assert payload["typecheck"]["fences"] == 1 + + +def test_only_python_tagged_fences_become_claims(tmp_path): + bad = "import json\n\njson.nosuch()\n" + agents = f"{FENCE}python\n{bad}{FENCE}\n\n{FENCE}bash\n{bad}{FENCE}\n\n{FENCE}json\n{bad}{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + assert payload["typecheck"]["fences"] == 1 + assert [c["status"] for c in _fence_claims(payload)] == ["fail"] + + +def test_a_fence_cannot_import_another_fence_as_a_module(tmp_path): + agents = f"{FENCE}python\nvalue: int = 1\n{FENCE}\n\n{FENCE}python\nimport fence_0\nprint(fence_0.value)\n{FENCE}\n" + payload = scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + # fence_0 is not a module of the environment: the import is unresolved, and the fence is not passed by another's file + assert [c["status"] for c in _fence_claims(payload)] == ["ok", "unresolved"] + + +def test_mypy_is_given_the_repository_interpreter(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + return _completed(returncode=0, stdout="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + l0_typecheck._run(Check("mypy", "/opt/repo/bin/python"), ["x: int = 1\n"]) + argv = seen[0] + assert argv[argv.index("--python-executable") + 1] == "/opt/repo/bin/python" + + +def test_nothing_to_check_runs_no_tool_not_even_a_version_probe(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + return _completed(returncode=1, stderr="No module named mypy") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + payload = scan_repo(_tree(tmp_path, "# Toy\n\nNo code here.\n"), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + assert seen == [] + assert payload["typecheck"]["tool_version"] is None + + +def test_fences_that_do_not_parse_run_no_tool_either(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + seen = [] + + def fake(argv, cwd, stdin=None): + seen.append(argv) + if "version_info" in argv[-1]: + return _completed(returncode=0, stdout="3 11\n") + return _completed(returncode=0, stdout=json.dumps(["invalid syntax"])) # the parse: one fence, not valid + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + payload = scan_repo(_tree(tmp_path, f"{FENCE}python\nx = (\n{FENCE}\n"), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable)) + assert all("--version" not in argv and "mypy" not in argv for argv in seen) # no tool run + assert payload["typecheck"]["syntax"] == 1 + assert payload["typecheck"]["tool_version"] is None + + +def test_basedpyright_exit_two_is_the_tools_own_failure(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=2, stdout="{}", stderr="fatal")) + with pytest.raises(l0_typecheck.TypecheckError, match="exited 2"): + l0_typecheck._run(Check("basedpyright", sys.executable), ["x: int = 1\n"]) + + +def test_an_exit_code_outside_zero_to_two_is_the_tools_own_failure(monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=3, stdout="", stderr="crashed")) + with pytest.raises(l0_typecheck.TypecheckError, match="exited 3"): + l0_typecheck._run(Check("mypy", sys.executable), ["x: int = 1\n"]) + + +def test_only_error_diagnostics_count_as_fence_errors(tmp_path): + from dokimi.repos.l0 import typecheck as l0_typecheck + + mypy_out = "\n".join(json.dumps({"file": "fence-0.py", "line": n, "severity": sev, "message": "m"}) + for n, sev in [(1, "error"), (2, "note"), (3, "warning")]) + assert l0_typecheck._errors("mypy", mypy_out) == {0: [(1, "m")]} + pyright_out = json.dumps({"generalDiagnostics": [ + {"file": str(tmp_path / "fence-0.py"), "severity": sev, "message": sev, "range": {"start": {"line": n}}} + for n, sev in [(0, "error"), (1, "warning"), (2, "information")]]}) + assert l0_typecheck._errors("basedpyright", pyright_out) == {0: [(1, "error")]} + + +@pytest.mark.parametrize("tool,stdout", [ + ("mypy", "not JSON"), + ("mypy", "[1, 2]"), + ("basedpyright", "not JSON"), + ("basedpyright", '{"generalDiagnostics": [{"severity": "error", "message": "m"}]}'), # no range + ("mypy", "{}"), # valid JSON, no diagnostic fields: not a clean run + ("mypy", '{"file": "fence-0.py", "line": 1, "message": "m"}'), # no severity + ("basedpyright", "{}"), # no generalDiagnostics list + ("basedpyright", '{"generalDiagnostics": 3}'), +]) +def test_a_nonempty_report_that_is_not_the_tools_json_is_a_tool_failure(monkeypatch, tool, stdout): + # Exit 1 says the tool found errors; a report we cannot read must not pass every fence as clean. + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "_run_tool", lambda argv, cwd, stdin=None: _completed( + returncode=1, stdout=stdout, stderr="")) + with pytest.raises(l0_typecheck.TypecheckError, match="report"): + l0_typecheck._run(Check(tool, sys.executable), ["x: int = 1\n"]) + + +def test_a_repository_older_than_mypy_supports_is_unresolved_and_mypy_is_not_run(tmp_path, monkeypatch): + # mypy 2.x refuses --python-version 3.9 and exits 2: the fences are unknown for that reason, not a tool failure + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 9)) + + def no_run(*args, **kwargs): + raise AssertionError("mypy must not run for a target it refuses") + + monkeypatch.setattr(l0_typecheck, "_run", no_run) + payload = scan_repo(_tree(tmp_path), parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["unresolved", "unresolved", "unresolved"] + assert "3.9" in claims[0]["why"] and "3.10" in claims[0]["why"] + assert payload["typecheck"]["checked"] == 0 + + +def test_a_fence_that_is_a_fragment_of_a_longer_example_is_unresolved_not_stale(tmp_path): + # `client` is defined by an earlier fence, and `await` at top level is only valid in an async body + fragments = f'{FENCE}python\nclient.decode("{{}}")\n{FENCE}\n\n{FENCE}python\nawait asyncio.sleep(1)\n{FENCE}\n' + claims = _fence_claims(scan_repo(_tree(tmp_path, fragments), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved", "unresolved"] + assert all("longer example" in c["why"] for c in claims) + + +def test_a_fragment_with_a_real_error_still_fails(tmp_path): + fence = f'{FENCE}python\nclient.decode("{{}}")\nx: int = "a"\n{FENCE}\n' + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + + +def test_a_top_level_async_with_or_async_for_is_a_fragment_not_stale(tmp_path): + # the same missing context as a top-level await: the snippet's body runs inside an async function + fences = ["import asyncio\nasync with asyncio.timeout(1):\n await asyncio.sleep(0)", + "async def gen():\n yield 1\nasync for x in gen():\n print(x)", + "import asyncio\nimport json\nasync with asyncio.timeout(1):\n json.dumps(1).nosuch()"] + claims = _fence_claims(scan_repo(_tree(tmp_path, "".join(f"{FENCE}python\n{c}\n{FENCE}\n\n" for c in fences)), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved", "unresolved", "fail"] + assert "nosuch" in claims[2]["why"] + + +def test_basedpyright_top_level_async_with_is_a_fragment_not_stale(tmp_path, monkeypatch): + from dokimi.repos.l0 import typecheck as l0_typecheck + + def fake(argv, cwd, stdin=None): + if stdin is not None: + return _completed(returncode=0, stdout=json.dumps([None])) + if argv[-1] == "--version": + return _completed(returncode=0, stdout="basedpyright 1.40.2\n") + report = {"generalDiagnostics": [ + {"file": "fence-0.py", "severity": "error", "message": 'Use of "async" not allowed outside of async function', + "rule": "reportGeneralTypeIssues", "range": {"start": {"line": 1}}}]} + return _completed(returncode=1, stdout=json.dumps(report), stderr="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 12)) + fence = f"{FENCE}python\nimport asyncio\nasync with asyncio.timeout(1):\n pass\n{FENCE}\n" + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("basedpyright", sys.executable))) + assert claim["status"] == "unresolved" + + +def test_the_claim_names_the_first_real_error_when_a_fragment_error_comes_first(tmp_path): + # `client` is an earlier fence's name: its error is a fragment, so the real error is the one the claim names + fence = f'{FENCE}python\nclient.decode("{{}}")\nx: int = "a"\n{FENCE}\n' + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + assert "Incompatible types" in claim["why"] and "not defined" not in claim["why"] + + +def test_basedpyright_top_level_await_return_and_yield_are_fragments_not_stale(tmp_path, monkeypatch): + # basedpyright words these in its own way (pyright's messages, an optional rule suffix): not mypy's wording + from dokimi.repos.l0 import typecheck as l0_typecheck + + fences = ["await asyncio.sleep(1)", "return 1", "yield 1"] + messages = ['"await" allowed only within async function', '"return" can be used only within a function', + '"yield" not allowed outside of a function or lambda'] + + def fake(argv, cwd, stdin=None): + if stdin is not None: # the fences parse + return _completed(returncode=0, stdout=json.dumps([None] * len(fences))) + if argv[-1] == "--version": + return _completed(returncode=0, stdout="basedpyright 1.31.6\n") + report = {"generalDiagnostics": [ + {"file": f"fence-{i}.py", "severity": "error", "message": message, "rule": "reportGeneralTypeIssues", + "range": {"start": {"line": 0}}} for i, message in enumerate(messages)]} + return _completed(returncode=1, stdout=json.dumps(report), stderr="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 12)) + payload = scan_repo(_tree(tmp_path, "".join(f"{FENCE}python\n{code}\n{FENCE}\n\n" for code in fences)), + parse_manifest(MANIFEST), typecheck=Check("basedpyright", sys.executable)) + claims = _fence_claims(payload) + assert [c["status"] for c in claims] == ["unresolved"] * 3 + assert all("longer example" in c["why"] for c in claims) + assert payload["typecheck"]["fail"] == 0 + + +@pytest.mark.parametrize("statement", ["yield from range(3)", "continue", "break", "nonlocal x"]) +def test_a_statement_that_needs_a_function_or_loop_around_it_is_a_fragment_not_stale(tmp_path, statement): + # mypy's wording for each outside a function or loop: the fence is a piece of a longer example + claims = _fence_claims(scan_repo(_tree(tmp_path, f"{FENCE}python\n{statement}\n{FENCE}\n"), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved"] + assert "longer example" in claims[0]["why"] + + +def test_a_top_level_return_is_a_fragment_not_stale(tmp_path): + # the real-mypy twin of the fake basedpyright case below: "return" outside function stays unresolved + claims = _fence_claims(scan_repo(_tree(tmp_path, f"{FENCE}python\nreturn 1\n{FENCE}\n"), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved"] + + +def test_a_repository_on_python_3_10_is_checked_not_unresolved(tmp_path, monkeypatch): + # mypy 2.x accepts --python-version 3.10: only 3.9 and older are unresolved, 3.10 is checked + from dokimi.repos.l0 import typecheck as l0_typecheck + + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 10)) + agents = f"{FENCE}python\nimport json\nprint(json.dumps(1))\n{FENCE}\n\n{FENCE}python\nx: int = 'a'\n{FENCE}\n" + claims = _fence_claims(scan_repo(_tree(tmp_path, agents), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["ok", "fail"] + + +def test_a_module_the_repository_environment_lacks_is_unresolved_not_stale(tmp_path): + claims = _fence_claims(scan_repo(_tree(tmp_path, f"{FENCE}python\nimport nosuchmodule_dokimi\n{FENCE}\n"), + parse_manifest(MANIFEST), typecheck=Check("mypy", sys.executable))) + assert [c["status"] for c in claims] == ["unresolved"] + assert "not type-checked" in claims[0]["why"] and "lacks" in claims[0]["why"] + + +def test_a_missing_module_with_a_real_error_fails_on_the_real_error(tmp_path): + fence = f"{FENCE}python\nimport nosuchmodule_dokimi\nx: int = 'a'\n{FENCE}\n" + [claim] = _fence_claims(scan_repo(_tree(tmp_path, fence), parse_manifest(MANIFEST), + typecheck=Check("mypy", sys.executable))) + assert claim["status"] == "fail" + assert "line 2" in claim["why"] and "nosuchmodule_dokimi" not in claim["why"] + + +def test_basedpyright_break_continue_nonlocal_and_a_missing_import_are_not_stale(tmp_path, monkeypatch): + # pyright's own messages (not run here: basedpyright is not installed in CI's test image) + from dokimi.repos.l0 import typecheck as l0_typecheck + + fences = ["break", "continue", "yield from range(3)", "nonlocal x", "import nosuchmodule_dokimi"] + messages = ['"break" can be used only within a loop', '"continue" can be used only within a loop', + '"yield from" not allowed outside of a function or lambda', + "Nonlocal declaration not allowed at module level", + 'Import "nosuchmodule_dokimi" could not be resolved'] + + def fake(argv, cwd, stdin=None): + if stdin is not None: # the fences parse + return _completed(returncode=0, stdout=json.dumps([None] * len(fences))) + if argv[-1] == "--version": + return _completed(returncode=0, stdout="basedpyright 1.31.6\n") + report = {"generalDiagnostics": [ + {"file": f"fence-{i}.py", "severity": "error", "message": message, "rule": "reportGeneralTypeIssues", + "range": {"start": {"line": 0}}} for i, message in enumerate(messages)]} + return _completed(returncode=1, stdout=json.dumps(report), stderr="") + + monkeypatch.setattr(l0_typecheck, "_run_tool", fake) + monkeypatch.setattr(l0_typecheck, "python_version", lambda check: (3, 12)) + payload = scan_repo(_tree(tmp_path, "".join(f"{FENCE}python\n{code}\n{FENCE}\n\n" for code in fences)), + parse_manifest(MANIFEST), typecheck=Check("basedpyright", sys.executable)) + assert [c["status"] for c in _fence_claims(payload)] == ["unresolved"] * len(fences) + assert payload["typecheck"]["fail"] == 0 + + +def test_the_fence_parser_runs_where_ast_parse_has_no_type_comments_keyword(monkeypatch): + # Python 3.7's ast.parse takes no type_comments keyword: the fence is parsed without it, not a crash + import ast + import io + + from dokimi.repos.l0 import typecheck as l0_typecheck + + real_parse = ast.parse + + def parse_3_7(source, filename="", mode="exec", *, feature_version=None): + return real_parse(source, filename, mode, feature_version=feature_version) + + monkeypatch.setattr(ast, "parse", parse_3_7) + monkeypatch.setattr(sys, "stdin", io.StringIO(json.dumps(["x: int = 1", "f("]))) + out = io.StringIO() + monkeypatch.setattr(sys, "stdout", out) + exec(compile(l0_typecheck._PARSE, "", "exec"), {"__name__": "__main__"}) # nosec B102 - our own constant + valid, invalid = json.loads(out.getvalue()) + assert valid is None + assert invalid is not None + + +def test_the_scan_command_refuses_a_python_env_that_is_a_directory(tmp_path, capsys): + # os.access(X_OK) is true for a directory: only a regular file can be the interpreter + args = build_parser().parse_args(["repo", "scan", str(tmp_path), "--typecheck", "--python-env", str(tmp_path)]) + from dokimi.commands.repo import _typecheck_or_exit + + with pytest.raises(SystemExit) as exc: + _typecheck_or_exit(args) + assert exc.value.code == 2 + assert "not an executable interpreter" in capsys.readouterr().err diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py new file mode 100644 index 000000000..7a24b6c9b --- /dev/null +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -0,0 +1,672 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The built-in grader ``dokimi:typecheck`` (contract §4.5): listed files type-check clean under the pinned mypy.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import mypy.version +import pytest + +from dokimi.repos.graders import main, typecheck + +PINNED = mypy.version.__version__ + + +def _task(**over) -> dict: + task = {"id": "typed", "grader": "dokimi:typecheck", "version": PINNED, "paths": ["pkg/api.py"]} + task.update(over) + return task + + +def _run(tmp_path: Path, files: dict[str, str], task: dict) -> Path: + """A run directory whose workspace holds ``files``; returns the run directory.""" + run = tmp_path / "run" + for name, text in files.items(): + path = run / "workspace" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + (run / "task.json").write_text(json.dumps(task), encoding="utf-8") + return run + + +CLEAN = "def add(a: int, b: int) -> int:\n return a + b\n" +WRONG = "def add(a: int, b: int) -> int:\n return f'{a}{b}'\n" + + +def test_listed_file_that_type_checks_passes(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_errors"] == 0 + assert result["tool_version"] == PINNED + + +def test_grading_leaves_the_recursion_limit_as_it_was(tmp_path): + # mypy's main raises the process-wide recursion limit; in-process, that leaked into later code. + before = sys.getrecursionlimit() + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + typecheck.grade(_task(), tmp_path, "", run) + assert sys.getrecursionlimit() == before + + +def test_listed_file_with_a_type_error_fails_and_names_the_error(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + assert result["errors"][0].startswith("pkg/api.py:2:") + assert "[return-value]" in result["errors"][0] + + +def test_an_error_in_an_unlisted_module_is_not_counted(tmp_path): + other = "def broken() -> int:\n return 'no'\n" + api = "from pkg.other import broken\n\ndef use() -> int:\n return broken()\n" + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/other.py": other, "pkg/api.py": api}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_errors"] == 0 + + +def test_a_listed_file_that_is_missing_fails_without_running_the_tool(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": ""}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["missing"] == ["pkg/api.py"] + assert result["n_errors"] == 0 + + +def test_a_pinned_version_other_than_the_image_is_a_grader_error(tmp_path): + run = _run(tmp_path, {"pkg/api.py": CLEAN}, _task(version="0.0.1")) + with pytest.raises(ValueError, match="pins 0.0.1"): + typecheck.grade(_task(version="0.0.1"), tmp_path, "", run) + + +def test_the_entry_point_turns_a_grader_error_into_pass_none_with_errors(tmp_path): + run = _run(tmp_path, {"pkg/api.py": CLEAN}, _task(version="0.0.1")) + out = main.grade("typecheck", tmp_path, run) + assert out["pass"] is None + assert out["errors"] and "pins 0.0.1" in out["errors"][0] + assert json.loads((run / "grade.json").read_text(encoding="utf-8"))["pass"] is None + + +def test_the_entry_point_writes_the_metrics_that_the_gate_reads(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + out = main.grade("typecheck", tmp_path, run) + assert out["pass"] is False + assert out["metrics"]["n_errors"] == 1 + assert out["metrics"]["n_missing"] == 0 + assert out["details"]["grader_version"] == typecheck.VERSION + + +@pytest.mark.parametrize("over", [ + {"tool": "basedpyright"}, + {"version": ""}, + {"version": None}, + {"paths": []}, + {"paths": ["/etc/passwd"]}, + {"paths": ["../outside.py"]}, + {"paths": "pkg/api.py"}, + {"paths": ["odd\nname/api.py"]}, # a line separator splits the name in mypy's output + {"paths": ["odd\u2028name/api.py"]}, +]) +def test_task_problems_refuse_bad_fields(over): + task = _task() + task.update(over) + assert typecheck.task_problems(task) + + +def test_task_problems_accept_a_well_formed_task(): + assert typecheck.task_problems(_task()) == [] + + +def test_the_manifest_checks_the_task_fields_at_parse_time(): + from dokimi.repos.manifest import _builtin_problems + + raw = _task(version="") + assert _builtin_problems(None, "tasks/typed", raw, "typecheck") + assert _builtin_problems(None, "tasks/typed", _task(), "typecheck") == [] + + +def test_the_grader_script_loads_its_siblings_by_name(monkeypatch): + import runpy + import sys + + from dokimi.repos.graders import main as grader_main + + monkeypatch.setattr(sys, "path", list(sys.path)) + for name in ("review", "typecheck"): + monkeypatch.delitem(sys.modules, name, raising=False) + script = runpy.run_path(grader_main.__file__, run_name="grader_script") # a script: no package + assert script["GRADERS"]["typecheck"] is script["typecheck"].GRADERS["typecheck"] + assert script["VERSIONS"]["typecheck"] == typecheck.VERSION + + +def test_the_subjects_mypy_ini_cannot_suppress_a_listed_file(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG, + "mypy.ini": "[mypy-pkg.api]\nignore_errors = True\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + +def test_a_colored_mypy_image_still_fails_a_listed_file_with_a_type_error(tmp_path, monkeypatch): + # the image's env is kept for the grade: MYPY_FORCE_COLOR puts ANSI escapes around the file name and the marker + monkeypatch.setenv("MYPY_FORCE_COLOR", "1") + run = _run(tmp_path, {"pkg/api.py": WRONG}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + +def test_a_listed_path_with_a_colon_is_matched(): + out = typecheck._errors("a:b.py:3: error: boom [misc]", {"a:b.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("a:b.py", 3)] + + +def test_a_listed_path_containing_the_error_separator_is_matched(): + # the name itself holds ": error: "; the first separator in the line is not the end of the name + out = typecheck._errors("pkg/a: error: b.py:2: error: incompatible return value [return-value]", + {"pkg/a: error: b.py"}, "") + assert [(e["file"], e["line"], e["code"]) for e in out] == [("pkg/a: error: b.py", 2, "return-value")] + + +def test_a_listed_name_that_is_a_prefix_of_another_does_not_take_its_errors(): + out = typecheck._errors("pkg/a: error: b.py:2: error: boom", {"pkg/a", "pkg/a: error: b.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("pkg/a: error: b.py", 2)] + + +def test_a_mypy_spelling_of_a_listed_file_is_normalised_before_matching(): + out = typecheck._errors("./pkg//a.py:3: error: boom [misc]", {"pkg/a.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("pkg/a.py", 3)] + + +def test_a_line_less_error_on_a_listed_file_is_counted(): + out = typecheck._errors("pkg/a.py: error: Duplicate module named 'a'", {"pkg/a.py"}, "") + assert [(e["file"], e["line"]) for e in out] == [("pkg/a.py", 0)] + + +NAMESPACE_API = "from pkg.other import broken\n\ndef use() -> int:\n return broken()\n" +NAMESPACE_OTHER = "def broken() -> int:\n return 1\n" + + +def test_a_relative_run_directory_is_graded(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + result = typecheck.grade(_task(), tmp_path, "", Path("run")) + assert result["pass"] is True + assert result["missing"] == [] + + +def test_a_namespace_package_resolves_from_the_workspace_root(tmp_path): + # both files listed: a listed other.py must be named pkg.other, the name api.py imports it by + task = _task(paths=["pkg/api.py", "pkg/other.py"]) + run = _run(tmp_path, {"pkg/api.py": NAMESPACE_API, "pkg/other.py": NAMESPACE_OTHER}, task) + result = typecheck.grade(task, tmp_path, "", run) + assert result["pass"] is True, result["errors"] + + +def test_a_src_layout_package_resolves_when_both_files_are_listed(tmp_path): + # a regular package under src/: pkg.x is imported from src/, so src/ is a root with the workspace + files = {"src/pkg/__init__.py": "", "src/pkg/x.py": "val: int = 1\n", + "src/pkg/y.py": "from pkg.x import val\n\nprint(val + 1)\n"} + task = _task(paths=["src/pkg/x.py", "src/pkg/y.py"]) + run = _run(tmp_path, files, task) + result = typecheck.grade(task, tmp_path, "", run) + assert result["pass"] is True, result["errors"] + + +@pytest.mark.skipif(sys.platform == "win32", reason="a directory named src:lib cannot be created on Windows") +@pytest.mark.parametrize("sep", [",", ":"]) +def test_a_source_root_with_a_mypy_path_delimiter_is_a_task_error(tmp_path, sep): + files = {f"src{sep}lib/pkg/__init__.py": "", f"src{sep}lib/pkg/x.py": "val: int = 1\n", + f"src{sep}lib/pkg/y.py": "from pkg.x import val\n\nprint(val + 1)\n"} + task = _task(paths=[f"src{sep}lib/pkg/x.py", f"src{sep}lib/pkg/y.py"]) + run = _run(tmp_path, files, task) + with pytest.raises(ValueError, match="mypy_path"): + typecheck.grade(task, tmp_path, "", run) + + +def test_a_blocking_syntax_error_in_a_listed_file_fails_the_grade(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "def broken(:\n pass\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] >= 1 + assert result["errors"][0].startswith("pkg/api.py:1:") + + +def test_mypy_failing_without_a_diagnostic_is_a_grader_error(tmp_path, monkeypatch): + run = _run(tmp_path, {"pkg/api.py": CLEAN}, _task()) + import mypy.api + + monkeypatch.setattr(mypy.api, "run", lambda args: ("", "cannot start", 2)) + with pytest.raises(RuntimeError, match="mypy exited 2"): + typecheck.grade(_task(), tmp_path, "", run) + + +def test_the_reference_directory_repairs_the_workspace_before_grading(tmp_path): + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + assert typecheck.grade(_task(), tmp_path, "", run)["pass"] is False + assert typecheck.oracle(_task(), task_dir, run) == "" + assert typecheck.grade(_task(), tmp_path, "", run)["pass"] is True + + +def test_the_entry_point_oracle_repairs_the_workspace_too(tmp_path): + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + main.oracle("typecheck", task_dir, run) + assert main.grade("typecheck", task_dir, run)["pass"] is True + + +def test_a_blocking_error_in_an_imported_module_fails_the_listed_file(tmp_path): + # mypy stops at the syntax error and never reaches api.py, so the listed file is not checked + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/other.py": "def broken(:\n pass\n", + "pkg/api.py": "from pkg.other import broken\n\ndef use() -> int:\n return 'x'\n"}, + _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] >= 1 + + +def test_an_external_mypypath_cannot_supply_a_module_the_image_lacks(tmp_path, monkeypatch): + stubs = tmp_path / "stubs" # outside the workspace, as an image's or the environment's MYPYPATH would be + stubs.mkdir() + (stubs / "unavailable.pyi").write_text("def value() -> int: ...\n", encoding="utf-8") + monkeypatch.setenv("MYPYPATH", str(stubs)) + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "from unavailable import value\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert any("unavailable" in error for error in result["errors"]) + assert os.environ["MYPYPATH"] == str(stubs) # restored for the caller + + +def test_a_listed_file_importing_an_unlisted_link_to_a_hidden_repair_is_refused(tmp_path): + # api.py stays a regular file and imports pkg.repair, which is not listed and is a link to a clean repair outside + # the workspace: mypy follows the import through the link and would pass api.py on a module this run cannot see + outside = tmp_path / "outside" + outside.mkdir() + (outside / "repair.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "from pkg.repair import add\nassert add(1, 2) == 3\n"}, + _task()) + (run / "workspace" / "pkg" / "repair.py").symlink_to(outside / "repair.py") + with pytest.raises(ValueError, match="pkg/repair.py is a symbolic link to a file outside the workspace"): + typecheck.grade(_task(), tmp_path, "", run) + + +@pytest.mark.parametrize("linked", ["file", "directory", "workspace"]) +def test_a_listed_path_reached_through_a_link_is_missing_not_graded(tmp_path, linked): + # The agent links the listed file (or its directory, or the workspace) to a clean repair outside the workspace: + # is_file() and mypy would follow the link and pass it. The link is refused, so the file is missing. + outside = tmp_path / "outside" + (outside / "pkg").mkdir(parents=True) + (outside / "pkg" / "__init__.py").write_text("", encoding="utf-8") + (outside / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + workspace = tmp_path / "run" / "workspace" + if linked == "file": + (workspace / "pkg").mkdir(parents=True) + (workspace / "pkg" / "__init__.py").write_text("", encoding="utf-8") + (workspace / "pkg" / "api.py").symlink_to(outside / "pkg" / "api.py") + elif linked == "directory": + workspace.mkdir(parents=True) + (workspace / "pkg").symlink_to(outside / "pkg", target_is_directory=True) + else: + workspace.parent.mkdir(parents=True) + workspace.symlink_to(outside, target_is_directory=True) + result = typecheck.grade(_task(), tmp_path, "", tmp_path / "run") + assert result["pass"] is False and result["n_missing"] == 1 and result["missing"] == ["pkg/api.py"] + + +@pytest.mark.parametrize("linked", ["file", "directory", "workspace"]) +def test_the_oracle_refuses_to_write_through_a_symbolic_link(tmp_path, linked): + outside = tmp_path / "outside" + (outside / "pkg").mkdir(parents=True) + sentinel = outside / "pkg" / "api.py" + sentinel.write_text("ORIGINAL", encoding="utf-8") + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text("REFERENCE", encoding="utf-8") + run = tmp_path / "run" + run.mkdir() + workspace = run / "workspace" + if linked == "file": + (workspace / "pkg").mkdir(parents=True) + (workspace / "pkg" / "api.py").symlink_to(sentinel) + elif linked == "directory": + workspace.mkdir() + (workspace / "pkg").symlink_to(outside / "pkg", target_is_directory=True) + else: + workspace.symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert sentinel.read_text(encoding="utf-8") == "ORIGINAL" + + +def test_the_oracle_refuses_a_linked_reference_directory_with_a_linked_destination(tmp_path): + # copytree follows a linked reference directory and writes its entries through a linked destination + # file; the walk of the reference does not see inside the link, so the link itself is refused. + repairs = tmp_path / "repairs" + repairs.mkdir() + (repairs / "api.py").write_text("REFERENCE", encoding="utf-8") + outside = tmp_path / "outside" + outside.mkdir() + sentinel = outside / "api.py" + sentinel.write_text("ORIGINAL", encoding="utf-8") + task_dir = tmp_path / "task" + (task_dir / "reference").mkdir(parents=True) + (task_dir / "reference" / "pkg").symlink_to(repairs, target_is_directory=True) + run = tmp_path / "run" + (run / "workspace" / "pkg").mkdir(parents=True) + (run / "workspace" / "pkg" / "api.py").symlink_to(sentinel) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert sentinel.read_text(encoding="utf-8") == "ORIGINAL" + + +def test_the_oracle_copies_the_reference_when_no_link_is_on_the_way(tmp_path): + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg").mkdir(parents=True) + (task_dir / "reference" / "pkg" / "api.py").write_text(CLEAN, encoding="utf-8") + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + typecheck.oracle(_task(), task_dir, run) + assert (run / "workspace" / "pkg" / "api.py").read_text(encoding="utf-8") == CLEAN + + +def test_the_oracle_refuses_an_empty_reference_directory_created_through_a_link(tmp_path): + outside = tmp_path / "outside" + outside.mkdir() + task_dir = tmp_path / "task" + (task_dir / "reference" / "pkg" / "sub").mkdir(parents=True) # no file under it + run = tmp_path / "run" + (run / "workspace").mkdir(parents=True) + (run / "workspace" / "pkg").symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert not (outside / "sub").exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not keep the 0700 mode this checks") +def test_the_oracle_refuses_a_linked_workspace_even_with_an_empty_reference(tmp_path): + # no entry in reference/, so the walk checks nothing; copytree still copies the mode onto the linked root + outside = tmp_path / "outside" + outside.mkdir(mode=0o700) + task_dir = tmp_path / "task" + (task_dir / "reference").mkdir(parents=True) + run = tmp_path / "run" + run.mkdir() + (run / "workspace").symlink_to(outside, target_is_directory=True) + with pytest.raises(ValueError, match="symbolic link"): + typecheck.oracle(_task(), task_dir, run) + assert outside.stat().st_mode & 0o777 == 0o700 + + +def test_linked_on_the_way_sees_a_link_in_the_middle_of_a_path(tmp_path): + # the oracle's walk also reaches this through the reference's directories; the helper is pinned on its own + outside = tmp_path / "outside" + (outside / "c").mkdir(parents=True) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "a").symlink_to(outside, target_is_directory=True) + assert typecheck._linked_on_the_way(workspace, workspace / "a" / "b" / "c.py") is True + assert typecheck._linked_on_the_way(workspace, workspace / "d" / "c.py") is False + + +def test_check_task_runs_no_oracle_step_for_a_typecheck_task_without_reference(tmp_path, monkeypatch): + # the gate in check_task itself: a mutated `_oracle_applies(...) and` would run the oracle here + from types import SimpleNamespace + + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": "1", "paths": ["pkg/api.py"]} + ran = [] + monkeypatch.setattr(selfcheck, "_host_reason", lambda ctx, task, work_dir, host: (None, None, None)) + monkeypatch.setattr(selfcheck, "_check_step", + lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) + monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + res = selfcheck.check_task(ctx, Task("t", raw, bare), tmp_path / "work") + assert ran == [("null", True), ("oracle", False)] + assert res["verdict"] == "ok" + + +def test_check_task_runs_the_oracle_step_for_a_typecheck_task_with_reference(tmp_path, monkeypatch): + # the mirror of the test above: a `False and (...)` gate would skip the oracle of a task that has a repair + from types import SimpleNamespace + + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + with_reference = tmp_path / "with" + (with_reference / "reference").mkdir(parents=True) + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": "1", "paths": ["pkg/api.py"]} + ran = [] + monkeypatch.setattr(selfcheck, "_host_reason", lambda ctx, task, work_dir, host: (None, None, None)) + monkeypatch.setattr(selfcheck, "_check_step", + lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) + monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + selfcheck.check_task(ctx, Task("t", raw, with_reference), tmp_path / "work") + assert ran == [("null", True), ("oracle", True)] + + +def test_a_typecheck_task_without_reference_runs_no_oracle_step(tmp_path): + # reference/ is optional: without it the grade reads the workspace as it is, and the self-check has no + # repair to apply, so it runs the null step alone (the untouched listed files must fail the gate). + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + with_reference = tmp_path / "with" + (with_reference / "reference").mkdir(parents=True) + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": "1", "paths": ["pkg/api.py"]} + assert selfcheck._oracle_applies(Task("t", raw, bare)) is False + assert selfcheck._oracle_applies(Task("t", raw, with_reference)) is True + assert selfcheck._oracle_applies(Task("t", {"grader": "dokimi:review_seeds"}, bare)) is True + assert selfcheck._oracle_applies(Task("t", {}, bare)) is True + + +@pytest.mark.parametrize("source", [ + "def add(a: int, b: int) -> int:\n return f'{a}{b}' # type: ignore\n", + "# mypy: ignore-errors\n" + WRONG, + "# mypy: disable-error-code=return-value\n" + WRONG, +]) +def test_a_suppression_in_a_listed_file_fails_the_grade(tmp_path, source): + # the agent must fix the code, not hide the error: a listed file with a suppression never passes + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_a_stub_in_the_workspace_that_hides_a_module_fails_the_grade(tmp_path): + # mypy_path puts the workspace first: a stub there replaces json, and its __getattr__ hides json.nosuch() + stub = "from typing import Any\n\ndef __getattr__(name: str) -> Any: ...\n" + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "import json\n\nprint(json.nosuch())\n", + "json.pyi": stub}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_an_error_in_an_unannotated_function_still_fails_the_grade(tmp_path): + # without --check-untyped-defs mypy skips the body: deleting the annotations would turn a fail into a pass + source = "def add(a, b):\n n: int = 'x'\n return a + n\n" + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + +@pytest.mark.parametrize("source", [ + "from typing import no_type_check\n\n@no_type_check\n" + WRONG, + "import typing\n\n@typing.no_type_check\n" + WRONG, + "from typing import no_type_check as ok\n\n@ok\n" + WRONG, + "from typing import no_type_check\n\n@no_type_check\ndef add(:\n return 1\n", # not tokenizable: by line +]) +def test_no_type_check_in_a_listed_file_is_a_suppression(tmp_path, source): + # mypy skips a function under @no_type_check: the error is hidden, not fixed + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] >= 1 + + +def test_no_type_check_in_a_string_is_not_a_suppression(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": 'NOTE = "no_type_check"\n' + CLEAN}, _task()) + assert typecheck.grade(_task(), tmp_path, "", run)["pass"] is True + + +@pytest.mark.parametrize("shadow", [ + {"json.py": "from typing import Any\n\ndef __getattr__(name: str) -> Any: ...\n"}, + {"json.py": "def nosuch() -> None: ...\n"}, + {"json/__init__.py": "def nosuch() -> None: ...\n"}, +]) +def test_a_workspace_module_named_as_a_standard_library_one_fails_the_grade(tmp_path, shadow): + # the .py twin of a json.pyi: the workspace is first on mypy_path, so it replaces json and hides json.nosuch() + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "import json\n\nprint(json.nosuch())\n", **shadow}, + _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +@pytest.mark.parametrize("shadow", [ + {"mypy.py": "def __getattr__(name: str) -> int: ...\n"}, + {"mypy/__init__.py": "def __getattr__(name: str) -> int: ...\n"}, +]) +def test_a_workspace_module_named_as_an_installed_package_fails_the_grade(tmp_path, shadow): + # mypy is in the image's site-packages: a workspace mypy.py would stand in for it, as json.py does for json + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "import mypy\n\nprint(mypy.nosuch)\n", **shadow}, + _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_image_provides_finds_an_installed_module_and_not_a_workspace_name(): + assert typecheck._image_provides("mypy") is True + assert typecheck._image_provides("pkg") is False + assert typecheck._image_provides("not-an-identifier") is False + assert typecheck._image_provides("mypy.version") is False # a dotted name is not looked up + + +def test_a_standard_library_name_on_a_source_root_fails_the_grade(tmp_path): + files = {"src/pkg/__init__.py": "", "src/pkg/api.py": "import json\n\nprint(json.nosuch())\n", + "src/json.py": "def nosuch() -> None: ...\n"} + run = _run(tmp_path, files, _task(paths=["src/pkg/api.py"])) + result = typecheck.grade(_task(paths=["src/pkg/api.py"]), tmp_path, "", run) + assert result["pass"] is False + assert "src/json.py: mypy is silenced here (fix the code instead)" in result["errors"] + + +def test_a_clean_listed_file_has_no_suppressions(tmp_path): + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": CLEAN}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_suppressions"] == 0 + + +@pytest.mark.parametrize("source", [ + 'MESSAGE = "# type: ignore"\n' + CLEAN, + '"""Show the marker:\n\n # mypy: ignore-errors\n x = 1 # type: ignore\n"""\n' + CLEAN, +]) +def test_a_marker_inside_a_string_is_not_a_suppression(tmp_path, source): + # mypy reads comments only: the text of a string or a docstring that shows a marker silences nothing + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is True + assert result["n_suppressions"] == 0 + + +def test_a_genuine_suppression_after_a_docstring_is_named_on_its_own_line(tmp_path): + source = '"""Show:\n\n # mypy: ignore-errors\n"""\nVALUE: int = 1 # type: ignore\n' + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": source}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + assert result["errors"] == ["pkg/api.py:5: mypy is silenced here (fix the code instead)"] + + +def test_a_source_that_does_not_tokenize_still_counts_its_marker(tmp_path): + # no comment stream to read: the marker is found line by line, and the file fails mypy's parse anyway + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": "def add(:\n return 1 # type: ignore\n"}, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_suppressions"] == 1 + + +def test_a_type_error_is_graded_from_any_working_directory(tmp_path, monkeypatch): + # mypy prints names relative to the cwd: "workspace/pkg/api.py" matched no listed name, so the error was dropped + run = _run(tmp_path, {"pkg/__init__.py": "", "pkg/api.py": WRONG}, _task()) + monkeypatch.chdir(run) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["pass"] is False + assert result["n_errors"] == 1 + + +def test_a_windows_diagnostic_path_matches_the_listed_name(): + # mypy on Windows prints backslash paths; the listed name is POSIX, so the error is still this file's + stdout = "C:\\run\\workspace\\pkg\\api.py:2: error: boom\n" + errors = typecheck._errors(stdout, {"pkg/api.py"}, "C:\\run\\workspace\\", sep="\\") + assert errors == [{"file": "pkg/api.py", "line": 2, "code": "", "message": "boom"}] + + +def test_a_backslash_in_a_posix_name_is_a_literal_character(): + # only the native separator is converted: on POSIX a file may be named with a backslash + errors = typecheck._errors("ws/a\\b.py:1: error: boom\n", {"a\\b.py"}, "ws/", sep="/") + assert [e["file"] for e in errors] == ["a\\b.py"] + + +def test_two_listed_names_for_one_file_are_a_task_error(): + # "a.py" and "./a.py" are one file: mypy reports it as a duplicate module, so the task could never pass + problems = typecheck.task_problems(_task(paths=["pkg/api.py", "./pkg/api.py"])) + assert any("distinct" in p for p in problems) + + +def test_a_typecheck_task_without_reference_whose_listed_path_is_missing_is_a_problem(tmp_path): + # with no reference the null grade is the only step: a typo'd path fails it, so no run could ever pass + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + raw = {"grader": "dokimi:typecheck", "tool": "mypy", "version": PINNED, "paths": ["src/pkg/modle.py"]} + rd = tmp_path / "rd" + rd.mkdir() + # the shape main.grade writes: a list is under details, not at the top level + mets, det = main.split_result({"pass": False, "n_missing": 1, "missing": ["src/pkg/modle.py"]}) + (rd / "grade.json").write_text(json.dumps({"pass": False, "metrics": mets, "details": det}), encoding="utf-8") + grade = {"status": "graded", "success": False, "score": None, "score_only": False, "pass": False} + problems = selfcheck._verdict_problems(Task("t", raw, bare), "null", grade, rd) + assert any("missing" in p and "src/pkg/modle.py" in p for p in problems) + + +def test_a_typecheck_task_without_reference_whose_workspace_already_silences_mypy_is_a_problem(tmp_path): + # a repository stub fails every grade, so with no reference even a run that repairs the code cannot pass + from dokimi.repos import selfcheck + from dokimi.repos.workspace import Task + + bare = tmp_path / "bare" + bare.mkdir() + files = {"pkg/__init__.py": "", "pkg/api.py": CLEAN, "pkg/_ext.pyi": "def f() -> int: ...\n"} + run = _run(tmp_path, files, _task()) + result = typecheck.grade(_task(), tmp_path, "", run) + assert result["n_errors"] == 0 and result["n_suppressions"] == 1 # repaired, and still failing + mets, det = main.split_result(result) + (run / "grade.json").write_text(json.dumps({"pass": False, "metrics": mets, "details": det}), encoding="utf-8") + grade = {"status": "graded", "success": False, "score": None, "score_only": False, "pass": False} + problems = selfcheck._verdict_problems(Task("t", _task(tool="mypy"), bare), "null", grade, run) + assert any("silences mypy" in p for p in problems)