From 2a6fd78e00c33a6d4b0693887e0754b34a0bebec Mon Sep 17 00:00:00 2001 From: David Peter Date: Tue, 3 Nov 2026 11:23:21 +0100 Subject: [PATCH 1/2] Fix Zuban benchmark configuration isolation --- tests/test_typecheck_benchmark_configs.py | 13 ++++++------- tests/test_typecheck_benchmark_isolation.py | 5 +++-- typecheck_benchmark/daily_runner.py | 16 +++++++--------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/tests/test_typecheck_benchmark_configs.py b/tests/test_typecheck_benchmark_configs.py index 7ee3a5b8..7915f997 100644 --- a/tests/test_typecheck_benchmark_configs.py +++ b/tests/test_typecheck_benchmark_configs.py @@ -149,7 +149,7 @@ def test_check_paths_in_files(self, tmp_path: Path) -> None: def test_returns_benchmark_ini_path(self, tmp_path: Path) -> None: path = _write_dummy_zuban_config(tmp_path) - assert path.name == "mypy.benchmark.ini" + assert path.name == "zuban.benchmark.ini" # --------------------------------------------------------------------------- @@ -241,17 +241,16 @@ def test_mypy_with_check_paths_no_extra_args(self, tmp_path: Path) -> None: content = (tmp_path / "mypy.benchmark.ini").read_text() assert "files = src\n" in content - def test_zuban_no_check_paths_passes_dot(self, tmp_path: Path) -> None: + def test_zuban_no_check_paths_passes_package_path(self, tmp_path: Path) -> None: cmd = self._run_and_capture_cmd("zuban", tmp_path) - # zuban ignores --config-file, so paths are always positional - assert cmd == ["zuban", "check", "."] + assert cmd == ["zuban", "check", "--config-file", str(tmp_path / "zuban.benchmark.ini"), str(tmp_path)] - def test_zuban_with_check_paths_passes_positional(self, tmp_path: Path) -> None: + def test_zuban_with_check_paths_uses_config(self, tmp_path: Path) -> None: src = tmp_path / "src" src.mkdir() cmd = self._run_and_capture_cmd("zuban", tmp_path, [src]) - # zuban ignores --config-file, so check paths are positional args - assert cmd == ["zuban", "check", "src"] + assert cmd == ["zuban", "check", "--config-file", str(tmp_path / "zuban.benchmark.ini")] + assert "files = src\n" in (tmp_path / "zuban.benchmark.ini").read_text() def test_unknown_checker_returns_error(self, tmp_path: Path) -> None: with patch("typecheck_benchmark.daily_runner.run_process_with_timeout") as mock: diff --git a/tests/test_typecheck_benchmark_isolation.py b/tests/test_typecheck_benchmark_isolation.py index ddb8fb92..bc40a276 100644 --- a/tests/test_typecheck_benchmark_isolation.py +++ b/tests/test_typecheck_benchmark_isolation.py @@ -95,10 +95,11 @@ def test_ty_gets_environment_dir(self, tmp_path: Path) -> None: idx = cmd.index("--python") assert cmd[idx + 1] == str(venv) - def test_zuban_flag_precedes_paths(self, tmp_path: Path) -> None: + def test_zuban_config_and_interpreter_flags(self, tmp_path: Path) -> None: venv = tmp_path / "venv" cmd, _ = self._run("zuban", tmp_path, venv) - assert cmd == ["zuban", "check", "--python-executable", str(venv_python(venv)), "."] + assert cmd == ["zuban", "check", "--config-file", str(tmp_path / "zuban.benchmark.ini"), + "--python-executable", str(venv_python(venv)), str(tmp_path)] @pytest.mark.parametrize("checker", ["pyright", "pyrefly", "ty", "mypy", "zuban"]) def test_virtual_env_set(self, checker: str, tmp_path: Path) -> None: diff --git a/typecheck_benchmark/daily_runner.py b/typecheck_benchmark/daily_runner.py index e9344d68..423afff0 100644 --- a/typecheck_benchmark/daily_runner.py +++ b/typecheck_benchmark/daily_runner.py @@ -578,7 +578,7 @@ def install_deps( # To ensure consistent benchmarking across packages, we write minimal config files # that embed the check_paths (from install_envs.json) directly into each checker's # native config format, then pass the config via CLI flag to override package-level -# config. This means paths are NOT passed as CLI arguments — they live in the config: +# config. Paths live in the config: # # pyright -- pyrightconfig.json with "include": [paths] (written in-place) # mypy -- [mypy] section with files = path1, path2 via --config-file @@ -651,11 +651,12 @@ def _write_dummy_zuban_config( package_path: Path, check_paths: list[str] | None = None, ) -> Path: """Write minimal mypy-style config for zuban benchmarking.""" - config_path = package_path / "mypy.benchmark.ini" + config_path = package_path / "zuban.benchmark.ini" with open(config_path, "w", encoding="utf-8") as f: f.write("[mypy]\n") if check_paths: f.write(f"files = {', '.join(check_paths)}\n") + f.write("check_untyped_defs = True\n") return config_path @@ -722,15 +723,12 @@ def run_checker( cmd.append(str(package_path)) cwd = package_path elif checker == "zuban": - # zuban ignores files= from --config-file, so we must always pass - # check paths as explicit positional arguments. - cmd = ["zuban", "check"] + config_path = _write_dummy_zuban_config(package_path, rel_paths) + cmd = ["zuban", "check", "--config-file", str(config_path)] if python: cmd.extend(["--python-executable", python]) - if rel_paths: - cmd.extend(rel_paths) - else: - cmd.append(".") + if not rel_paths: + cmd.append(str(package_path)) cwd = package_path else: return { From d3ae7096a3b870afaee081717a859b847f60ee02 Mon Sep 17 00:00:00 2001 From: David Peter Date: Thu, 5 Nov 2026 10:11:00 +0100 Subject: [PATCH 2/2] Test Zuban config checks untyped functions --- tests/test_typecheck_benchmark_configs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_typecheck_benchmark_configs.py b/tests/test_typecheck_benchmark_configs.py index 7915f997..a25fd17b 100644 --- a/tests/test_typecheck_benchmark_configs.py +++ b/tests/test_typecheck_benchmark_configs.py @@ -141,11 +141,13 @@ def test_default_no_files_line(self, tmp_path: Path) -> None: content = path.read_text() assert "[mypy]\n" in content assert "files" not in content + assert "check_untyped_defs = True\n" in content def test_check_paths_in_files(self, tmp_path: Path) -> None: path = _write_dummy_zuban_config(tmp_path, ["src", "lib"]) content = path.read_text() assert "files = src, lib\n" in content + assert "check_untyped_defs = True\n" in content def test_returns_benchmark_ini_path(self, tmp_path: Path) -> None: path = _write_dummy_zuban_config(tmp_path)