Create a dummy Zuban configuration for isolation (same as Mypy)

tests/test_typecheck_benchmark_configs.py

@@ -141,15 +141,17 @@ def test_default_no_files_line(self, tmp_path: Path) -> None:
         content = path.read_text()
         assert "[mypy]\n" in content
         assert "files" not in content
+        assert "check_untyped_defs = True\n" in content
 
     def test_check_paths_in_files(self, tmp_path: Path) -> None:
         path = _write_dummy_zuban_config(tmp_path, ["src", "lib"])
         content = path.read_text()
         assert "files = src, lib\n" in content
+        assert "check_untyped_defs = True\n" in content
 
     def test_returns_benchmark_ini_path(self, tmp_path: Path) -> None:
         path = _write_dummy_zuban_config(tmp_path)
-        assert path.name == "mypy.benchmark.ini"
+        assert path.name == "zuban.benchmark.ini"
 
 
 # ---------------------------------------------------------------------------
@@ -241,17 +243,16 @@ def test_mypy_with_check_paths_no_extra_args(self, tmp_path: Path) -> None:
         content = (tmp_path / "mypy.benchmark.ini").read_text()
         assert "files = src\n" in content
 
-    def test_zuban_no_check_paths_passes_dot(self, tmp_path: Path) -> None:
+    def test_zuban_no_check_paths_passes_package_path(self, tmp_path: Path) -> None:
         cmd = self._run_and_capture_cmd("zuban", tmp_path)
-        # zuban ignores --config-file, so paths are always positional
-        assert cmd == ["zuban", "check", "."]
+        assert cmd == ["zuban", "check", "--config-file", str(tmp_path / "zuban.benchmark.ini"), str(tmp_path)]
 
-    def test_zuban_with_check_paths_passes_positional(self, tmp_path: Path) -> None:
+    def test_zuban_with_check_paths_uses_config(self, tmp_path: Path) -> None:
         src = tmp_path / "src"
         src.mkdir()
         cmd = self._run_and_capture_cmd("zuban", tmp_path, [src])
-        # zuban ignores --config-file, so check paths are positional args
-        assert cmd == ["zuban", "check", "src"]
+        assert cmd == ["zuban", "check", "--config-file", str(tmp_path / "zuban.benchmark.ini")]
+        assert "files = src\n" in (tmp_path / "zuban.benchmark.ini").read_text()
 
     def test_unknown_checker_returns_error(self, tmp_path: Path) -> None:
         with patch("typecheck_benchmark.daily_runner.run_process_with_timeout") as mock:

tests/test_typecheck_benchmark_isolation.py

@@ -95,10 +95,11 @@ def test_ty_gets_environment_dir(self, tmp_path: Path) -> None:
         idx = cmd.index("--python")
         assert cmd[idx + 1] == str(venv)
 
-    def test_zuban_flag_precedes_paths(self, tmp_path: Path) -> None:
+    def test_zuban_config_and_interpreter_flags(self, tmp_path: Path) -> None:
         venv = tmp_path / "venv"
         cmd, _ = self._run("zuban", tmp_path, venv)
-        assert cmd == ["zuban", "check", "--python-executable", str(venv_python(venv)), "."]
+        assert cmd == ["zuban", "check", "--config-file", str(tmp_path / "zuban.benchmark.ini"),
+                       "--python-executable", str(venv_python(venv)), str(tmp_path)]
 
     @pytest.mark.parametrize("checker", ["pyright", "pyrefly", "ty", "mypy", "zuban"])
     def test_virtual_env_set(self, checker: str, tmp_path: Path) -> None:

typecheck_benchmark/daily_runner.py

@@ -578,7 +578,7 @@ def install_deps(
 # To ensure consistent benchmarking across packages, we write minimal config files
 # that embed the check_paths (from install_envs.json) directly into each checker's
 # native config format, then pass the config via CLI flag to override package-level
-# config.  This means paths are NOT passed as CLI arguments — they live in the config:
+# config. Paths live in the config:
 #
 #   pyright  -- pyrightconfig.json with "include": [paths] (written in-place)
 #   mypy     -- [mypy] section with files = path1, path2 via --config-file
@@ -651,11 +651,12 @@ def _write_dummy_zuban_config(
     package_path: Path, check_paths: list[str] | None = None,
 ) -> Path:
     """Write minimal mypy-style config for zuban benchmarking."""
-    config_path = package_path / "mypy.benchmark.ini"
+    config_path = package_path / "zuban.benchmark.ini"
     with open(config_path, "w", encoding="utf-8") as f:
         f.write("[mypy]\n")
         if check_paths:
             f.write(f"files = {', '.join(check_paths)}\n")
+        f.write("check_untyped_defs = True\n")
     return config_path
 
 
@@ -722,15 +723,12 @@ def run_checker(
             cmd.append(str(package_path))
         cwd = package_path
     elif checker == "zuban":
-        # zuban ignores files= from --config-file, so we must always pass
-        # check paths as explicit positional arguments.
-        cmd = ["zuban", "check"]
+        config_path = _write_dummy_zuban_config(package_path, rel_paths)
+        cmd = ["zuban", "check", "--config-file", str(config_path)]
         if python:
             cmd.extend(["--python-executable", python])
-        if rel_paths:
-            cmd.extend(rel_paths)
-        else:
-            cmd.append(".")
+        if not rel_paths:
+            cmd.append(str(package_path))
         cwd = package_path
     else:
         return {