Merge pull request #108 from codellm-devkit/fix/issue-107-env-interpreter-selection

fix(env): parso-version-aware interpreter selection for the analysis venv

Author: rahlk

CHANGELOG.md

@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- **Analysis env provisioning is parso-version-aware** (#107): on hosts whose default
+  `python3` is newer than the newest grammar the installed parso ships (e.g. Python 3.14
+  with parso ≤ 0.8.4), every file failed jedi parsing and the run completed "successfully"
+  with an empty symbol table. Provisioning now derives parso's supported ceiling at runtime
+  from its shipped grammar files and prefers the newest supported interpreter on the host
+  (versioned PATH names, then pyenv installs), falling back loudly only when none exists;
+  an explicit `SYSTEM_PYTHON` is still honored, with a warning when unsupported. A run in
+  which every discovered file fails now logs a prominent error instead of staying silent,
+  and `parso>=0.8.5` (the first release with the 3.14 grammar) is a direct dependency.
+
 ## [1.0.2] - 2026-07-16
 
 ### Fixed

codeanalyzer/core.py

@@ -163,8 +163,159 @@ def _cmd_exec_helper(
             stderr=None,
         )
 
+    @classmethod
+    def _get_base_interpreter(cls) -> Path:
+        """The interpreter used to provision the analysis virtualenv.
+
+        jedi parses the *analysis environment's* Python version with parso,
+        which ships one hardcoded grammar file per minor version — an
+        environment newer than the newest shipped grammar makes every file
+        fail with "Python version X.Y is currently not supported" while the
+        run still exits 0 (#107). So the default choice is gated on the
+        installed parso's ceiling: a too-new default is swapped for the
+        newest supported interpreter found on the host, falling back to the
+        default (loudly) only when none exists. An explicit ``SYSTEM_PYTHON``
+        always wins, with a warning when parso cannot parse its version.
+        """
+        # An explicit SYSTEM_PYTHON override wins (consulted only when running
+        # inside a virtualenv, matching the historical behavior).
+        if sys.prefix != sys.base_prefix:
+            system_python = os.getenv("SYSTEM_PYTHON")
+            if system_python:
+                system_python_path = Path(system_python)
+                if system_python_path.exists() and system_python_path.is_file():
+                    ceiling = cls._parso_supported_ceiling()
+                    version = cls._interpreter_version(system_python_path)
+                    if ceiling is not None and version is not None and version > ceiling:
+                        logger.warning(
+                            f"SYSTEM_PYTHON={system_python} is Python "
+                            f"{version[0]}.{version[1]}, newer than the newest grammar "
+                            f"the installed parso ships ({ceiling[0]}.{ceiling[1]}). "
+                            "jedi will likely reject every file in the analysis "
+                            "environment (#107); honoring the explicit override anyway."
+                        )
+                    return system_python_path
+
+        candidate = cls._default_base_interpreter()
+        ceiling = cls._parso_supported_ceiling()
+        if ceiling is None:
+            return candidate
+        version = cls._interpreter_version(candidate)
+        if version is None or version <= ceiling:
+            return candidate
+        logger.warning(
+            f"Default interpreter {candidate} is Python {version[0]}.{version[1]}, "
+            f"newer than the newest grammar the installed parso ships "
+            f"({ceiling[0]}.{ceiling[1]}) — looking for a supported interpreter "
+            "for the analysis environment (#107)."
+        )
+        supported = cls._find_supported_interpreter(ceiling)
+        if supported is not None:
+            logger.info(f"Provisioning the analysis environment with {supported}.")
+            return supported
+        logger.warning(
+            f"No interpreter <= {ceiling[0]}.{ceiling[1]} found on this host; "
+            f"falling back to {candidate}. jedi/parso will likely reject every "
+            "file — install a supported Python or upgrade parso."
+        )
+        return candidate
+
+    @staticmethod
+    def _versions_from_grammar_stems(stems: List[str]) -> List[tuple]:
+        """``grammar313`` → ``(3, 13)``, sorted ascending; malformed stems dropped."""
+        versions = []
+        for stem in stems:
+            digits = stem[len("grammar"):]
+            if len(digits) >= 2 and digits.isdigit():
+                versions.append((int(digits[0]), int(digits[1:])))
+        return sorted(versions)
+
+    @classmethod
+    def _parso_supported_ceiling(cls) -> Optional[tuple]:
+        """Newest ``(major, minor)`` the installed parso ships a grammar for,
+        derived from its ``python/grammar*.txt`` files so the ceiling moves
+        automatically when parso adds a version. ``None`` if undeterminable."""
+        try:
+            import parso
+
+            stems = [
+                p.stem
+                for p in (Path(parso.__file__).parent / "python").glob("grammar*.txt")
+            ]
+            versions = cls._versions_from_grammar_stems(stems)
+            return versions[-1] if versions else None
+        except Exception:
+            return None
+
+    @staticmethod
+    def _interpreter_version(interpreter: Path) -> Optional[tuple]:
+        """``(major, minor)`` of an interpreter, or ``None`` if it can't run."""
+        try:
+            result = subprocess.run(
+                [
+                    str(interpreter),
+                    "-c",
+                    "import sys; print('%d.%d' % sys.version_info[:2])",
+                ],
+                capture_output=True,
+                text=True,
+                timeout=5,
+            )
+            if result.returncode == 0:
+                major, minor = result.stdout.strip().split(".")
+                return (int(major), int(minor))
+        except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError, ValueError):
+            pass
+        return None
+
+    @staticmethod
+    def _pick_supported_interpreter(
+        candidates: List[tuple], ceiling: tuple
+    ) -> Optional[Path]:
+        """Newest candidate whose version is within the ceiling.
+
+        ``candidates`` is ``[(path, (major, minor) | None), ...]``."""
+        supported = [
+            (version, path)
+            for path, version in candidates
+            if version is not None and version <= ceiling
+        ]
+        return max(supported)[1] if supported else None
+
+    @classmethod
+    def _find_supported_interpreter(cls, ceiling: tuple) -> Optional[Path]:
+        """Search the host for the newest interpreter within the parso ceiling:
+        versioned names on PATH (``python3.13``, ``python3.12``, ...) first,
+        then pyenv installs."""
+        paths: List[Path] = []
+        for minor in range(ceiling[1], 7, -1):
+            which = shutil.which(f"python{ceiling[0]}.{minor}")
+            # Skip the current virtualenv's own interpreter (same rule as
+            # _default_base_interpreter): the analysis env must come from a
+            # base installation.
+            if which and not which.startswith(sys.prefix):
+                paths.append(Path(which))
+        for pyenv_root in (os.getenv("PYENV_ROOT"), str(Path.home() / ".pyenv")):
+            if not pyenv_root:
+                continue
+            versions_dir = Path(pyenv_root) / "versions"
+            if versions_dir.is_dir():
+                for install in sorted(versions_dir.iterdir(), reverse=True):
+                    exe = install / "bin" / "python3"
+                    if exe.exists():
+                        paths.append(exe)
+        seen = set()
+        candidates = []
+        for path in paths:
+            key = str(path)
+            if key in seen:
+                continue
+            seen.add(key)
+            candidates.append((path, cls._interpreter_version(path)))
+        return cls._pick_supported_interpreter(candidates, ceiling)
+
     @staticmethod
-    def _get_base_interpreter() -> Path:
+    def _default_base_interpreter() -> Path:
         """Get the base Python interpreter path.
 
         This method finds a suitable base Python interpreter that can be used
@@ -183,13 +334,6 @@ def _get_base_interpreter() -> Path:
 
         # We're inside a virtual environment; need to find the base interpreter
 
-        # First, check if user explicitly set SYSTEM_PYTHON
-        system_python = os.getenv("SYSTEM_PYTHON")
-        if system_python:
-            system_python_path = Path(system_python)
-            if system_python_path.exists() and system_python_path.is_file():
-                return system_python_path
-
         # Try to get the base interpreter from sys.base_executable (Python 3.3+)
         if hasattr(sys, "base_executable") and sys.base_executable:
             base_exec = Path(sys.base_executable)
@@ -778,6 +922,15 @@ def _build_symbol_table(self, cached_symbol_table: Optional[Dict[str, PyModule]]
             if files_from_cache > 0:
                 logger.info(f"Reused {files_from_cache} files from cache, processed {files_processed} new/changed files")
 
+        if py_files and not symbol_table:
+            logger.error(
+                "Every one of the %d discovered Python files failed to process — "
+                "the symbol table is empty. This usually means the analysis "
+                "environment's interpreter is newer than the installed jedi/parso "
+                "stack supports (#107); check the per-file errors above.",
+                len(py_files),
+            )
+
         logger.info(
             "✅ Symbol table: %d modules in %.1fs",
             len(symbol_table), time.perf_counter() - t0_st,

pyproject.toml

@@ -12,6 +12,9 @@ dependencies = [
     # jedi
     "jedi>=0.18.0,<0.20.0; python_version < '3.11'",
     "jedi<=0.19.2; python_version >= '3.11'",
+    # parso 0.8.5 is the first release shipping the Python 3.14 grammar; older
+    # resolutions make jedi reject every file in a 3.14 analysis env (#107)
+    "parso>=0.8.5",
     # msgpack
     "msgpack>=1.0.0,<1.0.7; python_version < '3.11'",
     "msgpack>=1.0.7,<2.0.0; python_version >= '3.11'",

test/test_env_interpreter.py

@@ -0,0 +1,156 @@
+"""Regression tests for #107: environment provisioning must prefer an
+interpreter the installed jedi/parso stack can actually parse, and a run
+where every module fails must not stay silent.
+
+parso ships one hardcoded grammar file per Python minor (grammar313.txt,
+grammar314.txt, ...). If the provisioned analysis venv is newer than the
+newest shipped grammar, jedi rejects every file and the symbol table comes
+back empty while the process still exits 0.
+"""
+import logging
+from pathlib import Path
+
+import pytest
+
+from codeanalyzer.core import Codeanalyzer
+
+
+# ----------------------------------------------------------------------------------------------
+# The parso ceiling: derived from the shipped grammar files at runtime, never hardcoded.
+# ----------------------------------------------------------------------------------------------
+
+
+def test_grammar_stems_parse_to_versions():
+    got = Codeanalyzer._versions_from_grammar_stems(
+        ["grammar36", "grammar39", "grammar310", "grammar313", "grammar314"]
+    )
+    assert got == [(3, 6), (3, 9), (3, 10), (3, 13), (3, 14)]
+
+
+def test_malformed_grammar_stems_are_ignored():
+    got = Codeanalyzer._versions_from_grammar_stems(
+        ["grammar", "grammarXY", "grammar3", "grammar312"]
+    )
+    assert got == [(3, 12)]
+
+
+def test_parso_ceiling_reflects_installed_parso():
+    """The ceiling must be the max of the grammars parso actually ships —
+    on any env with parso >= 0.8.5 that is at least (3, 13)."""
+    ceiling = Codeanalyzer._parso_supported_ceiling()
+    assert ceiling is not None
+    assert ceiling >= (3, 13)
+
+
+# ----------------------------------------------------------------------------------------------
+# Interpreter choice honors the ceiling.
+# ----------------------------------------------------------------------------------------------
+
+
+def test_pick_supported_interpreter_prefers_newest_within_ceiling():
+    candidates = [
+        (Path("/opt/py315"), (3, 15)),
+        (Path("/opt/py313"), (3, 13)),
+        (Path("/opt/py311"), (3, 11)),
+    ]
+    assert Codeanalyzer._pick_supported_interpreter(candidates, (3, 14)) == Path(
+        "/opt/py313"
+    )
+    assert Codeanalyzer._pick_supported_interpreter(candidates, (3, 10)) is None
+
+
+def test_base_interpreter_swaps_out_unsupported_default(monkeypatch):
+    """If the default interpreter is newer than parso's ceiling, provisioning
+    must pick a supported one instead."""
+    fake_default = Path("/opt/py399/bin/python3")
+    fake_supported = Path("/opt/py313/bin/python3")
+
+    monkeypatch.setattr(
+        Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default)
+    )
+    monkeypatch.setattr(
+        Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13))
+    )
+    monkeypatch.setattr(
+        Codeanalyzer,
+        "_interpreter_version",
+        staticmethod(lambda p: (3, 99) if p == fake_default else (3, 13)),
+    )
+    monkeypatch.setattr(
+        Codeanalyzer,
+        "_find_supported_interpreter",
+        staticmethod(lambda ceiling: fake_supported),
+    )
+    assert Codeanalyzer._get_base_interpreter() == fake_supported
+
+
+def test_base_interpreter_keeps_supported_default(monkeypatch):
+    fake_default = Path("/opt/py312/bin/python3")
+    monkeypatch.setattr(
+        Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default)
+    )
+    monkeypatch.setattr(
+        Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13))
+    )
+    monkeypatch.setattr(
+        Codeanalyzer, "_interpreter_version", staticmethod(lambda p: (3, 12))
+    )
+    assert Codeanalyzer._get_base_interpreter() == fake_default
+
+
+def test_base_interpreter_falls_back_loudly_when_nothing_supported(monkeypatch, caplog):
+    fake_default = Path("/opt/py399/bin/python3")
+    monkeypatch.setattr(
+        Codeanalyzer, "_default_base_interpreter", staticmethod(lambda: fake_default)
+    )
+    monkeypatch.setattr(
+        Codeanalyzer, "_parso_supported_ceiling", staticmethod(lambda: (3, 13))
+    )
+    monkeypatch.setattr(
+        Codeanalyzer, "_interpreter_version", staticmethod(lambda p: (3, 99))
+    )
+    monkeypatch.setattr(
+        Codeanalyzer, "_find_supported_interpreter", staticmethod(lambda ceiling: None)
+    )
+    # the codeanalyzer logger sets propagate=False (rich handler); caplog needs
+    # propagation to observe records
+    monkeypatch.setattr(logging.getLogger("codeanalyzer"), "propagate", True)
+    with caplog.at_level(logging.WARNING, logger="codeanalyzer"):
+        assert Codeanalyzer._get_base_interpreter() == fake_default
+    assert any("parso" in r.getMessage() for r in caplog.records)
+
+
+# ----------------------------------------------------------------------------------------------
+# A run where every module failed must be loud, not silently empty.
+# ----------------------------------------------------------------------------------------------
+
+
+def test_all_files_failing_emits_an_error(tmp_path, monkeypatch, caplog):
+    proj = tmp_path / "proj"
+    proj.mkdir()
+    (proj / "a.py").write_text("def f():\n    return 1\n", encoding="utf-8")
+    (proj / "b.py").write_text("def g():\n    return 2\n", encoding="utf-8")
+
+    from codeanalyzer.options.options import AnalysisOptions
+    from codeanalyzer.config import OutputFormat
+    from codeanalyzer.syntactic_analysis.symbol_table_builder import SymbolTableBuilder
+
+    def boom(self, py_file):
+        raise RuntimeError("Python version 3.99 is currently not supported.")
+
+    monkeypatch.setattr(SymbolTableBuilder, "build_pymodule_from_file", boom)
+
+    opts = AnalysisOptions(
+        input=proj, output=None, format=OutputFormat.JSON,
+        skip_tests=True, no_venv=True, cache_dir=tmp_path / "cache",
+        rebuild_analysis=True,
+    )
+    analyzer = Codeanalyzer(opts)
+    monkeypatch.setattr(logging.getLogger("codeanalyzer"), "propagate", True)
+    with caplog.at_level(logging.ERROR, logger="codeanalyzer"):
+        table = analyzer._build_symbol_table(cached_symbol_table={})
+    assert table == {}
+    assert any(
+        "every" in r.getMessage().lower() or "all " in r.getMessage().lower()
+        for r in caplog.records
+    ), "an empty symbol table from total per-file failure must be reported loudly"