Repository navigation
fix: CodeQL gets its memory budget and rebuilds a half-built database (#33) #34
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
6a11c8a
33c9b9e
80c1f70
95a01eb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,13 +26,15 @@ | |
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import fcntl | ||
| import json | ||
| import os | ||
| import re | ||
| import shutil | ||
| import subprocess | ||
| import sys | ||
| import tomllib | ||
| from collections.abc import Callable | ||
| from datetime import UTC, datetime | ||
| from pathlib import Path | ||
| from typing import NoReturn | ||
|
|
@@ -1050,6 +1052,12 @@ | |
| if config != default_config and not (REPO / config).is_file(): | ||
| cannot(f"codeql config {config} does not exist") | ||
| db = CACHE / "codeql" / project_name() | ||
| # Held until this process exits. The database directory is named for the | ||
| # worktree's basename, so two checkouts that share one -- or a push retried | ||
| # while the last is still running -- used ONE directory: each deleted the | ||
| # other's database mid-run, which left half-built caches and analyses dying | ||
| # with nothing to read (python-sdlc#33). | ||
| lock_codeql(db) | ||
| # Compiled query plans, kept between runs. CodeQL's default is ~/.codeql, | ||
| # which is /tmp in this image and gone when the container exits, so every | ||
| # run compiled all of security-extended again: 1m32s-3m13s of trentina's | ||
|
|
@@ -1059,15 +1067,15 @@ | |
| # a changed query recompiles and nothing stale is reused. | ||
| caches = CACHE / "codeql-common" | ||
| sarif = Path("/tmp/codeql-results.sarif") | ||
| shutil.rmtree(db, ignore_errors=True) | ||
| clear_database(db) | ||
| # An unwritable cache is a 2, not a traceback and not a 1. The same bug in | ||
| # `validate` surfaced as a PermissionError stack trace with an exit status | ||
| # that claimed a finding. | ||
| try: | ||
| db.parent.mkdir(parents=True, exist_ok=True) | ||
| caches.mkdir(parents=True, exist_ok=True) | ||
| except OSError as exc: | ||
| cannot(f"cannot create the CodeQL database directory {db.parent}: {exc}") | ||
| ram = f"--ram={codeql_ram_mb()}" | ||
|
|
||
| create = [ | ||
| "codeql", | ||
|
|
@@ -1078,6 +1086,7 @@ | |
| "--source-root=.", | ||
| "--overwrite", | ||
| f"--common-caches={caches}", | ||
| ram, | ||
| ] | ||
| analyze = [ | ||
| "codeql", | ||
|
|
@@ -1088,6 +1097,7 @@ | |
| f"--output={sarif}", | ||
| f"--threads={os.environ.get('SDLC_JOBS', '0')}", | ||
| f"--common-caches={caches}", | ||
| ram, | ||
| ] | ||
|
|
||
| # `--codescanning-config` is NOT a CLI flag in 2.27.4, on either subcommand. | ||
|
|
@@ -1186,12 +1196,101 @@ | |
| return queries, ignores | ||
|
|
||
|
|
||
| def lock_codeql(db: Path) -> None: | ||
| """Take this project's CodeQL lock, waiting for whoever holds it.""" | ||
| try: | ||
| db.parent.mkdir(parents=True, exist_ok=True) | ||
| # Deliberately never closed: the lock is released when the process ends. | ||
| fd = os.open(db.parent / f"{db.name}.lock", os.O_RDWR | os.O_CREAT, 0o644) | ||
| try: | ||
| fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) | ||
| except BlockingIOError: | ||
| print(f"codeql: another run holds {db.name}'s database; waiting for it", flush=True) | ||
| fcntl.flock(fd, fcntl.LOCK_EX) | ||
| except OSError as exc: | ||
| cannot(f"cannot lock the CodeQL database directory {db.parent}: {exc}") | ||
|
|
||
|
|
||
| def clear_database(db: Path) -> None: | ||
| """Remove the last run's database, and PROVE it is gone. | ||
|
|
||
| Nothing in it is reused, so a half-built one -- `trap/` and `working/` with | ||
| no `db-python`, from a run that was killed -- is just a cache miss. This | ||
| used to be `rmtree(ignore_errors=True)`, and when that quietly left anything | ||
| behind, `create --overwrite` refused the directory as "not a CodeQL | ||
| database" and a person had to delete it by hand. | ||
| """ | ||
| errors: list[str] = [] | ||
|
|
||
| def retry(func: Callable[[str], object], path: str, _exc: BaseException) -> None: | ||
| # A read-only directory cannot have its entries unlinked; open it up | ||
| # and try once more before recording the failure. | ||
| try: | ||
| os.chmod(os.path.dirname(path) or ".", 0o700) | ||
| if os.path.isdir(path) and not os.path.islink(path): | ||
| os.chmod(path, 0o700) | ||
| func(path) | ||
| except OSError as again: | ||
| errors.append(f"{path}: {again}") | ||
|
|
||
| if db.exists() or db.is_symlink(): | ||
| shutil.rmtree(db, onexc=retry) | ||
| if db.exists() or db.is_symlink(): | ||
| cannot(f"cannot clear the previous CodeQL database {db}: {'; '.join(errors[:3])}") | ||
|
|
||
|
|
||
| def memory_limit_mb() -> int | None: | ||
| """The smallest memory ceiling this container can see, in MB. | ||
|
|
||
| Three sources, because none is reliable alone. `SDLC_MEMORY` is the budget | ||
| the launcher gave `podman run --memory`, which rootless podman silently | ||
| drops when the cgroup memory controller is not delegated -- a gate inside a | ||
| developer container is exactly that. The cgroup's own `memory.max` is the | ||
| one the kernel enforces. MemTotal is the machine. | ||
| """ | ||
| found: list[int] = [] | ||
| budget = os.environ.get("SDLC_MEMORY", "").strip().lower() | ||
| match = re.fullmatch(r"(\d+)([kmg]?)b?", budget) | ||
| if match: | ||
| scale = {"": 1 / 2**20, "k": 1 / 1024, "m": 1, "g": 1024}[match.group(2)] | ||
| found.append(int(int(match.group(1)) * scale)) | ||
| try: | ||
| cg = Path("/sys/fs/cgroup/memory.max").read_text(encoding="utf-8").strip() | ||
| if cg.isdigit(): | ||
| found.append(int(cg) // 2**20) | ||
| except OSError: | ||
| pass | ||
| try: | ||
| for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): | ||
| if line.startswith("MemTotal:"): | ||
| found.append(int(line.split()[1]) // 1024) | ||
| except OSError, ValueError, IndexError: | ||
|
Check failure on line 1267 in sdlc
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. HIGH (Constitution): This Suggestion: Use a tuple of exception types: Evidence:
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. False positive, same as the thread above: PEP 758 syntax, valid on the image's Python 3.14 and tested there.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not a bug: |
||
| pass | ||
| return min((f for f in found if f > 0), default=None) | ||
|
|
||
|
|
||
| def codeql_ram_mb() -> int: | ||
| """`--ram` for the evaluator: three quarters of the ceiling, never under 2048. | ||
|
|
||
| Left to itself CodeQL does not look at the container at all. `codeql | ||
| resolve ram` gave a 1509 MB heap in a container with 8g and in one with 2g | ||
| alike, and 1253 MB on josui-skills' dataset -- where analyze then died with | ||
| status 70 (python-sdlc#33). The 8g the launcher budgets was never used. | ||
|
|
||
| The quarter left over is for the JVM outside its heap, file-backed maps of | ||
| the dataset, and /tmp, which is a tmpfs and so is charged to this same | ||
| cgroup. 2048 is CodeQL's own floor; it rounds anything smaller up. | ||
| """ | ||
| limit = memory_limit_mb() | ||
|
Check failure on line 1284 in sdlc
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. MEDIUM (Performance Check): When the detected memory limit is below 2048 MB, this forces CodeQL's RAM setting above that limit. For example, a 1 GB budget produces Suggestion: Avoid setting Evidence:
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Accepted as is. 2048 MB is CodeQL's own floor:
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. not a bug: 2048 MB is CodeQL's own floor, and |
||
| return max(2048, limit * 3 // 4) if limit else 2048 | ||
|
|
||
|
|
||
| def run_codeql(create: list[str], analyze: list[str], sarif: Path) -> None: | ||
| """Build the database and analyse it, or fail as 'could not run'.""" | ||
|
Check failure on line 1289 in sdlc
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. HIGH (Documentation): Suggestion: Update the docstring to describe both outcomes: CodeQL failures raise Evidence:
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Resolved by 80c1f70:
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed in 80c1f70 |
||
| step("codeql database create") | ||
| rc, _ = run(create, env=project_env()) | ||
|
Check failure on line 1291 in sdlc
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. MEDIUM (Test Coverage): The changed CodeQL failure paths are not covered by the modified resilience test, which now exercises database handling and memory only. Neither a nonzero database-create result nor a nonzero analyze result is tested for the new Suggestion: Add tests that mock Evidence:
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in the next commit.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed in 95a01eb |
||
| if rc != 0: | ||
| cannot("codeql could not build a database; nothing was analysed") | ||
| cannot(f"codeql could not build a database (exit {rc}); nothing was analysed") | ||
| step("codeql database analyze") | ||
| rc, _ = run(analyze, env=project_env()) | ||
| if rc != 0: | ||
|
|
@@ -1214,7 +1313,7 @@ | |
| for run_ in data.get("runs", []): | ||
| for result in run_.get("results", []): | ||
| phys = ((result.get("locations") or [{}])[0]).get("physicalLocation") or {} | ||
| uri = (phys.get("artifactLocation") or {}).get("uri", "?") | ||
|
Check failure on line 1316 in sdlc
|
||
| if path_ignored(uri, ignores): | ||
| continue | ||
| line = (phys.get("region") or {}).get("startLine", "?") | ||
|
|
@@ -1228,7 +1327,7 @@ | |
| "rule": result.get("ruleId", "?"), | ||
| "path": uri, | ||
| "fingerprint": fingerprint, | ||
| "line": line, | ||
|
Check failure on line 1330 in sdlc
|
||
| "message": (result.get("message") or {}).get("text", ""), | ||
| } | ||
| ) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| #!/usr/bin/env python3 | ||
| """CodeQL must not fail for want of memory or for a half-built cache. | ||
|
|
||
| python-sdlc#33: on josui-skills a killed run left a half-built database that a | ||
| person had to delete by hand, and the evaluator was sized by CodeQL's own guess | ||
| rather than the container's budget, so analyze died and every push was refused. | ||
|
|
||
| Run: tests/test_codeql_resilience.py (needs the image's python3 and PyYAML) | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import importlib.machinery | ||
| import importlib.util | ||
| import os | ||
| import stat | ||
| import sys | ||
| import tempfile | ||
| from pathlib import Path | ||
|
|
||
| # Loaded by path, as in test_codeql_globs.py: `sdlc` has no .py extension. | ||
| _SPEC = importlib.util.spec_from_loader( | ||
| "sdlc_under_test", | ||
| importlib.machinery.SourceFileLoader( | ||
| "sdlc_under_test", str(Path(__file__).resolve().parent.parent / "sdlc") | ||
| ), | ||
| ) | ||
| assert _SPEC is not None and _SPEC.loader is not None | ||
| _SDLC = importlib.util.module_from_spec(_SPEC) | ||
| _SPEC.loader.exec_module(_SDLC) | ||
|
|
||
| # CodeQL's own floor, and what an 8g budget should yield: three quarters of it. | ||
| FLOOR_MB = 2048 | ||
| EIGHT_G_MB = 8192 | ||
| FOUR_G_MB = 4096 | ||
| EIGHT_G_RAM_MB = 6144 | ||
|
|
||
|
|
||
| def check(ok: bool, what: str) -> bool: | ||
| print(f"{'ok ' if ok else 'FAIL'} {what}") | ||
| return ok | ||
|
|
||
|
|
||
| def database_and_lock() -> bool: | ||
| ok = True | ||
| with tempfile.TemporaryDirectory() as tmp: | ||
| # A killed run's leftovers: trap/ and working/, no db-python, and a | ||
| # read-only directory that a plain rmtree cannot empty. | ||
| db = Path(tmp) / "codeql" / "proj" | ||
| (db / "trap" / "deep").mkdir(parents=True) | ||
| (db / "working").mkdir() | ||
| (db / "trap" / "deep" / "x.trap").write_text("x") | ||
| os.chmod(db / "trap" / "deep", stat.S_IRUSR | stat.S_IXUSR) | ||
| _SDLC.clear_database(db) | ||
| ok &= check(not db.exists(), "a half-built database, read-only parts and all, is cleared") | ||
| _SDLC.clear_database(db) | ||
| ok &= check(not db.exists(), "clearing a database that is not there is not an error") | ||
|
|
||
| # The lock is per project, in a directory that may not exist yet. | ||
| _SDLC.lock_codeql(Path(tmp) / "fresh" / "proj") | ||
| lock = Path(tmp) / "fresh" / "proj.lock" | ||
| ok &= check(lock.is_file(), "the lock file is created beside the database") | ||
| return ok | ||
|
|
||
|
|
||
| def memory() -> bool: | ||
| ok = True | ||
| os.environ["SDLC_MEMORY"] = "8g" | ||
| limit = _SDLC.memory_limit_mb() | ||
| ok &= check( | ||
| limit is not None and limit <= EIGHT_G_MB, f"an 8g budget caps the limit ({limit} MB)" | ||
| ) | ||
| os.environ["SDLC_MEMORY"] = "4096m" | ||
| ok &= check(_SDLC.memory_limit_mb() <= FOUR_G_MB, "a budget in m is read as MB") | ||
| os.environ["SDLC_MEMORY"] = "1g" | ||
| ok &= check(_SDLC.codeql_ram_mb() == FLOOR_MB, "never below CodeQL's own 2048 MB floor") | ||
| os.environ["SDLC_MEMORY"] = "8g" | ||
| ram = _SDLC.codeql_ram_mb() | ||
| ok &= check( | ||
| FLOOR_MB <= ram <= EIGHT_G_RAM_MB, f"three quarters of 8g to the evaluator ({ram} MB)" | ||
| ) | ||
| return ok | ||
|
|
||
|
|
||
| def failures() -> bool: | ||
| """Either CodeQL step exiting non-zero is a 2 that names the step.""" | ||
| ok = True | ||
| real_run, real_step = _SDLC.run, _SDLC.step | ||
| _SDLC.step = lambda _msg: None | ||
| try: | ||
| for failing, says in ( | ||
| ("create", "could not build a database"), | ||
| ("analyze", "analyze failed"), | ||
| ): | ||
| _SDLC.run = lambda cmd, failing=failing, **_kw: (7 if cmd[2] == failing else 0, "") | ||
| try: | ||
| _SDLC.run_codeql( | ||
| ["codeql", "database", "create"], ["codeql", "database", "analyze"], Path("/x") | ||
| ) | ||
| raised = "" | ||
| except _SDLC.CannotRunError as exc: | ||
| raised = str(exc) | ||
| ok &= check( | ||
| says in raised and "exit 7" in raised, f"a failed {failing} is a 2 that says so" | ||
| ) | ||
| finally: | ||
| _SDLC.run, _SDLC.step = real_run, real_step | ||
| return ok | ||
|
|
||
|
|
||
| def main() -> int: | ||
| saved = dict(os.environ) | ||
| try: | ||
| ok = database_and_lock() | ||
| ok &= memory() | ||
| ok &= failures() | ||
| finally: | ||
| os.environ.clear() | ||
| os.environ.update(saved) | ||
| print("codeql resilience PASSED" if ok else "codeql resilience FAILED") | ||
| return 0 if ok else 1 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| sys.exit(main()) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
CRITICAL (Bug Hunter): This
exceptclause is invalid Python 3 syntax. The module fails to parse, so thesdlcimage cannot run any gate.Suggestion: Catch the exceptions as a tuple:
except (OSError, ValueError, IndexError):.Evidence:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
False positive.
except OSError, ValueError, IndexError:is valid on Python 3.14 (PEP 758), the interpreter this image runs. ruff format writes it this way for target 3.14, the file already uses the form atself_reported(), and every unit test ran this module in the image.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not a bug:
except A, B, C:is valid Python 3.14 syntax (PEP 758), the image's interpreter; ruff writes it for target 3.14 and the tests ran this module in the image.