diff --git a/.github/guid-index.json b/.github/guid-index.json index 7d72d976..2507c04d 100644 --- a/.github/guid-index.json +++ b/.github/guid-index.json @@ -875,7 +875,7 @@ "path": ".github/workflows/ci-tests.yml", "guid": "ci-tests-workflow-2025-automation", "title": "CI Self Tests", - "version": "1.2.0", + "version": "1.2.2", "header_path": ".github/workflows/ci-tests.yml", "path_mismatch": false }, @@ -979,7 +979,7 @@ "path": ".github/workflows/reusable-ci.yml", "guid": "reusable-ci-2025-09-24-core-workflow", "title": "Reusable CI Workflow", - "version": "1.17.1", + "version": "1.19.1", "header_path": ".github/workflows/reusable-ci.yml", "path_mismatch": false }, @@ -1051,7 +1051,7 @@ "path": ".github/workflows/scripts/ci_workflow.py", "guid": "d9c2b3a4-5e6f-47a8-9b0c-1d2e3f4a5b6c", "title": "ci workflow", - "version": "1.2.0", + "version": "1.3.2", "header_path": ".github/workflows/scripts/ci_workflow.py", "path_mismatch": false }, @@ -3663,6 +3663,14 @@ "header_path": "tests/workflow_scripts/test_automation_workflow.py", "path_mismatch": false }, + { + "path": "tests/workflow_scripts/test_ci_workflow.py", + "guid": "be8f15eb-d640-417d-8c19-bb923f338f54", + "title": "test ci workflow", + "version": "1.0.1", + "header_path": "tests/workflow_scripts/test_ci_workflow.py", + "path_mismatch": false + }, { "path": "tests/workflow_scripts/test_docs_workflow.py", "guid": "2f3a4b5c-6d7e-8f90-a1b2-c3d4e5f60718", diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index a1751ddf..5a446a6a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -1,6 +1,7 @@ # file: .github/workflows/ci-tests.yml -# version: 1.2.0 +# version: 1.2.2 # guid: ci-tests-workflow-2025-automation +# last-edited: 2026-10-20 name: CI Self Tests @@ -143,11 +144,9 @@ jobs: - name: Lint scripts run: | - if command -v ruff >/dev/null 2>&1; then - ruff --version - else - pip install ruff - fi + # pinned: an unpinned ruff adds findings on its own release schedule + python -m pip install ruff==0.17.0 + python -m ruff --version if command -v black >/dev/null 2>&1; then black --version else @@ -155,7 +154,7 @@ jobs: fi if find scripts -name "*.py" -o -path "tests/*.py" | head -1 | grep -q .; then black --check scripts tests || true - ruff check scripts tests || true + python -m ruff check scripts tests || true else echo "No Python sources detected for linting." fi diff --git a/.github/workflows/reusable-ci.yml b/.github/workflows/reusable-ci.yml index a0c64e90..4d908fda 100644 --- a/.github/workflows/reusable-ci.yml +++ b/.github/workflows/reusable-ci.yml @@ -1,7 +1,7 @@ # file: .github/workflows/reusable-ci.yml -# version: 1.18.0 +# version: 1.19.1 # guid: reusable-ci-2025-09-24-core-workflow -# last-edited: 2026-10-16 +# last-edited: 2026-10-20 name: Reusable CI Workflow @@ -23,6 +23,17 @@ on: required: false type: string default: '3.13' + ruff-version: + description: >- + Bare ruff version for the Python lint step, e.g. 0.17.0 (a leading v + is tolerated). Empty (default) uses RUFF_VERSION in ci_workflow.py. + Escape hatch for repos that format with a newer ruff. Takes effect + once the script ref this workflow checks out (ghcommon-ref.txt, or the + fallback pin) includes the commit that added the input; on an older + ref the value is ignored. + required: false + type: string + default: '' rust-version: description: 'Rust version to use' required: false @@ -507,6 +518,9 @@ jobs: python-version: ${{ inputs.python-version }} - name: Lint Python workflow scripts + # TODO: this job pins ruff==0.16.10 separately from RUFF_VERSION in + # ci_workflow.py (0.17.0, used by the consumer Python lint). Unify the two + # pins so one bump moves both. run: | set -euo pipefail python -m pip install --upgrade pip @@ -751,6 +765,8 @@ jobs: - name: Lint Python code if: ${{ !inputs.skip-linting }} + env: + RUFF_VERSION: ${{ inputs.ruff-version }} run: | python "$GHCOMMON_SCRIPTS_DIR/ci_workflow.py" python-lint diff --git a/.github/workflows/scripts/ci_workflow.py b/.github/workflows/scripts/ci_workflow.py index b457c73a..e7b119e5 100755 --- a/.github/workflows/scripts/ci_workflow.py +++ b/.github/workflows/scripts/ci_workflow.py @@ -1,29 +1,33 @@ #!/usr/bin/env python3 # file: .github/workflows/scripts/ci_workflow.py -# version: 1.2.0 +# version: 1.3.2 # guid: d9c2b3a4-5e6f-47a8-9b0c-1d2e3f4a5b6c +# last-edited: 2026-10-20 """Helper utilities invoked from GitHub Actions CI workflows.""" from __future__ import annotations import argparse +from collections.abc import Iterable +import functools import json import os +from pathlib import Path import re import shutil import subprocess import sys import textwrap import time -from collections.abc import Iterable -from pathlib import Path from typing import Any try: import requests # type: ignore[import-untyped] -except ModuleNotFoundError: # pragma: no cover - fallback when requests unavailable +except ( + ModuleNotFoundError +): # pragma: no cover - fallback when requests unavailable requests = None - from urllib.parse import urlencode + from urllib.parse import urlencode, urlparse from urllib.request import Request, urlopen class _HTTPResponse: @@ -45,8 +49,10 @@ def _http_get( if params: query = urlencode(params) url = f"{url}?{query}" - req = Request(url, headers=headers or {}) - with urlopen(req, timeout=timeout) as resp: + if urlparse(url).scheme != "https": + raise ValueError(f"Refusing to fetch non-https URL: {url}") + req = Request(url, headers=headers or {}) # noqa: S310 - https only, checked above + with urlopen(req, timeout=timeout) as resp: # noqa: S310 - https only, checked above status_code = resp.getcode() body = resp.read().decode("utf-8") try: @@ -63,10 +69,9 @@ def _http_get( params: dict[str, Any] | None = None, timeout: int = 30, ): - return requests.get(url, headers=headers, params=params, timeout=timeout) - - -_CONFIG_CACHE: dict[str, Any] | None = None + return requests.get( + url, headers=headers, params=params, timeout=timeout + ) def append_to_file(path_env: str, content: str) -> None: @@ -80,33 +85,34 @@ def append_to_file(path_env: str, content: str) -> None: def write_output(name: str, value: str) -> None: + """Write a step output (name=value) to the GITHUB_OUTPUT file.""" append_to_file("GITHUB_OUTPUT", f"{name}={value}\n") def append_env(name: str, value: str) -> None: + """Export an environment variable to later steps via GITHUB_ENV.""" append_to_file("GITHUB_ENV", f"{name}={value}\n") def append_summary(text: str) -> None: + """Append Markdown text to the GitHub step summary.""" append_to_file("GITHUB_STEP_SUMMARY", text) +@functools.lru_cache(maxsize=1) def get_repository_config() -> dict[str, Any]: - global _CONFIG_CACHE - if _CONFIG_CACHE is not None: - return _CONFIG_CACHE - + """Return the parsed REPOSITORY_CONFIG JSON, cached after the first call.""" raw = os.environ.get("REPOSITORY_CONFIG") if not raw: - _CONFIG_CACHE = {} - return _CONFIG_CACHE + return {} try: - _CONFIG_CACHE = json.loads(raw) + return json.loads(raw) except json.JSONDecodeError: - print("::warning::Unable to parse REPOSITORY_CONFIG JSON; falling back to defaults") - _CONFIG_CACHE = {} - return _CONFIG_CACHE + print( + "::warning::Unable to parse REPOSITORY_CONFIG JSON; falling back to defaults" + ) + return {} def _config_path(default: Any, *path: str) -> Any: @@ -119,6 +125,7 @@ def _config_path(default: Any, *path: str) -> Any: def debug_filter(_: argparse.Namespace) -> None: + """Print which path-filter categories reported changed files.""" mapping = { "Go files changed": os.environ.get("CI_GO_FILES", ""), "Frontend files changed": os.environ.get("CI_FRONTEND_FILES", ""), @@ -127,8 +134,12 @@ def debug_filter(_: argparse.Namespace) -> None: "Docker files changed": os.environ.get("CI_DOCKER_FILES", ""), "Docs files changed": os.environ.get("CI_DOCS_FILES", ""), "Workflow files changed": os.environ.get("CI_WORKFLOW_FILES", ""), - "Workflow YAML files changed": os.environ.get("CI_WORKFLOW_YAML_FILES", ""), - "Workflow scripts changed": os.environ.get("CI_WORKFLOW_SCRIPT_FILES", ""), + "Workflow YAML files changed": os.environ.get( + "CI_WORKFLOW_YAML_FILES", "" + ), + "Workflow scripts changed": os.environ.get( + "CI_WORKFLOW_SCRIPT_FILES", "" + ), "Linter config files changed": os.environ.get("CI_LINT_FILES", ""), } for label, value in mapping.items(): @@ -136,8 +147,11 @@ def debug_filter(_: argparse.Namespace) -> None: def determine_execution(_: argparse.Namespace) -> None: + """Decide whether CI runs, honouring [skip ci] markers, and write outputs.""" commit_message = os.environ.get("GITHUB_HEAD_COMMIT_MESSAGE", "") - skip_ci = bool(re.search(r"\[(skip ci|ci skip)\]", commit_message, flags=re.IGNORECASE)) + skip_ci = bool( + re.search(r"\[(skip ci|ci skip)\]", commit_message, flags=re.IGNORECASE) + ) write_output("skip_ci", "true" if skip_ci else "false") if skip_ci: print("Skipping CI due to commit message") @@ -146,13 +160,20 @@ def determine_execution(_: argparse.Namespace) -> None: write_output("should_lint", "true") write_output("should_test_go", os.environ.get("CI_GO_FILES", "false")) - write_output("should_test_frontend", os.environ.get("CI_FRONTEND_FILES", "false")) - write_output("should_test_python", os.environ.get("CI_PYTHON_FILES", "false")) + write_output( + "should_test_frontend", os.environ.get("CI_FRONTEND_FILES", "false") + ) + write_output( + "should_test_python", os.environ.get("CI_PYTHON_FILES", "false") + ) write_output("should_test_rust", os.environ.get("CI_RUST_FILES", "false")) - write_output("should_test_docker", os.environ.get("CI_DOCKER_FILES", "false")) + write_output( + "should_test_docker", os.environ.get("CI_DOCKER_FILES", "false") + ) def wait_for_pr_automation(_: argparse.Namespace) -> None: + """Poll until the PR Automation workflow has finished for the target SHA.""" repo = os.environ.get("GITHUB_REPOSITORY") token = os.environ.get("GITHUB_TOKEN") target_sha = os.environ.get("TARGET_SHA") @@ -161,7 +182,9 @@ def wait_for_pr_automation(_: argparse.Namespace) -> None: sleep_seconds = int(os.environ.get("SLEEP_SECONDS", "10")) if not (repo and token and target_sha): - print("Missing required environment values; skipping PR automation wait") + print( + "Missing required environment values; skipping PR automation wait" + ) return headers = { @@ -172,15 +195,21 @@ def wait_for_pr_automation(_: argparse.Namespace) -> None: print("🔄 Waiting for PR automation to complete...") for attempt in range(max_attempts): - print(f"Checking for PR automation completion (attempt {attempt + 1}/{max_attempts})...") + print( + f"Checking for PR automation completion (attempt {attempt + 1}/{max_attempts})..." + ) try: - response = _http_get(url, headers=headers, params={"per_page": 100}, timeout=30) + response = _http_get( + url, headers=headers, params={"per_page": 100}, timeout=30 + ) except Exception as exc: # pragma: no cover - network issues during CI print(f"::warning::Unable to query workflow runs: {exc}") time.sleep(sleep_seconds) continue if response.status_code != 200: - print(f"::warning::Unable to query workflow runs: {response.status_code}") + print( + f"::warning::Unable to query workflow runs: {response.status_code}" + ) time.sleep(sleep_seconds) continue @@ -188,7 +217,8 @@ def wait_for_pr_automation(_: argparse.Namespace) -> None: matching_runs = [ run for run in runs - if run.get("head_sha") == target_sha and run.get("name") == workflow_name + if run.get("head_sha") == target_sha + and run.get("name") == workflow_name ] if not matching_runs: @@ -219,6 +249,7 @@ def _export_env_from_file(file_path: Path) -> None: def load_super_linter_config(_: argparse.Namespace) -> None: + """Pick the PR or CI super-linter env file and export its settings.""" event_name = os.environ.get("EVENT_NAME", "") pr_env = Path(os.environ.get("PR_ENV_FILE", "super-linter-pr.env")) ci_env = Path(os.environ.get("CI_ENV_FILE", "super-linter-ci.env")) @@ -245,6 +276,7 @@ def load_super_linter_config(_: argparse.Namespace) -> None: def write_validation_summary(_: argparse.Namespace) -> None: + """Write the CI validation results header to the step summary.""" event_name = os.environ.get("EVENT_NAME", "unknown") config_name = os.environ.get("SUMMARY_CONFIG", "super-linter-ci.env") append_summary( @@ -272,6 +304,7 @@ def _ensure_go_context() -> bool: def go_setup(_: argparse.Namespace) -> None: + """Download Go modules and build every package.""" if not _ensure_go_context(): return subprocess.run(["go", "mod", "download"], check=True) @@ -287,6 +320,7 @@ def _parse_go_coverage(total_line: str) -> float: def go_test(_: argparse.Namespace) -> None: + """Run Go tests with the race detector and enforce the coverage threshold.""" if not _ensure_go_context(): return @@ -296,7 +330,9 @@ def go_test(_: argparse.Namespace) -> None: if threshold_env: threshold = float(threshold_env) else: - threshold = float(_config_path(0, "testing", "coverage", "threshold") or 0) + threshold = float( + _config_path(0, "testing", "coverage", "threshold") or 0 + ) subprocess.run( [ @@ -341,11 +377,14 @@ def go_test(_: argparse.Namespace) -> None: coverage = _parse_go_coverage(total_line) print(f"Coverage: {coverage}%") if coverage < threshold: - raise SystemExit(f"Coverage {coverage}% is below threshold {threshold}%") + raise SystemExit( + f"Coverage {coverage}% is below threshold {threshold}%" + ) print(f"✅ Coverage {coverage}% meets threshold {threshold}%") def check_go_coverage(_: argparse.Namespace) -> None: + """Fail when Go coverage is below the configured threshold.""" coverage_file = Path(os.environ.get("COVERAGE_FILE", "coverage.out")) html_output = Path(os.environ.get("COVERAGE_HTML", "coverage.html")) threshold = float(os.environ.get("COVERAGE_THRESHOLD", "0")) @@ -385,15 +424,20 @@ def check_go_coverage(_: argparse.Namespace) -> None: coverage = _parse_go_coverage(total_line) print(f"Coverage: {coverage}%") if coverage < threshold: - raise SystemExit(f"Coverage {coverage}% is below threshold {threshold}%") + raise SystemExit( + f"Coverage {coverage}% is below threshold {threshold}%" + ) print(f"✅ Coverage {coverage}% meets threshold {threshold}%") -def _run_command(command: Iterable[str], check: bool = True) -> subprocess.CompletedProcess[str]: +def _run_command( + command: Iterable[str], check: bool = True +) -> subprocess.CompletedProcess[str]: return subprocess.run(list(command), check=check) def frontend_install(_: argparse.Namespace) -> None: + """Install frontend dependencies with the detected package manager.""" working_dir = os.environ.get("FRONTEND_WORKING_DIR", ".") original_dir = Path.cwd() @@ -401,7 +445,9 @@ def frontend_install(_: argparse.Namespace) -> None: if working_dir != ".": target_dir = Path(working_dir) if not target_dir.exists(): - raise FileNotFoundError(f"Frontend working directory not found: {working_dir}") + raise FileNotFoundError( + f"Frontend working directory not found: {working_dir}" + ) os.chdir(target_dir) print(f"Changed to frontend working directory: {working_dir}") @@ -419,9 +465,14 @@ def frontend_install(_: argparse.Namespace) -> None: def frontend_run(_: argparse.Namespace) -> None: + """Run a frontend package script (lint, test or build).""" script_name = os.environ.get("FRONTEND_SCRIPT", "") - success_message = os.environ.get("FRONTEND_SUCCESS_MESSAGE", "Command succeeded") - failure_message = os.environ.get("FRONTEND_FAILURE_MESSAGE", "Command failed") + success_message = os.environ.get( + "FRONTEND_SUCCESS_MESSAGE", "Command succeeded" + ) + failure_message = os.environ.get( + "FRONTEND_FAILURE_MESSAGE", "Command failed" + ) working_dir = os.environ.get("FRONTEND_WORKING_DIR", ".") if not script_name: @@ -433,11 +484,15 @@ def frontend_run(_: argparse.Namespace) -> None: if working_dir != ".": target_dir = Path(working_dir) if not target_dir.exists(): - raise FileNotFoundError(f"Frontend working directory not found: {working_dir}") + raise FileNotFoundError( + f"Frontend working directory not found: {working_dir}" + ) os.chdir(target_dir) print(f"Changed to frontend working directory: {working_dir}") - result = subprocess.run(["npm", "run", script_name, "--if-present"], check=False) + result = subprocess.run( + ["npm", "run", script_name, "--if-present"], check=False + ) if result.returncode == 0: print(success_message) else: @@ -449,8 +504,11 @@ def frontend_run(_: argparse.Namespace) -> None: def python_install(_: argparse.Namespace) -> None: + """Install Python dependencies for the repository.""" python = sys.executable - subprocess.run([python, "-m", "pip", "install", "--upgrade", "pip"], check=True) + subprocess.run( + [python, "-m", "pip", "install", "--upgrade", "pip"], check=True + ) if Path("requirements.txt").is_file(): subprocess.run( @@ -461,12 +519,19 @@ def python_install(_: argparse.Namespace) -> None: if Path("pyproject.toml").is_file(): subprocess.run([python, "-m", "pip", "install", "-e", "."], check=True) - subprocess.run([python, "-m", "pip", "install", "pytest", "pytest-cov"], check=True) + subprocess.run( + [python, "-m", "pip", "install", "pytest", "pytest-cov"], check=True + ) def python_run_tests(_: argparse.Namespace) -> None: + """Run the Python test suite with pytest.""" + def has_tests() -> bool: - return any(any(Path(".").rglob(pattern)) for pattern in ("test_*.py", "*_test.py")) + return any( + any(Path(".").rglob(pattern)) + for pattern in ("test_*.py", "*_test.py") + ) if not has_tests(): print("â„šī¸ No Python tests found") @@ -486,6 +551,41 @@ def has_tests() -> bool: ) +# Pinned ruff for the shared Python lint. Consumers format against this exact +# version; an unpinned install lets a new ruff release break every consumer's CI +# overnight. Override per repo with the reusable-ci `ruff-version` input +# (exported as RUFF_VERSION). +RUFF_VERSION = "0.17.0" + + +def _installed_ruff_version() -> str | None: + """Return the version of ruff importable by this interpreter, or None.""" + result = subprocess.run( + [sys.executable, "-m", "ruff", "--version"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + # Output is "ruff 0.17.0". + parts = result.stdout.split() + return parts[-1] if parts else None + + +def _ensure_ruff(version: str) -> None: + """Install ruff==version for this interpreter and verify it is what runs.""" + if _installed_ruff_version() != version: + subprocess.run( + [sys.executable, "-m", "pip", "install", f"ruff=={version}"], + check=True, + ) + found = _installed_ruff_version() + if found != version: + msg = f"expected ruff {version} but `python -m ruff` reports {found!r}" + raise RuntimeError(msg) + + def python_lint(_: argparse.Namespace) -> None: """Run Python formatting and linting if sources are present.""" event_path = os.environ.get("GITHUB_EVENT_PATH") @@ -531,7 +631,9 @@ def python_lint(_: argparse.Namespace) -> None: if line.strip().endswith(".py") ] if not changed_python: - print("â„šī¸ No Python changes detected for linting; skipping format/lint checks.") + print( + "â„šī¸ No Python changes detected for linting; skipping format/lint checks." + ) return python_sources = [ @@ -560,17 +662,20 @@ def python_lint(_: argparse.Namespace) -> None: if not lint_targets: lint_targets = ["."] - required_tools = ["ruff"] - missing_tools = [tool for tool in required_tools if shutil.which(tool) is None] - if missing_tools: - python = sys.executable - subprocess.run( - [python, "-m", "pip", "install", "--upgrade", *missing_tools], - check=True, - ) + override = os.environ.get("RUFF_VERSION", "").strip().removeprefix("v") + ruff_version = override or RUFF_VERSION + _ensure_ruff(ruff_version) - subprocess.run(["ruff", "format", "--check", *lint_targets], check=True) - subprocess.run(["ruff", "check", *lint_targets], check=True) + # Run ruff as `python -m ruff` so the interpreter that just verified the + # version is the one that lints; a different ruff on PATH (pipx, the runner + # image) is never used. + subprocess.run( + [sys.executable, "-m", "ruff", "format", "--check", *lint_targets], + check=True, + ) + subprocess.run( + [sys.executable, "-m", "ruff", "check", *lint_targets], check=True + ) def rust_format(_: argparse.Namespace) -> None: @@ -628,13 +733,17 @@ def rust_test(_: argparse.Namespace) -> None: def ensure_cargo_llvm_cov(_: argparse.Namespace) -> None: + """Install cargo-llvm-cov when it is not already available.""" if shutil.which("cargo-llvm-cov"): print("cargo-llvm-cov already installed") return - subprocess.run(["cargo", "install", "cargo-llvm-cov", "--locked"], check=True) + subprocess.run( + ["cargo", "install", "cargo-llvm-cov", "--locked"], check=True + ) def generate_rust_lcov(_: argparse.Namespace) -> None: + """Generate an LCOV coverage report for Rust code.""" output_path = Path(os.environ.get("LCOV_OUTPUT", "lcov.info")) subprocess.run( [ @@ -651,6 +760,7 @@ def generate_rust_lcov(_: argparse.Namespace) -> None: def generate_rust_html(_: argparse.Namespace) -> None: + """Generate an HTML coverage report for Rust code.""" output_dir = Path(os.environ.get("HTML_OUTPUT_DIR", "htmlcov")) output_dir.mkdir(parents=True, exist_ok=True) subprocess.run( @@ -668,6 +778,7 @@ def generate_rust_html(_: argparse.Namespace) -> None: def compute_rust_coverage(_: argparse.Namespace) -> None: + """Compute the Rust line-coverage percentage and expose it as an output.""" path = Path(os.environ.get("LCOV_FILE", "lcov.info")) if not path.is_file(): raise FileNotFoundError(f"{path} not found") @@ -689,6 +800,7 @@ def compute_rust_coverage(_: argparse.Namespace) -> None: def enforce_coverage_threshold(_: argparse.Namespace) -> None: + """Fail when COVERAGE_PERCENT is below COVERAGE_THRESHOLD.""" threshold = float(os.environ.get("COVERAGE_THRESHOLD", "0")) percent_str = os.environ.get("COVERAGE_PERCENT") if percent_str is None: @@ -702,6 +814,7 @@ def enforce_coverage_threshold(_: argparse.Namespace) -> None: def docker_build(_: argparse.Namespace) -> None: + """Build the repository's Docker image.""" dockerfile = Path(os.environ.get("DOCKERFILE_PATH", "Dockerfile")) image_name = os.environ.get("DOCKER_IMAGE", "test-image") if not dockerfile.is_file(): @@ -715,21 +828,28 @@ def docker_build(_: argparse.Namespace) -> None: def docker_test_compose(_: argparse.Namespace) -> None: - if Path("docker-compose.yml").is_file() or Path("docker-compose.yaml").is_file(): + """Validate the docker compose configuration.""" + if ( + Path("docker-compose.yml").is_file() + or Path("docker-compose.yaml").is_file() + ): subprocess.run(["docker-compose", "config"], check=True) else: print("â„šī¸ No docker-compose file found") def docs_check_links(_: argparse.Namespace) -> None: + """Placeholder for documentation link checking.""" print("â„šī¸ Link checking would go here") def docs_validate_structure(_: argparse.Namespace) -> None: + """Placeholder for documentation structure validation.""" print("â„šī¸ Documentation structure validation would go here") def run_benchmarks(_: argparse.Namespace) -> None: + """Run Go benchmarks when the repository contains any.""" has_benchmarks = False for path in Path(".").rglob("*_test.go"): try: @@ -746,7 +866,9 @@ def run_benchmarks(_: argparse.Namespace) -> None: subprocess.run(["go", "test", "-bench=.", "-benchmem", "./..."], check=True) -def _matrix_entries(versions: list[str], oses: list[str], version_key: str) -> list[dict[str, Any]]: +def _matrix_entries( + versions: list[str], oses: list[str], version_key: str +) -> list[dict[str, Any]]: matrix: list[dict[str, Any]] = [] for os_index, runner in enumerate(oses): for ver_index, version in enumerate(versions): @@ -761,6 +883,7 @@ def _matrix_entries(versions: list[str], oses: list[str], version_key: str) -> l def generate_matrices(_: argparse.Namespace) -> None: + """Build the language version matrices and coverage threshold outputs.""" fallback_go = os.environ.get("FALLBACK_GO_VERSION", "1.24") fallback_python = os.environ.get("FALLBACK_PYTHON_VERSION", "3.13") fallback_rust = os.environ.get("FALLBACK_RUST_VERSION", "stable") @@ -781,7 +904,9 @@ def generate_matrices(_: argparse.Namespace) -> None: rust_matrix = _matrix_entries(rust_versions, os_list, "rust-version") frontend_matrix = _matrix_entries(node_versions, os_list, "node-version") - write_output("go-matrix", json.dumps({"include": go_matrix}, separators=(",", ":"))) + write_output( + "go-matrix", json.dumps({"include": go_matrix}, separators=(",", ":")) + ) write_output( "python-matrix", json.dumps({"include": python_matrix}, separators=(",", ":")), @@ -795,11 +920,14 @@ def generate_matrices(_: argparse.Namespace) -> None: json.dumps({"include": frontend_matrix}, separators=(",", ":")), ) - coverage_threshold = _config_path(fallback_threshold, "testing", "coverage", "threshold") + coverage_threshold = _config_path( + fallback_threshold, "testing", "coverage", "threshold" + ) write_output("coverage-threshold", str(coverage_threshold)) def generate_ci_summary(_: argparse.Namespace) -> None: + """Write the per-job CI results table to the step summary.""" primary_language = os.environ.get("PRIMARY_LANGUAGE", "unknown") steps = [ ("Detect Changes", os.environ.get("JOB_DETECT_CHANGES", "skipped")), @@ -842,7 +970,9 @@ def generate_ci_summary(_: argparse.Namespace) -> None: "## 🧭 Detection", f"- Primary language: {primary_language}", ] - summary_lines.extend(f"- {label}: {value}" for label, value in languages.items()) + summary_lines.extend( + f"- {label}: {value}" for label, value in languages.items() + ) summary_lines.extend( [ "", @@ -858,13 +988,16 @@ def generate_ci_summary(_: argparse.Namespace) -> None: "## 📁 Changed Files", ] ) - summary_lines.extend(f"- {label}: {value}" for label, value in files_changed.items()) + summary_lines.extend( + f"- {label}: {value}" for label, value in files_changed.items() + ) summary_lines.append("") append_summary("\n".join(summary_lines) + "\n") def check_ci_status(_: argparse.Namespace) -> None: + """Exit non-zero when any required CI job did not succeed.""" job_envs = { "Workflow Lint": os.environ.get("JOB_WORKFLOW_LINT"), "Workflow Scripts": os.environ.get("JOB_WORKFLOW_SCRIPTS"), @@ -876,7 +1009,11 @@ def check_ci_status(_: argparse.Namespace) -> None: "Docs CI": os.environ.get("JOB_DOCS"), } - failures = [job for job, status in job_envs.items() if status in {"failure", "cancelled"}] + failures = [ + job + for job, status in job_envs.items() + if status in {"failure", "cancelled"} + ] if failures: print(f"❌ CI Pipeline failed: {', '.join(failures)}") raise SystemExit(1) @@ -884,6 +1021,7 @@ def check_ci_status(_: argparse.Namespace) -> None: def build_parser() -> argparse.ArgumentParser: + """Build the argument parser mapping each subcommand to its handler.""" parser = argparse.ArgumentParser(description="CI workflow helper commands.") subparsers = parser.add_subparsers(dest="command", required=True) @@ -926,6 +1064,7 @@ def build_parser() -> argparse.ArgumentParser: def main() -> None: + """Parse the command line and run the selected subcommand.""" parser = build_parser() args = parser.parse_args() handler = getattr(args, "handler", None) diff --git a/changelog.d/20261020-pin-ruff-python-lint.md b/changelog.d/20261020-pin-ruff-python-lint.md new file mode 100644 index 00000000..319e7ef9 --- /dev/null +++ b/changelog.d/20261020-pin-ruff-python-lint.md @@ -0,0 +1,51 @@ +### Fixed + +#### The shared Python lint now runs a pinned ruff + +`ci_workflow.py python-lint` ran `pip install --upgrade ruff` whenever ruff was +not on `PATH`, so every consumer linted with whatever ruff was newest and a new +release could break all of their CI overnight. It also ran whichever `ruff` the +`PATH` resolved, so an already-installed ruff of another version (pipx, the +runner image) was used silently. + +- `RUFF_VERSION = "0.17.0"` in `ci_workflow.py` is the single pinned version + (the one `audiobook-organizer` formats against). The step installs + `ruff==` with `sys.executable -m pip` whenever the interpreter's ruff + differs, re-checks the version, and raises on a mismatch. +- ruff is invoked as `python -m ruff`, so the verified install is the one that + runs. +- `reusable-ci.yml` gains an optional `ruff-version` input (default empty, meaning + the pinned constant) passed to the step as `RUFF_VERSION`. It takes a bare + version such as `0.17.0`; a leading `v` is stripped. +- The `CI Self Tests` lint step in `ci-tests.yml` also installed ruff unpinned; + it now installs `ruff==0.17.0`. +- `ci_workflow.py` itself now passes the repo-root `ruff.toml` (31 missing + docstrings, the `_CONFIG_CACHE` global replaced by `functools.lru_cache`, and + the urllib fallback refusing non-https URLs), which the Python CI check + requires. + +#### Rollout: the pin reaches consumers in a follow-up PR + +`reusable-ci.yml` checks out `ci_workflow.py` from the ref in the caller's +`.github/ghcommon-ref.txt`, or from the hard-coded fallback SHA (currently +`1f0d70e`) when the caller has no such file. Neither ref contains this change, +and this PR cannot name its own merge SHA. As with github-common#374 (the +fallback moved to `1f0d70e` afterwards, in `4b163cf`), the SHA bump is a +follow-up PR: after this merges, point the five fallback `ref=` lines in +`reusable-ci.yml` at the merge commit. Until then the `ruff-version` input is +accepted but has no effect, and the lint step is unchanged for everyone. + +Known `reusable-ci.yml` consumers once the fallback moves: + +- `cockroach-rollout-agent` (`@main`, has Python): moves to ruff 0.17.0 + automatically, unless it has its own `ghcommon-ref.txt`. +- `magnet-handler` and `migrate-loop`: pinned to older workflow SHAs, so + unaffected until they bump. +- `audiobook-organizer`: needs both its `reusable-ci.yml` pin and its + `.github/ghcommon-ref.txt` bumped to a commit that includes this change. + +A repo that formats with a newer ruff than 0.17.0 can set +`with: ruff-version: ""` on its `reusable-ci.yml` call instead of +waiting for `RUFF_VERSION` to move. The separate `Lint Python workflow scripts` +job in `reusable-ci.yml` still pins `ruff==0.16.10` on its own; a TODO comment +there marks it for unification. diff --git a/docs/guid-index.md b/docs/guid-index.md index 146e6102..00a3138d 100644 --- a/docs/guid-index.md +++ b/docs/guid-index.md @@ -1,6 +1,6 @@ # GUID Index -Generated by `generate_guid_index.py` from 469 file headers. +Generated by `generate_guid_index.py` from 470 file headers. Do not edit by hand — regenerate instead. ## Duplicate GUIDs @@ -731,6 +731,7 @@ These are not valid UUIDs (likely leftover placeholder text). | test | `a6b7c8d9-e0f1-2a3b-4c5d-6e7f8a9b0c1d` | .github/test-files/test.sh | | test autofix | `7a8b9c0d-1e2f-3a4b-5c6d-7e8f9a0b1c2d` | .github/test-configs/test-autofix.env | | test automation workflow | `d9f5c8b3-2c4d-4e5f-9a7b-3c2d1f0e1a2b` | tests/workflow_scripts/test_automation_workflow.py | +| test ci workflow | `be8f15eb-d640-417d-8c19-bb923f338f54` | tests/workflow_scripts/test_ci_workflow.py | | test doc summary | `12345678-1234-5678-9012-123456789abc` | scripts/test-doc-summary.sh | | test docs workflow | `2f3a4b5c-6d7e-8f90-a1b2-c3d4e5f60718` | tests/workflow_scripts/test_docs_workflow.py | | test full | `c2d3e4f5-a6b7-8c9d-0e1f-2a3b4c5d6e7f` | .github/test-configs/test-full.env | diff --git a/tests/workflow_scripts/test_ci_workflow.py b/tests/workflow_scripts/test_ci_workflow.py index ea2b5283..0c0da3c2 100644 --- a/tests/workflow_scripts/test_ci_workflow.py +++ b/tests/workflow_scripts/test_ci_workflow.py @@ -1,3 +1,8 @@ +# file: tests/workflow_scripts/test_ci_workflow.py +# version: 1.0.1 +# guid: be8f15eb-d640-417d-8c19-bb923f338f54 +# last-edited: 2026-10-20 + import argparse import json import subprocess @@ -9,7 +14,7 @@ @pytest.fixture(autouse=True) def reset_config_cache(): - ci_workflow._CONFIG_CACHE = None # type: ignore[attr-defined] + ci_workflow.get_repository_config.cache_clear() def test_debug_filter_outputs(monkeypatch, capsys): @@ -131,7 +136,7 @@ def fake_run(cmd, check=False, capture_output=False, text=False, **kwargs): monkeypatch.setenv("COVERAGE_THRESHOLD", "70") monkeypatch.delenv("REPOSITORY_CONFIG", raising=False) - ci_workflow._CONFIG_CACHE = None # type: ignore[attr-defined] + ci_workflow.get_repository_config.cache_clear() monkeypatch.setattr(ci_workflow.subprocess, "run", fake_run) monkeypatch.setattr(ci_workflow.shutil, "which", lambda name: "go") @@ -247,7 +252,7 @@ def test_go_test_uses_config_threshold(tmp_path, monkeypatch): config = {"testing": {"coverage": {"threshold": 90}}} monkeypatch.setenv("REPOSITORY_CONFIG", json.dumps(config)) - ci_workflow._CONFIG_CACHE = None # type: ignore[attr-defined] + ci_workflow.get_repository_config.cache_clear() commands = [] @@ -287,7 +292,7 @@ def test_generate_matrices_uses_repository_config(tmp_path, monkeypatch): monkeypatch.setenv("FALLBACK_RUST_VERSION", "stable") monkeypatch.setenv("FALLBACK_NODE_VERSION", "22") monkeypatch.setenv("FALLBACK_COVERAGE_THRESHOLD", "80") - ci_workflow._CONFIG_CACHE = None # type: ignore[attr-defined] + ci_workflow.get_repository_config.cache_clear() ci_workflow.generate_matrices(argparse.Namespace()) @@ -350,3 +355,105 @@ def test_check_ci_status_failure(monkeypatch): monkeypatch.setenv("JOB_DOCS", "skipped") with pytest.raises(SystemExit): ci_workflow.check_ci_status(argparse.Namespace()) + + +def _lint_repo(tmp_path, monkeypatch): + """A git repo whose last commit touched one Python file.""" + monkeypatch.chdir(tmp_path) + (tmp_path / "ok.py").write_text('"""Doc."""\n', encoding="utf-8") + for cmd in ( + ["git", "init", "-q"], + ["git", "add", "ok.py"], + [ + "git", + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "commit", + "-qm", + "x", + ], + ): + subprocess.run(cmd, check=True) # noqa: S603 + + +def test_python_lint_installs_pinned_ruff_and_runs_via_interpreter( + tmp_path, monkeypatch +): + """A mismatched ruff is replaced by ruff==RUFF_VERSION and run as python -m ruff.""" + _lint_repo(tmp_path, monkeypatch) + monkeypatch.delenv("RUFF_VERSION", raising=False) + monkeypatch.delenv("GITHUB_EVENT_PATH", raising=False) + state = {"installed": "0.0.1"} + commands: list[list[str]] = [] + real_run = subprocess.run + + def fake_run(cmd, *args, **kwargs): + if cmd[:3] == [ci_workflow.sys.executable, "-m", "ruff"]: + commands.append(list(cmd)) + if cmd[3] == "--version": + return subprocess.CompletedProcess( + cmd, 0, stdout=f"ruff {state['installed']}\n" + ) + return subprocess.CompletedProcess(cmd, 0) + if "pip" in cmd: + commands.append(list(cmd)) + state["installed"] = cmd[-1].split("==")[1] + return subprocess.CompletedProcess(cmd, 0) + return real_run(cmd, *args, **kwargs) + + monkeypatch.setattr(ci_workflow.subprocess, "run", fake_run) + ci_workflow.python_lint(argparse.Namespace()) + + installs = [c for c in commands if "pip" in c] + assert installs == [ + [ + ci_workflow.sys.executable, + "-m", + "pip", + "install", + f"ruff=={ci_workflow.RUFF_VERSION}", + ] + ] + assert ci_workflow.RUFF_VERSION == "0.17.0" + ran = [c[3] for c in commands if c[3] in ("format", "check")] + assert ran == ["format", "check"] + + +@pytest.mark.parametrize("override", ["0.16.10", "v0.16.10", " v0.16.10 "]) +def test_python_lint_honors_ruff_version_override( + tmp_path, monkeypatch, override +): + """A RUFF_VERSION override, with or without a leading v, is what gets installed.""" + _lint_repo(tmp_path, monkeypatch) + monkeypatch.setenv("RUFF_VERSION", override) + monkeypatch.delenv("GITHUB_EVENT_PATH", raising=False) + installs: list[str] = [] + real_run = subprocess.run + + def fake_run(cmd, *args, **kwargs): + if "pip" in cmd: + installs.append(cmd[-1]) + return subprocess.CompletedProcess(cmd, 0) + if cmd[:3] == [ci_workflow.sys.executable, "-m", "ruff"]: + if cmd[3] == "--version": + version = "0.16.10" if installs else "0.0.1" + return subprocess.CompletedProcess( + cmd, 0, stdout=f"ruff {version}\n" + ) + return subprocess.CompletedProcess(cmd, 0) + return real_run(cmd, *args, **kwargs) + + monkeypatch.setattr(ci_workflow.subprocess, "run", fake_run) + ci_workflow.python_lint(argparse.Namespace()) + assert installs == ["ruff==0.16.10"] + + +def test_ensure_ruff_fails_loudly_on_version_mismatch(monkeypatch): + def fake_run(cmd, *args, **kwargs): + return subprocess.CompletedProcess(cmd, 0, stdout="ruff 9.9.9\n") + + monkeypatch.setattr(ci_workflow.subprocess, "run", fake_run) + with pytest.raises(RuntimeError, match=r"expected ruff 0\.17\.0"): + ci_workflow._ensure_ruff("0.17.0")