From 8648c45529d180a3ef2e681d8acb14d6bbb28344 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Sun, 25 Oct 2026 02:23:40 +0900 Subject: [PATCH 1/5] build(make): bootstrap Hatch once, type-check once and add install-ci Every Make target depends on bootstrap, which ran `pip install --upgrade pip` and `pip install hatch` each time, and `make check` ran `hatch run lint` (already style plus typecheck) and then `make typecheck` again. With stub pip and Hatch programs, one `make check-all` made 12 pip calls and two type-check runs. bootstrap now creates `.venv` only while it is missing and installs Hatch only while `.venv/bin/hatch` is not an executable regular file; it never upgrades pip. `check` is `lint`, so the type check runs once. `PYTHON` (default python3) selects the interpreter that creates `.venv`, and `make install-ci PYTHON=` installs `.[dev]` into exactly that interpreter, never through Hatch, and refuses Python older than 3.11. `check-all` and the security targets are unchanged. tests/test_make_contract.py asserts the expansion with `make -n` and runs the real recipes against stub programs on a PATH that holds nothing else. Refs #811 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- Makefile | 31 +++-- tests/test_make_contract.py | 227 ++++++++++++++++++++++++++++++++++++ 2 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 tests/test_make_contract.py diff --git a/Makefile b/Makefile index 35eaa74d..98406287 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ VENV_DIR := .venv -PYTHON := $(VENV_DIR)/bin/python +# The interpreter that creates $(VENV_DIR), which only hosts Hatch, and that `make install-ci` +# installs into: `make install PYTHON=python3.12`. Hatch still provisions its own default +# environment (Python 3.11, see [tool.hatch.envs.default] in pyproject.toml). +PYTHON ?= python3 PIP := $(VENV_DIR)/bin/pip HATCH := $(VENV_DIR)/bin/hatch SPEC_PREVIEW_PNG := docs/assets/webhook_receiver_openapi_spec_preview.png @@ -11,16 +14,13 @@ SWAGGER_PREVIEW_PORT := 8123 SPEC_PREVIEW_PORT := 8124 EXAMPLES_PREVIEW_PORT := 8130 +# Idempotent: the virtual environment is created only while it is missing, Hatch is installed only +# while it is not an executable regular file (`-x` alone also accepts a directory), and pip is +# never upgraded. .PHONY: bootstrap bootstrap: - @if [ ! -d "$(VENV_DIR)" ]; then \ - echo "Creating virtual environment..."; \ - python3 -m venv $(VENV_DIR); \ - fi - @echo "Ensuring Hatch is installed in virtual environment..." - @$(PIP) install --upgrade pip > /dev/null - @$(PIP) install hatch > /dev/null - @echo "Hatch installed at $(HATCH)" + @test -d "$(VENV_DIR)" || { echo "Creating virtual environment..."; $(PYTHON) -m venv "$(VENV_DIR)"; } + @{ [ -f "$(HATCH)" ] && [ -x "$(HATCH)" ]; } || { echo "Installing Hatch..."; "$(PIP)" install --disable-pip-version-check --quiet hatch; } .PHONY: ensure-hatch ensure-hatch: bootstrap @@ -34,6 +34,13 @@ install: ensure-hatch $(MAKE) precommit-install; \ fi +# Installs into the interpreter you select and never through Hatch, whose default environment is +# pinned to Python 3.11, so a CI matrix cell keeps its own Python: `make install-ci PYTHON=python3.12`. +.PHONY: install-ci +install-ci: + @$(PYTHON) -c 'import sys; sys.exit(sys.version_info < (3, 11) and "install-ci needs Python 3.11 or newer, got " + sys.version.split()[0])' + @$(PYTHON) -m pip install -e ".[dev]" + .PHONY: shell shell: ensure-hatch @$(HATCH) shell @@ -96,10 +103,10 @@ security-development: security-browser: @$(SECURITY_PYTHON) -m tools.browser_integrity --output "$(SECURITY_OUTPUT)" +# `hatch run lint` is `style` followed by `typecheck`, so `check` is `lint`: a further +# `make typecheck` would run the type check a second time. .PHONY: check -check: ensure-hatch - @$(MAKE) lint - @$(MAKE) typecheck +check: lint @echo "Lint and type check passed." .PHONY: lint-workflows diff --git a/tests/test_make_contract.py b/tests/test_make_contract.py new file mode 100644 index 00000000..9ceb3240 --- /dev/null +++ b/tests/test_make_contract.py @@ -0,0 +1,227 @@ +"""Contract of the Makefile targets that set up and check a checkout (#811). + +``make -n`` prints recipes without running them, so the expansion cases need no network, Hatch or +virtual environment. The behavior cases run the real recipes in a scratch directory whose ``PATH`` +holds only stub programs that log their arguments: nothing is installed, no network is used and no +real tool can run. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import sys +import tomllib + +import pytest + +pytestmark = pytest.mark.skipif( + sys.platform == "win32" or shutil.which("make") is None, + reason="runs the real Makefile recipes through /bin/sh", +) + +ROOT = Path(__file__).resolve().parents[1] +MAKE = shutil.which("make") or "make" +_PYPROJECT = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) +HATCH_SCRIPTS: dict[str, str] = _PYPROJECT["tool"]["hatch"]["envs"]["default"]["scripts"] + + +def script_runs(commands: str, name: str) -> int: + """Count the runs of Hatch script ``name``, following scripts that run other scripts. + + Hatch's ``lint`` script runs ``style`` and ``typecheck``, so a second type check can hide + inside a target that ``make -n`` shows as one ``hatch run lint``. + """ + called = re.findall(r"\bhatch run ([\w-]+)", commands) + return sum((s == name) + script_runs(HATCH_SCRIPTS.get(s, ""), name) for s in called) + + +def plan(*arguments: str) -> str: + """Return the commands ``make -n`` prints for this repository's own Makefile.""" + result = subprocess.run( + [MAKE, "--no-print-directory", "-n", *arguments], + cwd=ROOT, + env={"PATH": os.environ["PATH"]}, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + +def installs_hatch(call: str) -> bool: + words = call.split() + return words[:2] == ["pip", "install"] and words[-1] == "hatch" and "--upgrade" not in words + + +@dataclass(frozen=True) +class Sandbox: + """A scratch copy of the Makefile, run with ``PATH`` limited to stubs that log their calls.""" + + root: Path + + @property + def venv(self) -> Path: + return self.root / "venv" + + @property + def calls(self) -> list[str]: + log = self.root / "calls.log" + return log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + + def tool(self, name: str, directory: Path | None = None, body: str = "") -> Path: + """Create a stub that logs `` `` and then runs ``body``.""" + path = (directory or self.root / "bin") / name + path.parent.mkdir(parents=True, exist_ok=True) + log = shlex.quote(str(self.root / "calls.log")) + line = f'printf "%s\\n" "{name} $*" >> {log}\n' # printf: `echo` would expand backslashes + path.write_text(f"#!/bin/sh\n{line}{body}exit 0\n", encoding="utf-8") + path.chmod(0o755) + return path + + def pip(self, directory: Path) -> None: + """Add a ``pip`` whose ``install ... hatch`` puts a logging ``hatch`` beside it.""" + template = self.tool("hatch", self.root / "templates") # `cp` keeps its executable bit + install = f'for last; do :; done\n[ "$last" = hatch ] && cp {template} "${{0%/*}}/hatch"\n' + self.tool("pip", directory, install) + + def interpreter(self, name: str, version: tuple[int, int] = (3, 12)) -> Path: + """Add a Python stub that reports ``version`` and whose ``-m venv DIR`` holds a stub pip.""" + self.pip(self.root / "templates") + pip = self.root / "templates" / "pip" + probe = f"import sys; sys.version_info = ({version[0]}, {version[1]}, 0); exec(sys.argv[1])" + body = ( + f'[ "$1" = -c ] && exec {shlex.quote(sys.executable)} -c "{probe}" "$2"\n' + f'[ "$1 $2" = "-m venv" ] && mkdir -p "$3/bin" && cp {pip} "$3/bin/pip"\n' + ) + return self.tool(name, body=body) + + def make(self, *arguments: str, **environment: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [MAKE, "--no-print-directory", f"VENV_DIR={self.venv}", *arguments], + cwd=self.root, + env={"PATH": str(self.root / "bin"), **environment}, + capture_output=True, + text=True, + check=False, + ) + + +@pytest.fixture +def sandbox(tmp_path: Path) -> Sandbox: + shutil.copy(ROOT / "Makefile", tmp_path / "Makefile") + (tmp_path / "bin").mkdir() + for program in ("cp", "mkdir"): # the only real programs the stubs run + (tmp_path / "bin" / program).symlink_to(shutil.which(program) or program) + return Sandbox(tmp_path) + + +@pytest.mark.parametrize( + ("target", "style", "typecheck", "test"), [("check", 1, 1, 0), ("check-all", 1, 1, 1)] +) +def test_each_check_runs_once(target: str, style: int, typecheck: int, test: int) -> None: + # Given the commands the target would run + commands = plan(target) + + # Then style, the type check and the tests run the expected number of times + runs = [script_runs(commands, name) for name in ("style", "typecheck", "test")] + assert runs == [style, typecheck, test] + + +def test_check_all_keeps_every_gate_it_had() -> None: + # Given the commands `make check-all` would run + commands = plan("check-all") + + # Then each workflow lint and the Bandit evidence run once + gates = ("lint_release_workflows", "lint_workflow_pins", "lint_hatch_matrix", "bandit") + assert [commands.count(gate) for gate in gates] == [1, 1, 1, 1] + + +def test_bootstrap_installs_hatch_once_and_only_while_it_is_missing(sandbox: Sandbox) -> None: + # Given a virtual environment that has pip but no Hatch + sandbox.pip(sandbox.venv / "bin") + + # When bootstrap runs twice + for _ in range(2): + assert sandbox.make("bootstrap").returncode == 0 + + # Then one pip call across both runs installed Hatch, without --upgrade + (call,) = sandbox.calls + assert installs_hatch(call) + + +def test_bootstrap_leaves_an_installed_hatch_alone(sandbox: Sandbox) -> None: + # Given a virtual environment that already holds Hatch + sandbox.pip(sandbox.venv / "bin") + sandbox.tool("hatch", sandbox.venv / "bin") + + # When bootstrap runs + result = sandbox.make("bootstrap") + + # Then pip is never called + assert result.returncode == 0, result.stderr + assert sandbox.calls == [] + + +@pytest.mark.parametrize("kind", ["directory", "non-executable file"]) +def test_only_an_executable_regular_file_counts_as_hatch(sandbox: Sandbox, kind: str) -> None: + # Given something at the Hatch path that `test -x` or `command -v` alone would accept + sandbox.pip(sandbox.venv / "bin") + hatch = sandbox.venv / "bin" / "hatch" + if kind == "directory": + hatch.mkdir() + else: + hatch.write_text("#!/bin/sh\n", encoding="utf-8") # created without the executable bit + + # When bootstrap runs + result = sandbox.make("bootstrap") + + # Then it installs Hatch + assert result.returncode == 0, result.stderr + (call,) = sandbox.calls + assert installs_hatch(call) + + +def test_a_fresh_checkout_gets_one_venv_from_the_selected_interpreter(sandbox: Sandbox) -> None: + # Given no virtual environment and a selected interpreter + python = sandbox.interpreter("sentinel") + + # When bootstrap runs + result = sandbox.make("bootstrap", f"PYTHON={python}") + + # Then that interpreter creates the environment and Hatch is the only thing installed + assert result.returncode == 0, result.stderr + venv_call, pip_call = sandbox.calls + assert venv_call == f"sentinel -m venv {sandbox.venv}" + assert installs_hatch(pip_call) + + +def test_install_ci_installs_into_the_selected_interpreter_and_not_through_hatch() -> None: + # Given the commands `make install-ci PYTHON=python3.12` would run + commands = plan("install-ci", "PYTHON=python3.12") + + # Then the matrix interpreter receives the install, so a cell keeps its own Python + assert 'python3.12 -m pip install -e ".[dev]"' in commands + assert "hatch" not in commands + + +@pytest.mark.parametrize(("version", "installs"), [((3, 10), False), ((3, 11), True)]) +def test_install_ci_needs_python_311( + sandbox: Sandbox, version: tuple[int, int], installs: bool +) -> None: + # Given an interpreter of the given version + python = sandbox.interpreter("sentinel", version) + + # When install-ci runs + result = sandbox.make("install-ci", f"PYTHON={python}") + + # Then the version guard ran, and pip runs only for Python 3.11 or newer + assert any(call.startswith("sentinel -c ") for call in sandbox.calls) + assert (result.returncode == 0) is installs + assert ("sentinel -m pip install -e .[dev]" in sandbox.calls) is installs From 179042768699cf27dc314955575343d8bf4654b0 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Sun, 25 Oct 2026 02:26:36 +0900 Subject: [PATCH 2/5] build(make): build docs with --strict and the project's docs extra `make docs` ran `mkdocs build`, while the CI docs gate runs `mkdocs build --strict`, so a broken link built with a warning and exit 0. Its CI branch also hid pip's output and errors (`> /dev/null 2>&1`, `;` between the steps) and installed `mkdocstrings[python]<1.0`, which resolves to a mkdocstrings that cannot render api.md (`Highlight.__init__() missing 1 required positional argument: 'md'`), so `CI=1 make docs` never built the docs at all. Both branches now run `mkdocs build --strict`. The CI branch installs `-e ".[docs]"`, the extra the CI gate installs, so the dependency pins live only in pyproject.toml, and it stops with pip's own error when the install fails instead of building with whatever was already installed. No target upgrades pip any more, and a test reads the Makefile to keep it that way. Refs #811 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- Makefile | 12 ++++--- tests/test_make_contract.py | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 98406287..52423deb 100644 --- a/Makefile +++ b/Makefile @@ -178,16 +178,18 @@ version: ensure-hatch @echo "Current version:" @$(HATCH) version +# The CI docs gate (`mkdocs build --strict` after `pip install -e ".[docs]"`): a broken link or nav +# entry fails the build, the dependencies come from pyproject.toml, and a failed install stops the +# build with pip's own error instead of building with whatever was already installed. .PHONY: docs docs: @if [ -n "$$CI" ]; then \ echo "CI detected: running MkDocs directly"; \ - python -m pip install --upgrade pip > /dev/null 2>&1 || true; \ - pip install "mkdocs<2.0" "mkdocs-material<10.0" "mkdocstrings[python]<1.0" > /dev/null 2>&1; \ - mkdocs build; \ + pip install --disable-pip-version-check --quiet -e ".[docs]" && \ + mkdocs build --strict; \ else \ - $(MAKE) ensure-hatch > /dev/null; \ - $(HATCH) run mkdocs build; \ + $(MAKE) ensure-hatch > /dev/null && \ + $(HATCH) run mkdocs build --strict; \ fi .PHONY: docs-serve diff --git a/tests/test_make_contract.py b/tests/test_make_contract.py index 9ceb3240..2b50e48d 100644 --- a/tests/test_make_contract.py +++ b/tests/test_make_contract.py @@ -143,6 +143,11 @@ def test_check_all_keeps_every_gate_it_had() -> None: assert [commands.count(gate) for gate in gates] == [1, 1, 1, 1] +def test_makefile_never_upgrades_pip() -> None: + # Also covers `docs`: its recipe line contains $(MAKE), so `make -n` would run it for real. + assert "--upgrade" not in (ROOT / "Makefile").read_text(encoding="utf-8") + + def test_bootstrap_installs_hatch_once_and_only_while_it_is_missing(sandbox: Sandbox) -> None: # Given a virtual environment that has pip but no Hatch sandbox.pip(sandbox.venv / "bin") @@ -225,3 +230,65 @@ def test_install_ci_needs_python_311( assert any(call.startswith("sentinel -c ") for call in sandbox.calls) assert (result.returncode == 0) is installs assert ("sentinel -m pip install -e .[dev]" in sandbox.calls) is installs + + +@pytest.mark.parametrize( + ("environment", "build"), + [({}, "hatch run mkdocs build --strict"), ({"CI": "1"}, "mkdocs build --strict")], + ids=["local", "ci"], +) +def test_docs_builds_strictly(sandbox: Sandbox, environment: dict[str, str], build: str) -> None: + # Given Hatch, pip and MkDocs stubs + sandbox.tool("hatch", sandbox.venv / "bin") + sandbox.tool("pip") + sandbox.tool("mkdocs") + + # When docs runs locally and under CI + result = sandbox.make("docs", **environment) + + # Then MkDocs ran in strict mode + assert result.returncode == 0, result.stderr + assert build in sandbox.calls + + +def test_docs_under_ci_installs_the_docs_extra_the_ci_gate_installs(sandbox: Sandbox) -> None: + # Given pip and MkDocs stubs + sandbox.tool("pip") + sandbox.tool("mkdocs") + + # When docs runs under CI + result = sandbox.make("docs", CI="1") + + # Then the dependencies come from pyproject.toml, not from pins kept in the Makefile + assert result.returncode == 0, result.stderr + assert any( + call.startswith("pip install") and call.endswith("-e .[docs]") for call in sandbox.calls + ) + + +@pytest.mark.parametrize("environment", [{}, {"CI": "1"}], ids=["local", "ci"]) +def test_docs_fails_when_the_build_fails(sandbox: Sandbox, environment: dict[str, str]) -> None: + # Given a MkDocs build that fails + sandbox.tool("hatch", sandbox.venv / "bin", "exit 1\n") + sandbox.tool("pip") + sandbox.tool("mkdocs", body="exit 1\n") + + # When docs runs + result = sandbox.make("docs", **environment) + + # Then make fails + assert result.returncode != 0 + + +def test_docs_shows_a_failed_install_and_does_not_build_after_it(sandbox: Sandbox) -> None: + # Given a pip that fails with a message + sandbox.tool("pip", body='echo "ERROR: no matching distribution" >&2\nexit 1\n') + sandbox.tool("mkdocs") + + # When docs runs under CI + result = sandbox.make("docs", CI="1") + + # Then the error is visible and MkDocs never ran + assert result.returncode != 0 + assert "ERROR: no matching distribution" in result.stderr + assert "mkdocs build --strict" not in sandbox.calls From 746a3781d546729fda88dbaa06b416b3a070285b Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Sun, 25 Oct 2026 02:27:41 +0900 Subject: [PATCH 3/5] build(make): make doctor a read-only prerequisite check that can fail `make doctor` could not fail for a missing Hatch or func (`|| echo`): it failed only by accident, when `.venv/bin/python` did not exist, and its `hatch env run pip list` creates the Hatch environment when it is missing, so the report wrote. doctor now installs and writes nothing. It requires a Python 3.11 or newer on PATH and the `.venv/bin/hatch` this Makefile runs, each counted only as an executable regular file, and exits non-zero with a `MISSING` line when either is absent. func and azurite are listed as optional, because only a local Functions host needs them. The old output (Python version, pip list, func version, pre-commit hook state) is gone; nothing in the repository parses it. Refs #811 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- Makefile | 25 ++++++++--- tests/test_make_contract.py | 85 ++++++++++++++++++++++++++++++++++--- 2 files changed, 100 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 52423deb..29768a03 100644 --- a/Makefile +++ b/Makefile @@ -289,13 +289,28 @@ demo-examples: ensure-hatch kill $$SERVER_PID 2>/dev/null || true; \ wait $$SERVER_PID 2>/dev/null || true +# Read-only prerequisite check: it installs and writes nothing, and fails when a required tool is +# missing. A tool counts only if `command -v` resolves it to an executable regular file, because +# `command -v` also reports a non-executable file and `-x` alone also accepts a directory. Python +# 3.11 or newer is looked up on PATH, $(HATCH) is the Hatch this Makefile runs (`make install` +# creates it), and func and azurite are needed only to run a local Functions host. .PHONY: doctor doctor: - @echo "Python version:" && $(PYTHON) --version - @echo "Installed packages:" && $(HATCH) env run pip list || echo "No hatch environment found" - @echo "Azure Function Core Tools version:" && func --version || echo "func not found" - @echo "Pre-commit hook installed:" - @if [ -f .git/hooks/pre-commit ]; then echo yes; else echo no; fi + @missing=0; \ + have() { p="$$(command -v "$$1")"; [ -f "$$p" ] && [ -x "$$p" ]; }; \ + py=; \ + for tool in python3.14 python3.13 python3.12 python3.11 python3; do \ + if have "$$tool" && "$$tool" -c 'import sys; sys.exit(sys.version_info < (3, 11))'; then py=$$tool; break; fi; \ + done; \ + if [ -n "$$py" ]; then echo "ok $$py ($$("$$py" --version))"; \ + else echo "MISSING python >= 3.11 (required)"; missing=1; fi; \ + if have "$(HATCH)"; then echo "ok $(HATCH)"; \ + else echo "MISSING $(HATCH) (required; make install creates it)"; missing=1; fi; \ + for tool in func azurite; do \ + if have "$$tool"; then echo "ok $$tool"; \ + else echo "missing $$tool (optional: needed only to run a local Functions host)"; fi; \ + done; \ + exit $$missing .PHONY: clean clean: diff --git a/tests/test_make_contract.py b/tests/test_make_contract.py index 2b50e48d..7a27ea9d 100644 --- a/tests/test_make_contract.py +++ b/tests/test_make_contract.py @@ -8,6 +8,7 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass import os from pathlib import Path @@ -97,11 +98,19 @@ def interpreter(self, name: str, version: tuple[int, int] = (3, 12)) -> Path: pip = self.root / "templates" / "pip" probe = f"import sys; sys.version_info = ({version[0]}, {version[1]}, 0); exec(sys.argv[1])" body = ( + f'[ "$1" = --version ] && echo "Python {version[0]}.{version[1]}.0"\n' f'[ "$1" = -c ] && exec {shlex.quote(sys.executable)} -c "{probe}" "$2"\n' f'[ "$1 $2" = "-m venv" ] && mkdir -p "$3/bin" && cp {pip} "$3/bin/pip"\n' ) return self.tool(name, body=body) + def doctor_tools(self, missing: Iterable[str] = ()) -> None: + """Provide every tool ``make doctor`` looks for, except the ``missing`` ones.""" + if "python" not in missing: + self.interpreter("python3.12") + for name in {"hatch", "func", "azurite"} - set(missing): + self.tool(name, self.venv / "bin" if name == "hatch" else None) + def make(self, *arguments: str, **environment: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [MAKE, "--no-print-directory", f"VENV_DIR={self.venv}", *arguments], @@ -177,6 +186,7 @@ def test_bootstrap_leaves_an_installed_hatch_alone(sandbox: Sandbox) -> None: @pytest.mark.parametrize("kind", ["directory", "non-executable file"]) def test_only_an_executable_regular_file_counts_as_hatch(sandbox: Sandbox, kind: str) -> None: # Given something at the Hatch path that `test -x` or `command -v` alone would accept + sandbox.doctor_tools(missing={"hatch"}) sandbox.pip(sandbox.venv / "bin") hatch = sandbox.venv / "bin" / "hatch" if kind == "directory": @@ -184,12 +194,15 @@ def test_only_an_executable_regular_file_counts_as_hatch(sandbox: Sandbox, kind: else: hatch.write_text("#!/bin/sh\n", encoding="utf-8") # created without the executable bit - # When bootstrap runs - result = sandbox.make("bootstrap") + # When doctor and bootstrap run + doctor = sandbox.make("doctor") + bootstrap = sandbox.make("bootstrap") - # Then it installs Hatch - assert result.returncode == 0, result.stderr - (call,) = sandbox.calls + # Then doctor reports Hatch missing and bootstrap installs it + assert doctor.returncode != 0 + assert f"MISSING {hatch} (required" in doctor.stdout + assert bootstrap.returncode == 0, bootstrap.stderr + (call,) = (call for call in sandbox.calls if call.startswith("pip ")) assert installs_hatch(call) @@ -292,3 +305,65 @@ def test_docs_shows_a_failed_install_and_does_not_build_after_it(sandbox: Sandbo assert result.returncode != 0 assert "ERROR: no matching distribution" in result.stderr assert "mkdocs build --strict" not in sandbox.calls + + +@pytest.mark.parametrize("absent", [(), ("func", "azurite")], ids=["all tools", "optional absent"]) +def test_doctor_passes_when_every_required_tool_exists( + sandbox: Sandbox, absent: tuple[str, ...] +) -> None: + # Given Python 3.11 or newer and Hatch, with or without the Functions host tools + sandbox.doctor_tools(missing=absent) + + # When doctor runs + result = sandbox.make("doctor") + + # Then it passes and lists each absent optional tool + assert result.returncode == 0, result.stdout + assert "MISSING" not in result.stdout + for tool in absent: + assert f"missing {tool} (optional" in result.stdout + + +@pytest.mark.parametrize( + ("tool", "line"), [("python", "MISSING python >= 3.11 (required)"), ("hatch", "MISSING ")] +) +def test_doctor_fails_when_a_required_tool_is_missing( + sandbox: Sandbox, tool: str, line: str +) -> None: + # Given every tool except one required one + sandbox.doctor_tools(missing={tool}) + + # When doctor runs + result = sandbox.make("doctor") + + # Then it fails and names only that tool + assert result.returncode != 0 + assert line in result.stdout + assert result.stdout.count("MISSING ") == 1 + + +def test_doctor_rejects_a_python_older_than_311(sandbox: Sandbox) -> None: + # Given Hatch and a Python 3.10 as the only interpreter + sandbox.doctor_tools(missing={"python"}) + sandbox.interpreter("python3", (3, 10)) + + # When doctor runs + result = sandbox.make("doctor") + + # Then Python counts as missing + assert result.returncode != 0 + assert "MISSING python >= 3.11 (required)" in result.stdout + + +def test_doctor_neither_installs_nor_writes(sandbox: Sandbox) -> None: + # Given a stub pip and no Hatch, where bootstrap would install + sandbox.pip(sandbox.venv / "bin") + before = sorted(sandbox.root.rglob("*")) + + # When doctor runs + result = sandbox.make("doctor") + + # Then it fails, calls nothing and leaves the directory as it was + assert result.returncode != 0 + assert sandbox.calls == [] + assert sorted(sandbox.root.rglob("*")) == before From f04a29f627467e41d0563f7aa2e22063a552cc30 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Sun, 25 Oct 2026 02:28:16 +0900 Subject: [PATCH 4/5] docs(development): document install-ci, doctor, strict docs and the single type check The command tables still described `make check` as lint plus typecheck, `make docs` as a plain MkDocs build and `make doctor` as an environment dump. They now say what each target does after this series, and the development guide gains two short paragraphs: when `make install` acts and what `PYTHON` selects, and what `make doctor` requires, what it treats as optional and the one thing it cannot see (a Python 3.11 that only Hatch manages). AGENTS.md and CONTRIBUTING.md are policy-bearing files and stay untouched; nothing they say becomes wrong. Refs #811 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- docs/contributing.md | 4 ++-- docs/development.md | 20 +++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index b14d8756..e04347c6 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -123,9 +123,9 @@ Describe what changed and why. Reference related issues with `Fixes #N` or `Clos | `make test` | Run pytest | | `make cov` | Run tests with coverage (gated at 95%) | | `make lint-workflows` | Lint release workflows, action pins, Hatch matrix | -| `make check` | Run lint + typecheck | +| `make check` | Run `make lint`, which includes the type check, once | | `make check-all` | Run lint-workflows + check + test + security (full gate) | -| `make docs` | Build the MkDocs site | +| `make docs` | Build the MkDocs site with `mkdocs build --strict`, like the CI docs gate | See [Development Guide](development.md#makefile-targets) for the complete list. diff --git a/docs/development.md b/docs/development.md index 074d2e02..4e16d6f8 100644 --- a/docs/development.md +++ b/docs/development.md @@ -129,6 +129,7 @@ Use these as the **golden commands** for local validation and CI parity. Prefer | Target | Description | |--------|-------------| | `make install` | Create Hatch env and install pre-commit hooks | +| `make install-ci` | Install `.[dev]` into the interpreter you select (`PYTHON=python3.12`), never through Hatch | | `make shell` | Open a shell inside the Hatch env | | `make format` | Format code (ruff format) | | `make format-check` | Check formatting without writing (ruff format --check) | @@ -141,9 +142,9 @@ Use these as the **golden commands** for local validation and CI parity. Prefer | `make e2e-local` | Run e2e tests against local Azurite on `:7071` | | `make e2e-azure` | Run e2e tests against Azure (`E2E_BASE_URL` required) | | `make lint-workflows` | Lint release workflows, action pins, and the Hatch matrix | -| `make check` | Run lint + typecheck | +| `make check` | Run `make lint`, which includes the type check, once | | `make check-all` | Run lint-workflows + check + test + security | -| `make docs` | Build the MkDocs site | +| `make docs` | Build the MkDocs site with `mkdocs build --strict`, like the CI docs gate | | `make docs-serve` | Serve the docs locally with live reload | | `make demo` | Run the Swagger UI and example demos | | `make build` | Build package | @@ -151,7 +152,7 @@ Use these as the **golden commands** for local validation and CI parity. Prefer | `make publish-test` | Publish to TestPyPI | | `make precommit` | Run all pre-commit hooks | | `make precommit-install` | Install pre-commit hooks | -| `make doctor` | Show environment diagnostic info | +| `make doctor` | Check the prerequisites (read-only); fails when a required tool is missing | | `make reset` | Deep clean, then reinstall the env | | `make hatch-clean` | Remove the Hatch environment | | `make clean` | Remove build artifacts | @@ -161,6 +162,19 @@ Use these as the **golden commands** for local validation and CI parity. Prefer Coverage is enforced at **95%** (`fail_under = 95` in `pyproject.toml`); a drop below that fails the build. +`make install` creates `.venv` only while it is missing and installs Hatch only while `.venv/bin/hatch` +is missing or not an executable file; no target upgrades pip. `PYTHON` selects the interpreter that +creates `.venv` (default `python3`, for example `make install PYTHON=python3.12`); Hatch still +provisions its own default environment on Python 3.11. `make install-ci PYTHON=` installs +`.[dev]` into exactly that interpreter, which must be Python 3.11 or newer, and never goes through +Hatch, so a CI matrix cell keeps its own Python. + +`make doctor` checks this repository's prerequisites and installs or writes nothing. A Python 3.11 or +newer on `PATH` and `.venv/bin/hatch` (created by `make install`) are required: each prints `ok` or +`MISSING ...`, and a missing one makes the target fail. `func` and `azurite` are optional, because +only a local Functions host needs them. A Python 3.11 that only Hatch manages is not on `PATH`, so +`doctor` reports it as missing. + > For the full release workflow, see [Release Process](release_process.md). --- From 297a0c41a7347f435babe38761bd2567e9cd7cbd Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Sun, 25 Oct 2026 02:34:58 +0900 Subject: [PATCH 5/5] test(make): compact the Makefile contract tests The contract file had grown to 369 lines over the series. The helpers now share one `make` runner, the stub interpreter only answers the version probes the Makefile runs, related cases are merged and the three-line given/when/then comments are gone in favor of blank-line blocks. The virtual-environment case now reads the expansion of `make -n bootstrap` instead of stubbing `-m venv`, and also asserts that `python3` stays the default interpreter. Every one of 43 one-line mutations of the Makefile (bootstrap, check, check-all, docs, doctor and install-ci) still turns at least one test red. Refs #811 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- tests/test_make_contract.py | 192 ++++++++++-------------------------- 1 file changed, 51 insertions(+), 141 deletions(-) diff --git a/tests/test_make_contract.py b/tests/test_make_contract.py index 7a27ea9d..172cc33a 100644 --- a/tests/test_make_contract.py +++ b/tests/test_make_contract.py @@ -1,9 +1,8 @@ """Contract of the Makefile targets that set up and check a checkout (#811). -``make -n`` prints recipes without running them, so the expansion cases need no network, Hatch or -virtual environment. The behavior cases run the real recipes in a scratch directory whose ``PATH`` -holds only stub programs that log their arguments: nothing is installed, no network is used and no -real tool can run. +``make -n`` prints recipes without running them. The behavior cases run the real recipes in a +scratch directory whose ``PATH`` holds only stub programs that log their arguments, so nothing is +installed, no network is used and no real tool can run. """ from __future__ import annotations @@ -42,16 +41,22 @@ def script_runs(commands: str, name: str) -> int: return sum((s == name) + script_runs(HATCH_SCRIPTS.get(s, ""), name) for s in called) -def plan(*arguments: str) -> str: - """Return the commands ``make -n`` prints for this repository's own Makefile.""" - result = subprocess.run( - [MAKE, "--no-print-directory", "-n", *arguments], - cwd=ROOT, - env={"PATH": os.environ["PATH"]}, +def run_make( + cwd: Path, path: str, *arguments: str, **environment: str +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [MAKE, "--no-print-directory", *arguments], + cwd=cwd, + env={"PATH": path, **environment}, capture_output=True, text=True, check=False, ) + + +def plan(*arguments: str) -> str: + """Return the commands ``make -n`` prints for this repository's own Makefile.""" + result = run_make(ROOT, os.environ["PATH"], "-n", *arguments) assert result.returncode == 0, result.stderr return result.stdout @@ -93,14 +98,12 @@ def pip(self, directory: Path) -> None: self.tool("pip", directory, install) def interpreter(self, name: str, version: tuple[int, int] = (3, 12)) -> Path: - """Add a Python stub that reports ``version`` and whose ``-m venv DIR`` holds a stub pip.""" - self.pip(self.root / "templates") - pip = self.root / "templates" / "pip" - probe = f"import sys; sys.version_info = ({version[0]}, {version[1]}, 0); exec(sys.argv[1])" + """Add a Python stub that reports ``version`` to the probes the Makefile runs.""" + major, minor = version + probe = f"import sys; sys.version_info = ({major}, {minor}, 0); exec(sys.argv[1])" body = ( - f'[ "$1" = --version ] && echo "Python {version[0]}.{version[1]}.0"\n' + f'[ "$1" = --version ] && echo "Python {major}.{minor}.0"\n' f'[ "$1" = -c ] && exec {shlex.quote(sys.executable)} -c "{probe}" "$2"\n' - f'[ "$1 $2" = "-m venv" ] && mkdir -p "$3/bin" && cp {pip} "$3/bin/pip"\n' ) return self.tool(name, body=body) @@ -112,44 +115,29 @@ def doctor_tools(self, missing: Iterable[str] = ()) -> None: self.tool(name, self.venv / "bin" if name == "hatch" else None) def make(self, *arguments: str, **environment: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [MAKE, "--no-print-directory", f"VENV_DIR={self.venv}", *arguments], - cwd=self.root, - env={"PATH": str(self.root / "bin"), **environment}, - capture_output=True, - text=True, - check=False, - ) + arguments = (f"VENV_DIR={self.venv}", *arguments) + return run_make(self.root, str(self.root / "bin"), *arguments, **environment) @pytest.fixture def sandbox(tmp_path: Path) -> Sandbox: shutil.copy(ROOT / "Makefile", tmp_path / "Makefile") (tmp_path / "bin").mkdir() - for program in ("cp", "mkdir"): # the only real programs the stubs run - (tmp_path / "bin" / program).symlink_to(shutil.which(program) or program) + (tmp_path / "bin" / "cp").symlink_to(shutil.which("cp") or "cp") # the one real program return Sandbox(tmp_path) -@pytest.mark.parametrize( - ("target", "style", "typecheck", "test"), [("check", 1, 1, 0), ("check-all", 1, 1, 1)] -) -def test_each_check_runs_once(target: str, style: int, typecheck: int, test: int) -> None: - # Given the commands the target would run - commands = plan(target) +@pytest.mark.parametrize(("target", "tests"), [("check", 0), ("check-all", 1)]) +def test_each_check_runs_once(target: str, tests: int) -> None: + runs = [script_runs(plan(target), name) for name in ("style", "typecheck", "test")] - # Then style, the type check and the tests run the expected number of times - runs = [script_runs(commands, name) for name in ("style", "typecheck", "test")] - assert runs == [style, typecheck, test] + assert runs == [1, 1, tests] def test_check_all_keeps_every_gate_it_had() -> None: - # Given the commands `make check-all` would run - commands = plan("check-all") - - # Then each workflow lint and the Bandit evidence run once gates = ("lint_release_workflows", "lint_workflow_pins", "lint_hatch_matrix", "bandit") - assert [commands.count(gate) for gate in gates] == [1, 1, 1, 1] + + assert [plan("check-all").count(gate) for gate in gates] == [1, 1, 1, 1] def test_makefile_never_upgrades_pip() -> None: @@ -158,34 +146,17 @@ def test_makefile_never_upgrades_pip() -> None: def test_bootstrap_installs_hatch_once_and_only_while_it_is_missing(sandbox: Sandbox) -> None: - # Given a virtual environment that has pip but no Hatch sandbox.pip(sandbox.venv / "bin") - # When bootstrap runs twice for _ in range(2): assert sandbox.make("bootstrap").returncode == 0 - # Then one pip call across both runs installed Hatch, without --upgrade - (call,) = sandbox.calls + (call,) = sandbox.calls # the second run finds the Hatch the first run installed assert installs_hatch(call) -def test_bootstrap_leaves_an_installed_hatch_alone(sandbox: Sandbox) -> None: - # Given a virtual environment that already holds Hatch - sandbox.pip(sandbox.venv / "bin") - sandbox.tool("hatch", sandbox.venv / "bin") - - # When bootstrap runs - result = sandbox.make("bootstrap") - - # Then pip is never called - assert result.returncode == 0, result.stderr - assert sandbox.calls == [] - - @pytest.mark.parametrize("kind", ["directory", "non-executable file"]) def test_only_an_executable_regular_file_counts_as_hatch(sandbox: Sandbox, kind: str) -> None: - # Given something at the Hatch path that `test -x` or `command -v` alone would accept sandbox.doctor_tools(missing={"hatch"}) sandbox.pip(sandbox.venv / "bin") hatch = sandbox.venv / "bin" / "hatch" @@ -194,11 +165,9 @@ def test_only_an_executable_regular_file_counts_as_hatch(sandbox: Sandbox, kind: else: hatch.write_text("#!/bin/sh\n", encoding="utf-8") # created without the executable bit - # When doctor and bootstrap run doctor = sandbox.make("doctor") bootstrap = sandbox.make("bootstrap") - # Then doctor reports Hatch missing and bootstrap installs it assert doctor.returncode != 0 assert f"MISSING {hatch} (required" in doctor.stdout assert bootstrap.returncode == 0, bootstrap.stderr @@ -206,41 +175,20 @@ def test_only_an_executable_regular_file_counts_as_hatch(sandbox: Sandbox, kind: assert installs_hatch(call) -def test_a_fresh_checkout_gets_one_venv_from_the_selected_interpreter(sandbox: Sandbox) -> None: - # Given no virtual environment and a selected interpreter - python = sandbox.interpreter("sentinel") - - # When bootstrap runs - result = sandbox.make("bootstrap", f"PYTHON={python}") - - # Then that interpreter creates the environment and Hatch is the only thing installed - assert result.returncode == 0, result.stderr - venv_call, pip_call = sandbox.calls - assert venv_call == f"sentinel -m venv {sandbox.venv}" - assert installs_hatch(pip_call) - - -def test_install_ci_installs_into_the_selected_interpreter_and_not_through_hatch() -> None: - # Given the commands `make install-ci PYTHON=python3.12` would run - commands = plan("install-ci", "PYTHON=python3.12") - - # Then the matrix interpreter receives the install, so a cell keeps its own Python - assert 'python3.12 -m pip install -e ".[dev]"' in commands - assert "hatch" not in commands +def test_the_selected_interpreter_creates_the_virtual_environment() -> None: + assert "python3 -m venv" in plan("bootstrap") + assert "sentinel -m venv" in plan("bootstrap", "PYTHON=sentinel") @pytest.mark.parametrize(("version", "installs"), [((3, 10), False), ((3, 11), True)]) -def test_install_ci_needs_python_311( +def test_install_ci_installs_into_the_selected_interpreter_from_python_311( sandbox: Sandbox, version: tuple[int, int], installs: bool ) -> None: - # Given an interpreter of the given version - python = sandbox.interpreter("sentinel", version) + python = sandbox.interpreter("sentinel", version) # no Hatch and no pip on PATH - # When install-ci runs result = sandbox.make("install-ci", f"PYTHON={python}") - # Then the version guard ran, and pip runs only for Python 3.11 or newer - assert any(call.startswith("sentinel -c ") for call in sandbox.calls) + assert any(call.startswith("sentinel -c ") for call in sandbox.calls) # the version guard ran assert (result.returncode == 0) is installs assert ("sentinel -m pip install -e .[dev]" in sandbox.calls) is installs @@ -251,119 +199,81 @@ def test_install_ci_needs_python_311( ids=["local", "ci"], ) def test_docs_builds_strictly(sandbox: Sandbox, environment: dict[str, str], build: str) -> None: - # Given Hatch, pip and MkDocs stubs + for name in ("pip", "mkdocs"): + sandbox.tool(name) sandbox.tool("hatch", sandbox.venv / "bin") - sandbox.tool("pip") - sandbox.tool("mkdocs") - # When docs runs locally and under CI result = sandbox.make("docs", **environment) - # Then MkDocs ran in strict mode - assert result.returncode == 0, result.stderr - assert build in sandbox.calls - - -def test_docs_under_ci_installs_the_docs_extra_the_ci_gate_installs(sandbox: Sandbox) -> None: - # Given pip and MkDocs stubs - sandbox.tool("pip") - sandbox.tool("mkdocs") - - # When docs runs under CI - result = sandbox.make("docs", CI="1") - - # Then the dependencies come from pyproject.toml, not from pins kept in the Makefile assert result.returncode == 0, result.stderr - assert any( - call.startswith("pip install") and call.endswith("-e .[docs]") for call in sandbox.calls - ) + assert sandbox.calls[-1] == build + installs = [call for call in sandbox.calls if call.startswith("pip install")] + assert all(call.endswith("-e .[docs]") for call in installs) # the CI gate's extra, not pins + assert bool(installs) is bool(environment) @pytest.mark.parametrize("environment", [{}, {"CI": "1"}], ids=["local", "ci"]) def test_docs_fails_when_the_build_fails(sandbox: Sandbox, environment: dict[str, str]) -> None: - # Given a MkDocs build that fails - sandbox.tool("hatch", sandbox.venv / "bin", "exit 1\n") sandbox.tool("pip") sandbox.tool("mkdocs", body="exit 1\n") + sandbox.tool("hatch", sandbox.venv / "bin", "exit 1\n") - # When docs runs - result = sandbox.make("docs", **environment) - - # Then make fails - assert result.returncode != 0 + assert sandbox.make("docs", **environment).returncode != 0 def test_docs_shows_a_failed_install_and_does_not_build_after_it(sandbox: Sandbox) -> None: - # Given a pip that fails with a message sandbox.tool("pip", body='echo "ERROR: no matching distribution" >&2\nexit 1\n') sandbox.tool("mkdocs") - # When docs runs under CI result = sandbox.make("docs", CI="1") - # Then the error is visible and MkDocs never ran assert result.returncode != 0 assert "ERROR: no matching distribution" in result.stderr assert "mkdocs build --strict" not in sandbox.calls @pytest.mark.parametrize("absent", [(), ("func", "azurite")], ids=["all tools", "optional absent"]) -def test_doctor_passes_when_every_required_tool_exists( +def test_doctor_passes_when_the_required_tools_exist( sandbox: Sandbox, absent: tuple[str, ...] ) -> None: - # Given Python 3.11 or newer and Hatch, with or without the Functions host tools sandbox.doctor_tools(missing=absent) - # When doctor runs result = sandbox.make("doctor") - # Then it passes and lists each absent optional tool assert result.returncode == 0, result.stdout assert "MISSING" not in result.stdout - for tool in absent: - assert f"missing {tool} (optional" in result.stdout + assert all(f"missing {tool} (optional" in result.stdout for tool in absent) -@pytest.mark.parametrize( - ("tool", "line"), [("python", "MISSING python >= 3.11 (required)"), ("hatch", "MISSING ")] -) -def test_doctor_fails_when_a_required_tool_is_missing( - sandbox: Sandbox, tool: str, line: str +@pytest.mark.parametrize("tool", ["python", "hatch"]) +def test_doctor_fails_naming_the_one_required_tool_that_is_missing( + sandbox: Sandbox, tool: str ) -> None: - # Given every tool except one required one sandbox.doctor_tools(missing={tool}) - # When doctor runs result = sandbox.make("doctor") - # Then it fails and names only that tool assert result.returncode != 0 - assert line in result.stdout assert result.stdout.count("MISSING ") == 1 + assert ("python >= 3.11" if tool == "python" else str(sandbox.venv)) in result.stdout def test_doctor_rejects_a_python_older_than_311(sandbox: Sandbox) -> None: - # Given Hatch and a Python 3.10 as the only interpreter sandbox.doctor_tools(missing={"python"}) sandbox.interpreter("python3", (3, 10)) - # When doctor runs result = sandbox.make("doctor") - # Then Python counts as missing assert result.returncode != 0 - assert "MISSING python >= 3.11 (required)" in result.stdout + assert "MISSING python >= 3.11" in result.stdout def test_doctor_neither_installs_nor_writes(sandbox: Sandbox) -> None: - # Given a stub pip and no Hatch, where bootstrap would install - sandbox.pip(sandbox.venv / "bin") + sandbox.pip(sandbox.venv / "bin") # a stub pip and no Hatch: bootstrap would install here before = sorted(sandbox.root.rglob("*")) - # When doctor runs result = sandbox.make("doctor") - # Then it fails, calls nothing and leaves the directory as it was assert result.returncode != 0 assert sandbox.calls == [] assert sorted(sandbox.root.rglob("*")) == before