diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index 53fbe38..b827fb8 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -93,28 +93,17 @@ jobs:
cache-dependency-path: pyproject.toml
- name: Install package with dev extras
- run: |
- python -m pip install --upgrade pip
- pip install -e ".[dev]"
-
- - name: Check endpoint schema hash pin
- run: pytest --no-cov tests/test_endpoint_schema.py::TestSchemaHashPin
-
- - name: Lint workflows (pin hygiene + release-gate canonical SHAs)
- run: |
- python tools/lint_release_workflows.py
- python tools/lint_workflow_pins.py
-
- - name: Hatch/matrix hygiene lint
- run: python tools/lint_hatch_matrix.py
+ run: make install-ci
- - name: Ruff style check
- run: |
- ruff check src tests
- ruff format --check src tests
+ - name: Complete non-mutating quality and contract gate
+ run: make check
- - name: Type check (mypy, strict)
- run: mypy src tests
+ - name: Upload quality evidence
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: quality-evidence
+ path: test-results/**
test:
name: Test (Py ${{ matrix.python-version }})
@@ -157,9 +146,7 @@ jobs:
PY
- name: Install package with dev extras (on the matrix interpreter)
- run: |
- python -m pip install --upgrade pip
- pip install -e ".[dev]"
+ run: make install-ci
- name: Run tests on the matrix interpreter
# Raw pytest on the interpreter provisioned above (NOT through the
@@ -171,26 +158,36 @@ jobs:
IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' }}
run: |
if [ "$IS_COVERAGE_CELL" = "true" ]; then
- pytest tests/
+ python -m tools.verification test
else
- pytest tests/ --no-cov
+ python -m tools.verification test --no-cov
fi
- name: Verify coverage output
if: matrix.python-version == '3.11'
- run: ls -lh coverage.xml
+ run: ls -lh test-results/test/coverage.xml
- name: Upload coverage to Codecov
if: matrix.python-version == '3.11' && github.actor != 'dependabot[bot]'
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
token: ${{ secrets.CODECOV_TOKEN }}
- files: ./coverage.xml
+ files: ./test-results/test/coverage.xml
verbose: true
fail_ci_if_error: false
env:
CODECOV_RETRIES: 3
+ - name: Required worker signature and registration evidence
+ run: python -m tools.verification worker-compat
+
+ - name: Upload test and worker evidence
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: test-evidence-py${{ matrix.python-version }}
+ path: test-results/**
+
docs-check:
name: Documentation checks
needs: changes
@@ -211,7 +208,14 @@ jobs:
run: pip install -e ".[docs]" pytest pytest-cov PyYAML
- name: Build docs (strict)
- run: mkdocs build --strict
+ run: make docs
+
+ - name: Upload docs evidence
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: docs-evidence
+ path: test-results/**
- name: Check README examples and screenshot manifest
run: |
@@ -297,7 +301,14 @@ jobs:
PY
- name: Run full suite against minimum runtime dependencies
- run: pytest tests/ --no-cov
+ run: python -m tools.verification test --no-cov
+
+ - name: Upload minimum-dependency evidence
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: minimum-dependencies-evidence
+ path: test-results/**
artifact-build:
name: Build compatibility artifacts
@@ -444,26 +455,17 @@ jobs:
npm install --global azurite
- name: Install smoke app dependencies
- working-directory: tests/host_smoke
- run: pip install -r requirements.txt
+ run: make install-ci
- - name: Start Azurite and Functions host
- run: |
- azurite --silent > /tmp/azurite.log 2>&1 &
- cd tests/host_smoke
- func start --python --port 7071 > /tmp/functions-host.log 2>&1 &
+ - name: Managed real Functions host and worker signature smoke
+ run: python -m tools.verification integration
- - name: Call validated handler
- run: |
- for attempt in {1..30}; do
- if response="$(curl --fail --silent http://127.0.0.1:7071/api/smoke)"; then
- python -c 'import json, sys; assert json.loads(sys.argv[1]) == {"function_name": "smoke"}' "$response"
- exit 0
- fi
- sleep 2
- done
- cat /tmp/functions-host.log
- exit 1
+ - name: Upload host and worker evidence
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: host-evidence-py${{ matrix.python-version }}
+ path: test-results/**
# azure-functions 2.x compatibility proof lane (issue #351), modeled on
# langgraph d433b63. This is a SINGLE representative 2.x lane, not a matrix:
@@ -528,6 +530,11 @@ jobs:
# pythonpath so the installed WHEEL is exercised, not the source tree.
env:
AZFUNC_2X_SPIKE: "1"
- run: |
- pytest -m compat2x -o addopts= -o pythonpath= \
- tests/test_worker_compat_2x_spike.py -v
+ run: python -m tools.verification sdk-compat -v
+
+ - name: Upload SDK compatibility spike evidence (not Azure certification)
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: sdk-2x-evidence
+ path: test-results/**
diff --git a/.github/workflows/e2e-azure.yml b/.github/workflows/e2e-azure.yml
index 21e12b8..9c671ac 100644
--- a/.github/workflows/e2e-azure.yml
+++ b/.github/workflows/e2e-azure.yml
@@ -218,7 +218,7 @@ jobs:
echo "=== Live logstream (20s) ==="
timeout 20 az webapp log tail --name "$APP" --resource-group "$RG" 2>&1 | head -200 || true
- name: Run e2e tests
- run: pytest -v -m e2e tests/e2e/ --no-cov --html=e2e-report/report.html --self-contained-html
+ run: python -m tools.verification azure-e2e -v --html=test-results/azure-e2e/report.html --self-contained-html
env:
E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net
@@ -287,7 +287,7 @@ jobs:
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
- path: e2e-report/
+ path: test-results/**
retention-days: 30
cleanup:
diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml
index f513d85..eabfa6b 100644
--- a/.github/workflows/publish-pypi.yml
+++ b/.github/workflows/publish-pypi.yml
@@ -108,11 +108,18 @@ jobs:
with:
python-version: "3.11"
- - name: Install Hatch
- run: pip install hatch
+ - name: Install test dependencies on the selected interpreter
+ run: make install-ci
- name: Run test suite
- run: hatch run test
+ run: python -m tools.verification test
+
+ - name: Upload release library evidence
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: release-library-evidence
+ path: test-results/**
cookbook-smoke:
name: Cookbook dogfood smoke (candidate wheel)
diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
index 6f76dbd..be26acc 100644
--- a/.github/workflows/security.yml
+++ b/.github/workflows/security.yml
@@ -23,10 +23,8 @@ jobs:
with:
python-version: "3.11"
- - name: Install Hatch and dependencies via Makefile
- run: |
- pip install hatch
- make install
+ - name: Install dependencies on the selected interpreter
+ run: make install-ci
- name: Run Bandit
run: make security
diff --git a/.github/workflows/worker-nightly.yml b/.github/workflows/worker-nightly.yml
index 67cb6b5..44411fb 100644
--- a/.github/workflows/worker-nightly.yml
+++ b/.github/workflows/worker-nightly.yml
@@ -61,9 +61,14 @@ jobs:
python -c "import azure.functions as f; print('azure-functions', f.__version__)"
- name: Run WorkerCompat suite against pre-release worker library
- run: |
- python -m pytest tests/test_decorator.py tests/test_integration.py \
- -v --no-header -p no:cacheprovider --no-cov
+ run: python -m tools.verification worker-compat -v --no-header -p no:cacheprovider
+
+ - name: Upload pre-release worker evidence (not Azure certification)
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: worker-prerelease-py${{ matrix.python-version }}
+ path: test-results/**
- name: Explain a WorkerCompat break
if: failure()
diff --git a/.gitignore b/.gitignore
index 2ff99df..7a001a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,7 @@ env/
.coverage
coverage.xml
htmlcov/
+test-results/
# Build artifacts
build/
diff --git a/AGENTS.md b/AGENTS.md
index 83144be..25fe886 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -18,7 +18,7 @@
### Test Coverage
- Maintain test coverage at **95% or above** for committed changes and PRs.
-- Run `hatch run pytest --cov --cov-report=term-missing -q` to verify before submitting changes.
+- Run `make test` to verify the offline suite and coverage before submitting changes.
- Any PR that drops coverage below 95% must include additional tests to compensate.
- Runtime code must remain compatible with Python 3.11-3.14 (`requires-python = ">=3.11,<3.15"`).
- Public APIs must be fully typed.
@@ -106,10 +106,11 @@ This repository is **issue-based, not milestone-based**. Track and group work us
- Before adding any organizational structure, check whether the repository already has an established convention. A category being empty or unused (zero milestones, no `epic:*` labels) is evidence to follow the existing pattern, not to introduce a new one.
## Validation
-- `make test`
-- `make lint`
-- `make typecheck`
-- `make build`
+- `make doctor` (read-only prerequisites, actual interpreter)
+- `make check` (complete non-mutating quality and schema/metadata contracts)
+- `make test` (offline, coverage >=95%)
+- `make integration` (worker signatures and real host/private Azurite)
+- `make verify` (`make check-all` is an exact alias)
## Release Process
@@ -186,22 +187,31 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor
| Purpose | Command |
| --- | --- |
| Environment setup | `make install` |
+| CI-selected interpreter setup | `make install-ci` |
+| Read-only prerequisite diagnostics | `make doctor` |
| Format code | `make format` |
| Check formatting (`src`, `tests`) | `make format-check` |
-| Lint | `make lint` |
-| Type check | `make typecheck` |
-| Tests | `make test` |
-| Coverage | `make cov` |
-| Full validation | `make check-all` |
-| Docs build | `make docs` |
-| Package build | `make build` |
+| Complete non-mutating quality/contracts | `make check` |
+| Offline tests, coverage >=95% | `make test` |
+| Managed worker/host/emulator lanes | `make integration` |
+| Full local validation | `make verify` (`make check-all` exact alias) |
+| Configured real Azure certification | `make e2e-azure` |
+| Render saved evidence only | `make report` |
+| Generated artifacts only | `make clean` |
+| Partial/focused lint/types/coverage | `make lint`, `make typecheck`, `make cov` |
+| Partial/focused docs/package/wheel | `make docs`, `make build`, `make wheel-smoke` |
+
+See CONTRIBUTING.md for evidence/status semantics. Required worker and host lanes
+have separate summaries under gitignored `test-results/`. SDK spikes are not Azure
+certification. Verification must never rewrite schema/hash pins. `clean-all` is
+maintenance-only deep cleanup, including bytecode and the project-local `.venv`.
## Testing Rules
- Public APIs require tests.
- Bug fixes require regression tests.
- Representative and complex examples must remain smoke-tested.
-- `make check-all` is the minimum merge gate.
+- `make verify` is the minimum local merge gate; `make check-all` aliases it exactly.
## Commit Rules
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1c1aef2..eb0af7f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -6,11 +6,12 @@ We welcome contributions to the `azure-functions-validation` project.
The package supports Python 3.11, 3.12, 3.13, and 3.14. Local development
requires Python 3.11 because the default Hatch environment is pinned to that
-interpreter. No repository-specific services or infrastructure are required
-for the standard checks.
+interpreter. Offline tests and quality checks need no services. Full local
+verification additionally requires Functions Core Tools (`func`) and Azurite.
-Clone the repository, enter it, and let the Makefile create a local virtual
-environment and install Hatch:
+Install Hatch once in an isolated tool environment, then clone the repository
+and create the development environment. Routine checks never upgrade pip or
+reinstall Hatch, and never change the global interpreter:
```bash
git clone https://github.com/yeongseon/azure-functions-validation-python.git
@@ -22,12 +23,13 @@ Run the standard verification commands before opening a pull request:
```bash
make test
-make lint
-make typecheck
-make build
+make check
+make verify
```
-`make check-all` runs the complete local quality gate used for contributions.
+`make check-all` is an exact backward-compatible alias of `make verify`.
+CI uses `make install-ci PYTHON=python` on the explicitly selected matrix
+interpreter, not Hatch's pinned Python 3.11 environment.
## Branch Strategy
@@ -61,17 +63,66 @@ Recommended branch prefixes:
## Project Commands
-```bash
-make format # Format code with ruff
-make lint # Lint with ruff
-make typecheck # Type check with mypy
-make test # Run tests
-make cov # Run tests with coverage
-make check-all # Run the full local gate
+| Command | Contract |
+| --- | --- |
+| `make install` | Create isolated Hatch dev/docs environment and install local hooks |
+| `make install-ci` | Install extras into CI's selected `PYTHON`, preserving the matrix cell |
+| `make doctor` | Read-only prerequisite check; fail for missing required tools |
+| `make format` / `make format-check` | Mutating format action / non-mutating check |
+| `make check` | Ruff lint/format, strict mypy once, Bandit, workflow policy and pinned metadata/schema contracts |
+| `make test` | Offline unit/mock suite only, existing 95% coverage floor |
+| `make integration` | Required worker signatures/registration plus managed real host and private Azurite |
+| `make verify` | Always check + test + integration + wheel/schema install smoke + strict docs |
+| `make e2e-azure` | Separate real Azure run against explicit `E2E_BASE_URL`; never provisions resources |
+| `make report` | Render saved results without rerunning tests |
+| `make clean` | Remove generated artifacts only; preserve environments/shared emulator storage |
+
+`make check-all` means exactly `make verify`. `make clean-all` retains its historical
+deep-clean meaning: artifacts, bytecode and the project-local `.venv`.
+Focused/partial targets remain: `style`, `lint`, `typecheck`, `security`, `cov`,
+`lint-workflows`, `check-schema-hash`, `contracts`, `worker-compat`, `sdk-compat`,
+`build`, `wheel-smoke`, `docs`, screenshot checks and maintenance commands.
+`sdk-compat` requires `AZFUNC_2X_SPIKE=1` and tests an installed wheel without source
+path injection; it is a compatibility spike, never general Azure certification.
+`e2e-local` targets an already-running local app and is not Azure certification.
+
+### Verification evidence
+
+Generated evidence is gitignored:
+
+```text
+test-results/
+ test/{junit.xml,coverage.xml,summary.json,environment.json,logs/}
+ worker-compat/{junit.xml,summary.json,environment.json,logs/}
+ host-smoke/{junit/junit.xml,summary.json,environment.json,logs/}
+ integration/{summary.json,environment.json,lanes.json}
+ verify/{summary.json,environment.json,lanes.json,logs/}
+ azure-e2e/{junit.xml,summary.json,environment.json,logs/}
```
-Before opening a PR, run `make format-check` to check `src` and `tests` without
-changing files. Run `make format` to fix those trees. CI also checks every
+Each summary has a schema version, lane/command, selected/passed/failed/skipped
+counts, exit status and supporting paths. Actual Python/runtime/SDK versions are
+recorded; credentials, connection strings, authorization headers, SAS and tokens
+are redacted. Console and Actions Job Summary use the same saved fields; CI uploads
+`test-results/**` with `if: always()`.
+
+- **PASS** requires actual execution and successful required checks.
+- **FAIL** means a process/check/test failed or timed out.
+- **INCOMPLETE** means missing prerequisites, missing/invalid JUnit, zero tests,
+ all skipped, or a skipped required live/worker test. These exit non-zero.
+- **NOT_APPLICABLE** is reserved for explicitly unsupported lanes, never missing tools.
+
+Host smoke uses temporary storage, unique ephemeral ports and bounded readiness;
+it stops only processes it owns, including on interruption. Core Tools may need
+its extension bundle cached. Worker signature evidence and host evidence stay
+separate. Local verification never implies real Azure certification. Azure release
+certification keeps the existing exact-ref/source-version record; this repository's
+live endpoint does not currently expose a deployed package-version assertion.
+Schema/hash checks never rewrite pinned metadata. Wheel smoke verifies packaged
+schema bytes and installed imports outside the checkout, separately from source tests.
+
+Before opening a PR, run `make format-check` to check `src`, `tests` and verification
+helpers without changing files. Run `make format` to fix those trees. CI also checks every
changed Python file, including files outside `src` and `tests`.
## GitHub Actions Pinning
diff --git a/Makefile b/Makefile
index c781c5f..4d22010 100644
--- a/Makefile
+++ b/Makefile
@@ -1,177 +1,111 @@
VENV_DIR := .venv
-PYTHON := $(VENV_DIR)/bin/python
-PIP := $(VENV_DIR)/bin/pip
-HATCH := $(VENV_DIR)/bin/hatch
-
-.PHONY: bootstrap
-bootstrap:
- @if [ ! -d "$(VENV_DIR)" ]; then \
- echo "Creating virtual environment..."; \
- python3 -m venv $(VENV_DIR); \
- fi
- @echo "Ensuring Hatch is installed in virtual environment..."
- @$(PIP) install --upgrade pip > /dev/null
- @$(PIP) install hatch > /dev/null
- @echo "Hatch installed at $(HATCH)"
-
-.PHONY: ensure-hatch
-ensure-hatch: bootstrap
+PYTHON ?= python
+HATCH ?= hatch
+ifeq ($(CI),)
+RUN := $(HATCH) run python
+else
+RUN := $(PYTHON)
+endif
+MAINTENANCE_PYTHON ?= $(shell for p in "$$VIRTUAL_ENV/bin/python" "$(VENV_DIR)/bin/python" python3.11 python3 python; do if "$$p" -c 'import sys; sys.exit(not (3,11) <= sys.version_info[:2] < (3,15))' 2>/dev/null; then command -v "$$p"; break; fi; done)
+
+.PHONY: bootstrap ensure-hatch
+bootstrap ensure-hatch:
+ @command -v $(HATCH) >/dev/null || { echo "Install Hatch in an isolated tool environment first"; exit 1; }
.PHONY: install
install: ensure-hatch
@$(HATCH) env create
- @if [ -n "$$CI" ]; then \
- echo "CI detected: skipping pre-commit hook installation"; \
- else \
- $(MAKE) precommit-install; \
- fi
+ @if [ -z "$$CI" ]; then $(MAKE) precommit-install; fi
+
+.PHONY: install-ci
+install-ci:
+ @$(PYTHON) -c 'import sys; assert (3, 11) <= sys.version_info[:2] < (3, 15), sys.version'
+ @$(PYTHON) -m pip install -e ".[dev,docs]"
-.PHONY: shell
+.PHONY: shell reset hatch-clean
shell: ensure-hatch
@$(HATCH) shell
-
-.PHONY: reset
reset: clean-all install
- @echo "Project reset complete."
-
-.PHONY: hatch-clean
hatch-clean: ensure-hatch
- @$(HATCH) env remove || echo "No hatch environment to remove"
-
-.PHONY: format
-format: ensure-hatch
- @$(HATCH) run format
-
-.PHONY: format-check
-format-check: ensure-hatch
- @$(HATCH) run format-check
-
-.PHONY: style
-style: ensure-hatch
- @$(HATCH) run style
-
-.PHONY: typecheck
-typecheck: ensure-hatch
- @$(HATCH) run typecheck
-
-.PHONY: lint
-lint: ensure-hatch
- @$(HATCH) run lint
-
-.PHONY: security
-security: ensure-hatch
- @$(HATCH) run security
-
-.PHONY: check-schema-hash
-check-schema-hash: ensure-hatch
- @$(HATCH) run python -c "import hashlib, pathlib, sys; d = pathlib.Path('src/azure_functions_validation/schemas'); expected = (d / 'endpoint.schema.sha256').read_text().split()[0]; actual = hashlib.sha256((d / 'endpoint.schema.json').read_bytes()).hexdigest(); print('endpoint.schema.json digest matches pin.') if expected == actual else sys.exit('endpoint.schema.json digest drift:\n pinned: ' + expected + '\n actual: ' + actual + '\nUpdate endpoint.schema.sha256 if this change is intentional.')"
-
-.PHONY: lint-workflows
-lint-workflows: ensure-hatch
- @$(HATCH) run python tools/lint_release_workflows.py
- @$(HATCH) run python tools/lint_workflow_pins.py
- @$(HATCH) run python tools/lint_hatch_matrix.py
-
-.PHONY: check
-check: ensure-hatch
- @$(MAKE) lint
- @$(MAKE) typecheck
- @echo "Lint and type check passed."
-
-.PHONY: check-all
-check-all: ensure-hatch
- @$(MAKE) check-schema-hash
- @$(MAKE) lint-workflows
- @$(MAKE) check
- @$(MAKE) test
- @$(MAKE) security
- @echo "All checks passed including tests and security scan."
-
-.PHONY: screenshots-check
-screenshots-check: ensure-hatch
- @$(HATCH) run python scripts/check_screenshots.py --strict
-
-.PHONY: screenshots-update
-screenshots-update: ensure-hatch
- @$(HATCH) run python scripts/check_screenshots.py --update
-
-.PHONY: precommit
+ @$(HATCH) env remove
+
+.PHONY: format format-check style typecheck lint security
+format:
+ @$(RUN) -m ruff format src tests tools/verification*.py
+format-check:
+ @$(RUN) -m ruff format --check src tests tools/verification*.py
+style: format-check
+ @$(RUN) -m ruff check src tests tools/verification*.py
+typecheck:
+ @$(RUN) -m mypy src tests tools/verification*.py
+lint: style typecheck
+security:
+ @$(RUN) -m bandit -r src
+
+.PHONY: check-schema-hash contracts lint-workflows check check-all
+check-schema-hash:
+ @$(RUN) -m tools.verification_contracts schema-hash
+contracts:
+ @$(RUN) -m tools.verification contracts
+lint-workflows:
+ @$(RUN) tools/lint_release_workflows.py
+ @$(RUN) tools/lint_workflow_pins.py
+ @$(RUN) tools/lint_hatch_matrix.py
+.PHONY: check-steps
+check:
+ @$(RUN) -m tools.verification check
+check-steps: style typecheck security lint-workflows check-schema-hash contracts
+check-all: verify
+
+.PHONY: test integration worker-compat sdk-compat verify report wheel-smoke
+test integration worker-compat sdk-compat verify:
+ @$(RUN) -m tools.verification $@
+report:
+ @$(MAINTENANCE_PYTHON) -m tools.verification report
+wheel-smoke:
+ @$(RUN) -m tools.verification focused wheel-smoke -m tools.verification_wheel
+
+.PHONY: cov e2e-local e2e-azure build docs docs-serve
+cov:
+ @$(RUN) -m tools.verification test --cov-report=html
+e2e-local:
+ @E2E_BASE_URL=$${E2E_BASE_URL:-http://localhost:7071} $(RUN) -m tools.verification local-e2e
+e2e-azure:
+ @$(RUN) -m tools.verification azure-e2e
+build:
+ @$(RUN) -m tools.verification focused build -m build --no-isolation
+docs:
+ @$(RUN) -m tools.verification focused docs -m mkdocs build --strict
+docs-serve:
+ @$(RUN) -m mkdocs serve
+
+.PHONY: screenshots-check screenshots-update precommit precommit-install publish-test version
+screenshots-check:
+ @$(RUN) scripts/check_screenshots.py --strict
+screenshots-update:
+ @$(RUN) scripts/check_screenshots.py --update
precommit: ensure-hatch
@$(HATCH) run precommit
-
-.PHONY: precommit-install
precommit-install: ensure-hatch
@$(HATCH) run precommit-install
-
-.PHONY: test
-test: ensure-hatch
- @echo "Running tests..."
- @$(HATCH) run test
-
-.PHONY: cov
-cov: ensure-hatch
- @$(HATCH) run cov
- @echo "Open htmlcov/index.html in your browser to view the coverage report."
- @echo "coverage.xml generated for Codecov upload."
-
-.PHONY: e2e-local
-e2e-local: ensure-hatch
- @echo "Running e2e tests against local Azurite (E2E_BASE_URL=http://localhost:7071)..."
- @E2E_BASE_URL=http://localhost:7071 $(HATCH) run e2e-azure
-
-.PHONY: e2e-azure
-e2e-azure: ensure-hatch
- @echo "Running e2e tests against Azure (E2E_BASE_URL must be set)..."
- @$(HATCH) run e2e-azure
-
-.PHONY: build
-build: ensure-hatch
- @$(HATCH) build
-
-.PHONY: publish-test
publish-test: ensure-hatch
@$(HATCH) publish --repo test
-
-.PHONY: version
version: ensure-hatch
- @echo "Current version:"
@$(HATCH) version
-.PHONY: docs
-docs:
- @if [ -n "$$CI" ]; then \
- echo "CI detected: running MkDocs directly"; \
- python -m pip install --upgrade pip > /dev/null 2>&1 || true; \
- pip install "mkdocs<2.0" "mkdocs-material<10.0" "mkdocstrings[python]<1.0" > /dev/null 2>&1; \
- mkdocs build; \
- else \
- $(MAKE) ensure-hatch > /dev/null; \
- $(HATCH) run mkdocs build; \
- fi
-
-.PHONY: docs-serve
-docs-serve: ensure-hatch
- @$(HATCH) run docs
-
-.PHONY: doctor
+.PHONY: doctor clean clean-all help
doctor:
- @echo "Python version:" && $(PYTHON) --version
- @echo "Installed packages:" && $(HATCH) env run pip list || echo "No hatch environment found"
- @echo "Azure Function Core Tools version:" && func --version || echo "func not found"
- @echo "Pre-commit hook installed:"
- @if [ -f .git/hooks/pre-commit ]; then echo yes; else echo no; fi
-
-.PHONY: clean
+ @$(MAINTENANCE_PYTHON) -m tools.verification doctor
clean:
- @rm -rf *.egg-info dist build __pycache__ .pytest_cache
-
-.PHONY: clean-all
+ @$(MAINTENANCE_PYTHON) -m tools.verification clean
+# Historical deep cleanup is intentionally separate from artifact-only clean.
clean-all: clean
- @find . -type d -name "__pycache__" -exec rm -rf {} +
- @find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -delete
- @rm -rf .mypy_cache .ruff_cache .pytest_cache .coverage coverage.xml htmlcov .DS_Store $(VENV_DIR) site
-
-.PHONY: help
+ @find . -type d -name "__pycache__" -not -path "./.git/*" -exec rm -rf {} +
+ @find . -type f \( -name "*.pyc" -o -name "*.pyo" \) -not -path "./.git/*" -delete
+ @rm -rf .DS_Store $(VENV_DIR)
help:
- @echo "Available commands:" && \
- grep -E '^\.PHONY: ' Makefile | cut -d ':' -f2 | xargs -n1 echo " - make"
+ @echo "Canonical: install install-ci doctor format format-check check test integration verify e2e-azure report clean"
+ @echo "check-all: exact backward-compatible alias of verify (requires func and Azurite)"
+ @echo "Partial/focused: style lint typecheck security check-schema-hash contracts lint-workflows cov worker-compat sdk-compat build wheel-smoke docs docs-serve screenshots-check screenshots-update"
+ @echo "Local only, not Azure certification: e2e-local; SDK spike only: sdk-compat"
+ @echo "Maintenance: bootstrap ensure-hatch shell reset hatch-clean clean-all precommit precommit-install publish-test version"
diff --git a/docs/METADATA_SPEC.md b/docs/METADATA_SPEC.md
index baac284..f891c96 100644
--- a/docs/METADATA_SPEC.md
+++ b/docs/METADATA_SPEC.md
@@ -50,7 +50,7 @@ its own local conformance tests against the versioned dict convention.
own their own local conformance tests against the same versioned dict
convention (same philosophy as the vendored `_metadata_helpers.py`).
- Local drift is caught mechanically: `endpoint.schema.sha256` pins the file
- digest and `make check-schema-hash` (wired into `make check-all`) fails on
+digest and `make check-schema-hash` (wired into `make check` and therefore `make verify` / `make check-all`) fails on
any change. When you intentionally change the schema, update the pin and
bump the `version` field if the change is breaking.
diff --git a/docs/agent-playbook.md b/docs/agent-playbook.md
index 2520f50..6b33b9b 100644
--- a/docs/agent-playbook.md
+++ b/docs/agent-playbook.md
@@ -32,8 +32,13 @@ advisory, not a required check — it does not change the zero-native-approval m
required CI contexts in `AGENTS.md`.
## Validation
-- `make test`
-- `make lint`
-- `make typecheck`
-- `make security`
-- `make build`
+- `make doctor`: read-only prerequisites and actual Python interpreter.
+- `make check`: complete non-mutating quality/schema/metadata contracts.
+- `make test`: offline unit/mock suite with 95% coverage.
+- `make integration`: separate worker signatures and managed host/Azurite evidence.
+- `make verify`: full local gate; `make check-all` is an exact alias.
+- `make e2e-azure`: separately configured Azure certification, never provisioning.
+- `make report`: render saved `test-results/` without rerunning tests.
+- `make clean`: generated artifacts only; `clean-all` also removes local `.venv`.
+
+Other useful targets are focused/partial. See [Development Guide](development.md#makefile-targets).
diff --git a/docs/contributing.md b/docs/contributing.md
index 8f2fd46..276d8ac 100644
--- a/docs/contributing.md
+++ b/docs/contributing.md
@@ -61,7 +61,13 @@ We maintain strict quality standards to ensure the reliability of the validation
Tool versions are pinned in the `dev` dependency group of `pyproject.toml` — that
file is the single source of truth, so this guide does not restate the numbers.
-Run `make check-all` to execute all these tools locally before pushing your changes.
+Run `make verify` (exact alias: `make check-all`) before pushing. It always includes
+quality/contracts, offline tests, worker signature/registration, managed real host
+and private Azurite, wheel/schema install smoke and strict docs. Core Tools and
+Azurite are required for the full gate; `make check` and `make test` stay offline.
+Other targets such as `lint`, `typecheck`, `security`, `cov`, `build` and `docs` are
+focused/partial. `make report` renders saved redacted `test-results/`; see
+[Development Guide](development.md#makefile-targets) for commands and status semantics.
## Testing Requirements
diff --git a/docs/development.md b/docs/development.md
index f0a6e9e..f98bfb4 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -118,22 +118,53 @@ Use these as the **golden commands** for local validation and CI parity. Prefer
| Target | Description |
|--------|-------------|
| `make install` | Create Hatch env and install pre-commit hooks |
+| `make install-ci` | Install extras into CI's explicitly selected interpreter |
| `make format` | Format code (ruff) |
| `make lint` | Run linter (ruff + mypy) |
| `make typecheck` | Run mypy type checking |
| `make security` | Run Bandit security scan |
-| `make test` | Run pytest |
+| `make test` | Offline unit/mock pytest, coverage >=95%, saved JUnit/coverage/summary |
| `make cov` | Run tests with coverage |
-| `make check` | Run lint + typecheck |
-| `make check-all` | Run lint + typecheck + test |
+| `make check` | Non-mutating Ruff lint/format, mypy once, Bandit, workflow and pinned schema/metadata contracts |
+| `make integration` | Required worker signatures plus managed real host/private Azurite |
+| `make verify` | Check + test + integration + wheel/schema install smoke + strict docs |
+| `make check-all` | Exact backward-compatible alias of verify |
+| `make e2e-azure` | Real Azure suite against explicit E2E_BASE_URL, no provisioning |
+| `make report` | Render saved evidence, never rerun tests |
| `make build` | Build package |
| `make publish-test` | Publish to TestPyPI |
| `make precommit` | Run all pre-commit hooks |
| `make precommit-install` | Install pre-commit hooks |
-| `make doctor` | Show environment diagnostic info |
-| `make clean` | Remove build artifacts |
+| `make doctor` | Read-only prerequisites; missing required tools fail non-zero |
+| `make clean` | Generated artifacts only, preserve environments/shared storage |
| `make clean-all` | Deep clean (caches, coverage, venv) |
+Install Hatch once in an isolated tool environment. Routine commands never upgrade
+pip or reinstall Hatch. Local commands use Hatch's Python 3.11; CI runs on its
+selected Python directly, preserving Python/SDK variants. Full local verification
+requires `func` and `azurite` on PATH; offline tests require neither.
+
+The canonical contract is `install`, `install-ci`, `doctor`, `format`, `format-check`,
+`check`, `test`, `integration`, `verify`, `e2e-azure`, `report`, `clean`. Other targets
+are focused/partial or maintenance. `sdk-compat` requires `AZFUNC_2X_SPIKE=1` and an
+installed wheel; neither it nor `e2e-local` is Azure certification.
+
+Evidence lives in gitignored `test-results/`: `test/` has JUnit and coverage XML;
+`worker-compat/` and `host-smoke/` each have their own summary, JUnit and logs;
+`integration/` aggregates those lanes; `verify/` adds lane results/environment;
+`azure-e2e/` has separate live evidence. Each summary includes schema version,
+command, selected/passed/failed/skipped counts, status and supporting paths.
+Actual interpreter/SDK versions are captured and secrets are redacted.
+Console and Actions Job Summary render the same fields; uploads run even on failure.
+
+PASS requires execution; failed checks/timeouts are FAIL; zero/all-skipped/missing
+prerequisites are INCOMPLETE and non-zero. NOT_APPLICABLE is only for explicitly
+unsupported lanes. Required live/worker skips cannot pass. Integration owns temporary
+storage and processes on ephemeral ports and never uses shared emulator volumes.
+Schema/metadata checks never rewrite pins. Wheel imports/schema resources are checked
+outside the source tree. Azure certification retains its exact-ref/version record,
+not an inferred result from local success. See the root CONTRIBUTING.md for details.
+
---
## Tips
diff --git a/docs/testing.md b/docs/testing.md
index 3d6bf92..7caa3d7 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -79,7 +79,7 @@ The project uses `pytest` as its primary testing framework. You can run the test
The simplest way to run tests is using the provided `Makefile` targets:
```bash
-# Run all tests
+# Run offline unit/mock tests (no Azure, host or emulator)
make test
# Run tests and generate a coverage report
@@ -88,6 +88,15 @@ make cov
The `make cov` command generates a terminal report, an XML report for CI integration, and a detailed HTML report in the `htmlcov/` directory.
+`make integration` separately runs required worker signature/registration checks
+and the real validation handler on a managed Functions host with private Azurite.
+Each lane has its own `test-results/` summary. `make verify` (exact alias:
+`make check-all`) always adds quality/schema/metadata contracts, offline coverage,
+integration, installed wheel/schema smoke and strict docs. `make report` reads saved
+evidence only. Missing prerequisites, zero execution and skipped required lanes fail
+closed; local success and opt-in SDK compatibility never certify real Azure.
+See [Development Guide](development.md#makefile-targets) for the full contract.
+
### Direct Commands
If you prefer to run commands manually via `hatch`:
diff --git a/pyproject.toml b/pyproject.toml
index 1a62864..dd11514 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -104,14 +104,14 @@ python = "3.11"
features = ["dev", "docs"]
[tool.hatch.envs.default.scripts]
-format = "ruff format src tests"
-format-check = "ruff format --check src tests"
-style = "ruff check src tests && ruff format --check src tests"
-typecheck = "mypy src tests"
+format = "ruff format src tests tools/verification*.py"
+format-check = "ruff format --check src tests tools/verification*.py"
+style = "ruff check src tests tools/verification*.py && ruff format --check src tests tools/verification*.py"
+typecheck = "mypy src tests tools/verification*.py"
lint = "hatch run style && hatch run typecheck"
-test = "pytest -v tests"
+test = "python -m tools.verification test"
cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/"
-e2e-azure = "pytest -v -m e2e tests/e2e/ --no-cov"
+e2e-azure = "python -m tools.verification azure-e2e"
docs = "mkdocs serve"
precommit = "pre-commit run --all-files"
precommit-install = "pre-commit install"
diff --git a/tests/conftest.py b/tests/conftest.py
index 291f8cf..5284b19 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,6 +8,7 @@
from __future__ import annotations
from collections.abc import Callable
+from pathlib import Path
from unittest.mock import Mock
from azure.functions import HttpRequest
@@ -42,3 +43,11 @@ def _create_request(
return mock_req
return _create_request
+
+
+@pytest.fixture(autouse=True)
+def isolate_verification_summary(
+ request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ if request.module.__name__.split(".")[-1].startswith("test_verification"):
+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary"))
diff --git a/tests/test_verification_ci.py b/tests/test_verification_ci.py
new file mode 100644
index 0000000..a4b8f91
--- /dev/null
+++ b/tests/test_verification_ci.py
@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from pathlib import Path
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_ci_uses_canonical_runners_and_failure_artifacts() -> None:
+ workflows = ROOT / ".github/workflows"
+ ci = (workflows / "ci-test.yml").read_text()
+ assert 'python-version: ["3.11", "3.12", "3.13", "3.14"]' in ci
+ assert "python -m tools.verification test --no-cov" in ci
+ assert "python -m tools.verification integration" in ci
+ assert "python -m tools.verification sdk-compat" in ci
+ assert "make install-ci" in ci
+ for name in ("ci-test.yml", "e2e-azure.yml", "publish-pypi.yml", "worker-nightly.yml"):
+ text = (workflows / name).read_text()
+ assert "path: test-results/**" in text
+ blocks = text.split(" - name:")
+ evidence = [block for block in blocks if "path: test-results/**" in block]
+ assert evidence
+ assert all("if: always()" in block for block in evidence)
+
+
+def test_check_all_is_exact_verify_alias() -> None:
+ text = (ROOT / "Makefile").read_text()
+ assert "check-all: verify\n\n" in text
+ result = subprocess.run(
+ ["make", "--no-print-directory", "-n", "test", "CI=true", "PYTHON=matrix-python"],
+ cwd=ROOT,
+ text=True,
+ capture_output=True,
+ )
+ assert result.returncode == 0
+ assert result.stdout.strip() == "matrix-python -m tools.verification test"
+
+
+def test_clean_all_retains_deep_cleanup_contract() -> None:
+ result = subprocess.run(
+ ["make", "-n", "clean-all"],
+ cwd=ROOT,
+ text=True,
+ capture_output=True,
+ )
+ assert result.returncode == 0
+ assert "-m tools.verification clean" in result.stdout
+ assert "__pycache__" in result.stdout
+ assert "rm -rf .DS_Store .venv" in result.stdout
+
+
+def test_schema_and_wheel_commands_are_non_mutating() -> None:
+ result = subprocess.run(
+ ["make", "-n", "check-schema-hash", "wheel-smoke"],
+ cwd=ROOT,
+ text=True,
+ capture_output=True,
+ )
+ assert result.returncode == 0
+ assert "--update" not in result.stdout
+ assert "tools.verification_contracts schema-hash" in result.stdout
+ assert "tools.verification_wheel" in result.stdout
diff --git a/tests/test_verification_commands.py b/tests/test_verification_commands.py
new file mode 100644
index 0000000..6a419ce
--- /dev/null
+++ b/tests/test_verification_commands.py
@@ -0,0 +1,111 @@
+"""Regression contracts for the canonical developer commands."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+CONTRACT = (
+ "install",
+ "install-ci",
+ "doctor",
+ "format",
+ "format-check",
+ "check",
+ "test",
+ "integration",
+ "verify",
+ "e2e-azure",
+ "report",
+ "clean",
+)
+
+
+@pytest.mark.parametrize("target", CONTRACT)
+def test_make_contract_is_install_free_on_routine_runs(target: str) -> None:
+ # Given the public command contract, when dry-running a command.
+ result = subprocess.run(["make", "-n", target], cwd=ROOT, capture_output=True, text=True)
+ # Then routine runs never upgrade pip or bootstrap Hatch.
+ assert result.returncode == 0, result.stderr
+ assert "install --upgrade pip" not in result.stdout
+ if target not in {"install", "install-ci"}:
+ assert "pip install hatch" not in result.stdout
+
+
+def test_check_runs_types_once_and_read_only_contract_checks() -> None:
+ # Given the complete quality command, when expanding its recipe.
+ result = subprocess.run(["make", "-n", "check-steps"], cwd=ROOT, capture_output=True, text=True)
+ # Then types run once and pinned contracts are checked, never regenerated.
+ assert result.returncode == 0
+ assert result.stdout.count("-m mypy") == 1
+ assert "schema-hash" in result.stdout
+ assert "contracts" in result.stdout
+ assert "--update" not in result.stdout
+
+
+def test_schema_mismatch_does_not_rewrite_pin(tmp_path: Path) -> None:
+ # Given a checkout-shaped directory with an intentionally stale schema pin.
+ schema = tmp_path / "src/azure_functions_validation/schemas"
+ schema.mkdir(parents=True)
+ (schema / "endpoint.schema.json").write_text("{}\n")
+ pin = schema / "endpoint.schema.sha256"
+ before = b"incorrect-pin endpoint.schema.json\n"
+ pin.write_bytes(before)
+ environment = dict(os.environ, PYTHONPATH=str(ROOT))
+ # When invoking the same non-mutating checker used by make check.
+ result = subprocess.run(
+ [sys.executable, "-m", "tools.verification_contracts", "schema-hash"],
+ cwd=tmp_path,
+ env=environment,
+ capture_output=True,
+ text=True,
+ )
+ # Then drift fails without changing pinned metadata bytes.
+ assert result.returncode != 0
+ assert "digest drift" in result.stdout
+ assert pin.read_bytes() == before
+
+
+def test_unconfigured_azure_saves_incomplete(tmp_path: Path) -> None:
+ # Given no Azure URL and an isolated evidence root.
+ environment = dict(os.environ, PYTHONPATH=str(ROOT))
+ environment.pop("E2E_BASE_URL", None)
+ # When using the canonical CLI surface.
+ result = subprocess.run(
+ [sys.executable, "-m", "tools.verification", "azure-e2e"],
+ cwd=tmp_path,
+ env=environment,
+ capture_output=True,
+ text=True,
+ )
+ # Then configuration absence is saved as INCOMPLETE, not certification.
+ assert result.returncode != 0
+ data = json.loads((tmp_path / "test-results/azure-e2e/summary.json").read_text())
+ assert data["status"] == "INCOMPLETE"
+ assert data["selected"] == 0
+
+
+def test_make_check_retains_failed_prerequisite_evidence(tmp_path: Path) -> None:
+ # Given a failing lint prerequisite in a checkout-shaped Makefile.
+ makefile = (ROOT / "Makefile").read_text()
+ (tmp_path / "Makefile").write_text(makefile + "\nstyle:\n\t@echo password=CANARY; exit 7\n")
+ environment = dict(os.environ, CI="1", PYTHONPATH=str(ROOT))
+ # When invoking the actual public quality entry point.
+ result = subprocess.run(
+ ["make", "check", f"PYTHON={sys.executable}"],
+ cwd=tmp_path,
+ env=environment,
+ capture_output=True,
+ text=True,
+ )
+ # Then Make fails and the shared runner retains redacted failure evidence.
+ assert result.returncode != 0
+ data = json.loads((tmp_path / "test-results/check/summary.json").read_text())
+ assert data["lane"] == "check" and data["status"] == "FAIL"
+ assert "CANARY" not in (tmp_path / "test-results/check/logs/runner.log").read_text()
diff --git a/tests/test_verification_evidence.py b/tests/test_verification_evidence.py
new file mode 100644
index 0000000..084d49d
--- /dev/null
+++ b/tests/test_verification_evidence.py
@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import sys
+
+import pytest
+
+from tools.verification import execute, pytest_command
+from tools.verification_environment import clean, doctor
+from tools.verification_evidence import Summary, collect, redact, report, save
+from tools.verification_process import child, run, terminate
+
+
+@pytest.mark.parametrize(
+ ("cases", "exit_code", "status"),
+ [
+ ("", 0, "INCOMPLETE"),
+ ("", 5, "INCOMPLETE"),
+ ("", 0, "INCOMPLETE"),
+ ("", 0, "INCOMPLETE"),
+ ("", 1, "FAIL"),
+ ("", 0, "PASS"),
+ ("", 124, "FAIL"),
+ ],
+)
+def test_junit_classifies_execution_fail_closed(
+ tmp_path: Path, cases: str, exit_code: int, status: str
+) -> None:
+ (tmp_path / "junit.xml").write_text(f"{cases}")
+ summary = collect(tmp_path, ["pytest"], exit_code)
+ assert summary.status == status
+ assert (summary.exit_code == 0) == (status == "PASS")
+
+
+@pytest.mark.parametrize(
+ "secret",
+ [
+ "AccountKey=VERYSECRET;",
+ "SharedAccessSignature=VERYSECRET;",
+ "code=VERYSECRET&x=1",
+ "x-functions-key: VERYSECRET",
+ "Authorization: Bearer VERYSECRET",
+ "password=VERYSECRET",
+ "token=VERYSECRET",
+ "AzureWebJobsStorage=VERYSECRET",
+ "sas=VERYSECRET",
+ "api_key=VERYSECRET",
+ "DefaultEndpointsProtocol=https;AccountName=PRIVATE;AccountKey=VERYSECRET;",
+ ],
+)
+def test_secrets_are_redacted_from_all_saved_evidence(tmp_path: Path, secret: str) -> None:
+ (tmp_path / "junit.xml").write_text(
+ ""
+ + secret.replace("&", "&")
+ + ""
+ )
+ (tmp_path / "output.log").write_text(secret)
+ save(tmp_path, collect(tmp_path, ["pytest", secret], 0))
+ for path in tmp_path.rglob("*"):
+ if path.is_file():
+ assert "VERYSECRET" not in path.read_text()
+ json.loads((tmp_path / "summary.json").read_text())
+
+
+def test_environment_secrets_are_removed(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("CUSTOM_ACCESS_TOKEN", "opaque-sensitive-value")
+ assert redact("diagnostic opaque-sensitive-value") == "diagnostic [REDACTED]"
+
+
+def test_summary_schema_and_saved_report(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ directory = tmp_path / "test"
+ directory.mkdir()
+ (directory / "junit.xml").write_text("")
+ (directory / "coverage.xml").write_text('')
+ save(directory, collect(directory, ["not-an-executable"], 0))
+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(tmp_path / "job-summary"))
+ assert report(tmp_path) == 0
+ data = json.loads((directory / "summary.json").read_text())
+ assert data["schema_version"] == 1
+ assert (data["selected"], data["passed"], data["failed"], data["skipped"]) == (1, 1, 0, 0)
+ assert data["coverage"] == 0.99
+ assert str(directory / "junit.xml") in data["supporting_files"]
+ assert "passed=1" in (tmp_path / "job-summary").read_text()
+
+
+def test_report_without_results_does_not_create_evidence(tmp_path: Path) -> None:
+ assert report(tmp_path) == 1
+ assert list(tmp_path.iterdir()) == []
+
+
+def test_required_tools_fail_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr("tools.verification_environment.shutil.which", lambda _: None)
+ assert doctor() != 0
+ assert execute("host-smoke", []) != 0
+ data = json.loads((tmp_path / "test-results/host-smoke/summary.json").read_text())
+ assert data["status"] == "INCOMPLETE"
+
+
+def test_sdk_spike_is_opt_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.delenv("AZFUNC_2X_SPIKE", raising=False)
+ assert execute("sdk-compat", []) != 0
+ data = json.loads((tmp_path / "test-results/sdk-compat/summary.json").read_text())
+ assert data["status"] == "INCOMPLETE"
+
+
+def test_offline_and_worker_selection_are_explicit() -> None:
+ unit = pytest_command("test", [])
+ assert "--ignore=tests/e2e" in unit
+ assert "--ignore=tests/host_smoke" in unit
+ assert "not e2e and not compat2x" in unit
+ assert "tests/test_trigger_arg_name_worker_compat.py" in pytest_command("worker-compat", [])
+ assert "pythonpath=" in pytest_command("sdk-compat", [])
+
+
+def test_clean_preserves_environments_and_shared_storage(tmp_path: Path) -> None:
+ for name in (".venv", "src", "shared-azurite", "test-results", "dist", "build"):
+ (tmp_path / name).mkdir()
+ (tmp_path / name / "keep").touch()
+ clean(tmp_path)
+ assert sorted(p.name for p in tmp_path.iterdir()) == [".venv", "shared-azurite", "src"]
+
+
+def test_clean_unlinks_artifact_symlink_without_following(tmp_path: Path) -> None:
+ (tmp_path / "shared").mkdir()
+ (tmp_path / "shared/keep").touch()
+ (tmp_path / "test-results").symlink_to(tmp_path / "shared", target_is_directory=True)
+ clean(tmp_path)
+ assert (tmp_path / "shared/keep").exists()
+
+
+def test_process_failure_is_preserved(tmp_path: Path) -> None:
+ assert run([sys.executable, "-c", "raise SystemExit(7)"], tmp_path / "log") == 7
+ assert run(["/nonexistent/verification-command"], tmp_path / "missing") == 127
+
+
+def test_owned_process_is_reaped_after_interrupt(tmp_path: Path) -> None:
+ with (
+ pytest.raises(KeyboardInterrupt),
+ child([sys.executable, "-c", "import time; time.sleep(30)"], tmp_path / "log") as process,
+ ):
+ raise KeyboardInterrupt
+ assert process.poll() is not None
+
+
+def test_timeout_reaps_process_and_redacts_logs(tmp_path: Path) -> None:
+ code = run(
+ [sys.executable, "-u", "-c", "import time; print('password=VERYSECRET'); time.sleep(30)"],
+ tmp_path / "log",
+ 1,
+ )
+ assert code == 124
+ assert "VERYSECRET" not in (tmp_path / "log").read_text()
+
+
+def test_cleanup_tolerates_process_exit_race(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ def exited_group(pid: int, number: int) -> None:
+ raise ProcessLookupError(pid)
+
+ with child([sys.executable, "-c", "raise SystemExit(0)"], tmp_path / "log") as process:
+ monkeypatch.setattr("tools.verification_process.os.killpg", exited_group)
+ terminate(process)
+ assert process.returncode == 0
+
+
+def test_wheel_smoke_rejects_stale_dist_when_build_produces_nothing(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from tools.verification_wheel import main
+
+ (tmp_path / "dist").mkdir()
+ (tmp_path / "dist/stale.whl").write_bytes(b"stale")
+
+ def empty_build(command: list[str], log: Path, timeout: float = 600) -> int:
+ assert "--outdir" in command
+ assert str(tmp_path / "dist") not in command
+ return 0
+
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr("tools.verification_wheel.run", empty_build)
+ assert main() != 0
+
+
+def test_integration_preserves_separate_required_lanes(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from tools.verification import integration
+
+ calls: list[str] = []
+
+ def fake_execute(lane: str, extra: list[str]) -> int:
+ calls.append(lane)
+ save(
+ Path("test-results") / lane,
+ Summary(
+ 1,
+ lane,
+ ("test",),
+ 0,
+ 0,
+ 0,
+ 0,
+ "INCOMPLETE",
+ 1,
+ ),
+ )
+ return 1
+
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr("tools.verification.execute", fake_execute)
+ assert integration() != 0
+ assert calls == ["worker-compat", "host-smoke"]
+ assert (
+ json.loads(Path("test-results/integration/summary.json").read_text())["status"]
+ == "INCOMPLETE"
+ )
+
+
+def test_verify_runs_all_lanes_after_failure(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ from tools.verification import verify
+
+ calls: list[str] = []
+
+ def fake_run(command: list[str], log: Path, timeout: float = 600) -> int:
+ calls.append(command[1])
+ return 1
+
+ monkeypatch.chdir(tmp_path)
+ save(Path("test-results/test"), Summary(1, "test", ("old",), 1, 1, 0, 0, "PASS", 0))
+ monkeypatch.setattr("tools.verification.run", fake_run)
+ assert verify() != 0
+ assert calls == ["check", "test", "integration", "wheel-smoke", "docs"]
+ lanes = json.loads(Path("test-results/verify/lanes.json").read_text())
+ assert lanes[1]["status"] == "INCOMPLETE"
diff --git a/tests/test_verification_review.py b/tests/test_verification_review.py
new file mode 100644
index 0000000..337fc56
--- /dev/null
+++ b/tests/test_verification_review.py
@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from contextlib import nullcontext
+import json
+import os
+from pathlib import Path
+import subprocess
+from unittest.mock import Mock
+
+import pytest
+
+from tools.verification import focused
+from tools.verification_evidence import Summary, redact, save
+from tools.verification_host import wait_ready
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ '{"Authorization": "Bearer CANARY"}',
+ "{'Authorization': 'Bearer CANARY'}",
+ "Proxy-Authorization: Bearer CANARY",
+ '{"password": "alpha CANARY beta"}',
+ '{"token": "with \\"escaped\\" CANARY"}',
+ "connection_string=Endpoint=sb://ns/;SharedAccessKey=CANARY",
+ "InstrumentationKey=CANARY;IngestionEndpoint=https://x/",
+ "https://user:CANARY@example.test/path",
+ 'x',
+ ],
+)
+def test_quoted_credentials_are_consumed_whole(text: str) -> None:
+ # Given credential-bearing text, when redacting, then no suffix leaks.
+ assert "CANARY" not in redact(text)
+
+
+def test_save_keeps_structured_evidence_valid_and_binary_untouched(tmp_path: Path) -> None:
+ # Given JSON, binary and a symlink outside the evidence tree.
+ png = b"\x89PNG\r\n\x1a\n\xff\xfe\x00binary"
+ (tmp_path / "chart.png").write_bytes(png)
+ outside = tmp_path.parent / f"{tmp_path.name}-outside.log"
+ outside.write_text("password=CANARY")
+ (tmp_path / "link.log").symlink_to(outside)
+ (tmp_path / "extra.json").write_text('{"nested": ["Authorization: Bearer CANARY"]}')
+ # When saving evidence including tuple-valued command arguments.
+ save(tmp_path, Summary(1, "test", ('Authorization: "Bearer CANARY"',), 1, 1, 0, 0, "PASS", 0))
+ # Then JSON parses, secrets are absent, and non-text/outside assets stay intact.
+ for name in ("summary.json", "extra.json"):
+ assert "CANARY" not in str(json.loads((tmp_path / name).read_text()))
+ assert (tmp_path / "chart.png").read_bytes() == png
+ assert outside.read_text() == "password=CANARY"
+
+
+def test_known_secret_is_replaced_before_partial_patterns(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Given a complete secret containing separators and escaped quotes.
+ monkeypatch.setenv("MY_API_TOKEN", 'pre;fix&CANARY "value"')
+ # When redacting raw and JSON-escaped spellings, then the full value disappears.
+ for value in (os.environ["MY_API_TOKEN"], json.dumps(os.environ["MY_API_TOKEN"])[1:-1]):
+ assert "CANARY" not in redact(value)
+
+
+@pytest.mark.parametrize("error", [TimeoutError(), ConnectionError(), OSError()])
+def test_host_readiness_retries_transient_errors(
+ error: OSError, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ # Given a transient read failure followed by a ready host.
+ request = Mock(side_effect=[error, nullcontext(Mock(status=200))])
+ monkeypatch.setattr("tools.verification_host.urlopen", request)
+ monkeypatch.setattr("tools.verification_host.time.sleep", lambda _: None)
+ # When awaiting readiness, then the second probe succeeds.
+ wait_ready(Mock(poll=Mock(return_value=None)), "http://localhost/")
+ assert request.call_count == 2
+
+
+def test_failed_quality_command_saves_redacted_log_and_summary(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ import sys
+
+ # Given a failing quality command and a private evidence directory.
+ monkeypatch.chdir(tmp_path)
+ # When executing through the shared evidence runner.
+ code = focused("check", [sys.executable, "-c", "print('password=CANARY');exit(7)"])
+ # Then the exit code and failure survive, with a safe log.
+ assert code == 7
+ data = json.loads(Path("test-results/check/summary.json").read_text())
+ assert data["status"] == "FAIL" and data["exit_code"] == 7
+ assert "CANARY" not in Path("test-results/check/logs/runner.log").read_text()
+
+
+@pytest.mark.parametrize("target", ["doctor", "report", "clean"])
+def test_maintenance_never_bootstraps_hatch(target: str, tmp_path: Path) -> None:
+ # Given an empty Hatch data directory and a Hatch command that cannot run.
+ root = Path(__file__).resolve().parents[1]
+ environment = dict(os.environ, HATCH_DATA_DIR=str(tmp_path / "hatch"))
+ environment.pop("CI", None)
+ # When running a maintenance command, then no Hatch environment is created.
+ (tmp_path / "Makefile").write_bytes((root / "Makefile").read_bytes())
+ environment["PYTHONPATH"] = str(root)
+ result = subprocess.run(
+ ["make", target, "HATCH=false"],
+ cwd=tmp_path,
+ env=environment,
+ capture_output=True,
+ text=True,
+ )
+ assert "false run" not in result.stdout + result.stderr
+ assert not (tmp_path / "hatch").exists()
+ if target == "clean":
+ assert result.returncode == 0, result.stderr
diff --git a/tools/__init__.py b/tools/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/verification.py b/tools/verification.py
new file mode 100644
index 0000000..9d22413
--- /dev/null
+++ b/tools/verification.py
@@ -0,0 +1,261 @@
+"""Canonical verification CLI: python -m tools.verification, normally through Make."""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import asdict, replace
+import json
+import os
+from pathlib import Path
+import shutil
+import sys
+import xml.etree.ElementTree as ET
+
+if __package__ in (None, ""):
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from tools.verification_environment import clean, doctor
+from tools.verification_evidence import Status, Summary, collect, publish, report, save
+from tools.verification_host import host_smoke
+from tools.verification_process import run
+
+WORKER_TESTS = (
+ "tests/test_decorator.py",
+ "tests/test_integration.py",
+ "tests/test_trigger_arg_name_worker_compat.py",
+)
+CONTRACT_TESTS = (
+ "tests/test_endpoint_schema.py",
+ "tests/test_metadata_helpers_drift.py",
+ "tests/test_metadata_contract.py",
+ "tests/test_toolkit_metadata.py",
+ "tests/test_metadata.py",
+ "tests/test_check_wheel_schemas.py",
+)
+
+
+def pytest_command(lane: str, extra: list[str]) -> list[str]:
+ directory = Path("test-results") / lane
+ command = [sys.executable, "-m", "pytest"]
+ match lane:
+ case "test":
+ command += [
+ "tests/",
+ "-m",
+ "not e2e and not compat2x",
+ "--ignore=tests/e2e",
+ "--ignore=tests/host_smoke",
+ "--cov-report=xml:" + str(directory / "coverage.xml"),
+ ]
+ case "worker-compat":
+ command += [*WORKER_TESTS, "--no-cov", "-m", "not e2e and not compat2x"]
+ case "sdk-compat":
+ command += [
+ "tests/test_worker_compat_2x_spike.py",
+ "-m",
+ "compat2x",
+ "-o",
+ "addopts=",
+ "-o",
+ "pythonpath=",
+ ]
+ case "contracts":
+ command += [*CONTRACT_TESTS, "--no-cov"]
+ case _:
+ command += ["tests/e2e/", "-m", "e2e", "--no-cov"]
+ return command + ["--junitxml=" + str(directory / "junit.xml"), *extra]
+
+
+def execute(lane: str, extra: list[str]) -> int:
+ directory = Path("test-results") / lane
+ if directory.exists():
+ shutil.rmtree(directory)
+ (directory / "logs").mkdir(parents=True)
+ command = pytest_command(lane, extra)
+ reason = ""
+ code = 0
+ blocked = False
+ try:
+ if lane in {"azure-e2e", "local-e2e"} and not os.environ.get("E2E_BASE_URL"):
+ blocked, code, reason = True, 2, "Missing prerequisite: E2E_BASE_URL"
+ if lane == "sdk-compat" and os.environ.get("AZFUNC_2X_SPIKE") != "1":
+ blocked, code, reason = True, 2, "Opt-in SDK spike requires AZFUNC_2X_SPIKE=1"
+ if lane == "host-smoke":
+ command = ["func", "start", "(managed validation host smoke)"]
+ if doctor():
+ blocked, code, reason = True, 2, "Missing required integration tools"
+ else:
+ code = host_smoke(directory)
+ elif not blocked:
+ code = run(command, directory / "logs/runner.log")
+ except KeyboardInterrupt:
+ code, reason = 130, "Interrupted; owned child processes terminated"
+ except (OSError, ET.ParseError, ValueError) as error:
+ code, reason = 2, str(error)
+ try:
+ summary = collect(directory, command, code)
+ except (ET.ParseError, ValueError) as error:
+ summary = Summary(
+ 1,
+ lane,
+ tuple(command),
+ 0,
+ 0,
+ 0,
+ 0,
+ "INCOMPLETE",
+ 2,
+ reason=f"Invalid evidence: {error}",
+ )
+ if blocked:
+ summary = replace(summary, status="INCOMPLETE")
+ ET.ElementTree(ET.Element("testsuite", name=lane, tests="0")).write(
+ directory / "junit.xml", encoding="unicode"
+ )
+ if lane == "sdk-compat":
+ reason = (
+ reason + "; " if reason else ""
+ ) + "SDK compatibility spike, NOT Azure certification"
+ summary = save(directory, replace(summary, reason=reason or summary.reason))
+ publish(summary)
+ return summary.exit_code
+
+
+def aggregate(lane: str, outcomes: list[Summary]) -> int:
+ status: Status = "FAIL" if any(item.status == "FAIL" for item in outcomes) else "PASS"
+ if status == "PASS" and any(item.status != "PASS" for item in outcomes):
+ status = "INCOMPLETE"
+ directory = Path("test-results") / lane
+ directory.mkdir(parents=True, exist_ok=True)
+ summary = Summary(
+ 1,
+ lane,
+ ("make", lane),
+ sum(s.selected for s in outcomes),
+ sum(s.passed for s in outcomes),
+ sum(s.failed for s in outcomes),
+ sum(s.skipped for s in outcomes),
+ status,
+ int(status != "PASS"),
+ reason="; ".join(f"{item.lane}={item.status}" for item in outcomes),
+ )
+ (directory / "lanes.json").write_text(json.dumps([asdict(s) for s in outcomes], indent=2))
+ publish(save(directory, summary))
+ return summary.exit_code
+
+
+def integration() -> int:
+ directory = Path("test-results/integration")
+ if directory.exists():
+ shutil.rmtree(directory)
+ outcomes = []
+ for lane in ("worker-compat", "host-smoke"):
+ execute(lane, [])
+ source = Path("test-results") / lane
+ path = source / "summary.json"
+ outcomes.append(Summary(**json.loads(path.read_text())))
+ for pattern, destination in (("**/*junit.xml", "junit"), ("logs/*", "logs")):
+ for artifact in source.glob(pattern):
+ target = directory / destination / f"{lane}-{artifact.name}"
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(artifact, target)
+ return aggregate("integration", outcomes)
+
+
+def verify() -> int:
+ directory = Path("test-results/verify")
+ if directory.exists():
+ shutil.rmtree(directory)
+ outcomes = []
+ for target in ("check", "test", "integration", "wheel-smoke", "docs"):
+ path = Path("test-results") / target / "summary.json"
+ path.unlink(missing_ok=True)
+ code = run(["make", target], directory / "logs" / f"{target}.log", 1200)
+ if path.exists():
+ summary = Summary(**json.loads(path.read_text()))
+ if code and summary.status == "PASS":
+ summary = replace(summary, status="FAIL", exit_code=code)
+ else:
+ required = target != "check"
+ summary = Summary(
+ 1,
+ target,
+ ("make", target),
+ int(not required),
+ int(not required and code == 0),
+ int(code != 0),
+ 0,
+ "INCOMPLETE" if required else ("FAIL" if code else "PASS"),
+ code or int(required),
+ reason="Required lane summary missing" if required else "",
+ )
+ outcomes.append(summary)
+ return aggregate("verify", outcomes)
+
+
+def focused(lane: str, command: list[str]) -> int:
+ directory = Path("test-results") / lane
+ if directory.exists():
+ shutil.rmtree(directory)
+ code = run(command, directory / "logs/runner.log", 1200)
+ summary = Summary(
+ 1,
+ lane,
+ tuple(command),
+ 1,
+ int(code == 0),
+ int(code != 0),
+ 0,
+ "PASS" if code == 0 else "FAIL",
+ code,
+ )
+ publish(save(directory, summary))
+ return code
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "lane",
+ choices=(
+ "test",
+ "check",
+ "integration",
+ "worker-compat",
+ "host-smoke",
+ "sdk-compat",
+ "contracts",
+ "verify",
+ "azure-e2e",
+ "local-e2e",
+ "report",
+ "doctor",
+ "clean",
+ "focused",
+ ),
+ )
+ args, extra = parser.parse_known_args()
+ if not (3, 11) <= sys.version_info[:2] < (3, 15):
+ parser.error("Python >=3.11,<3.15 is required")
+ match args.lane:
+ case "report":
+ return report(Path("test-results"))
+ case "doctor":
+ return doctor()
+ case "clean":
+ clean(Path.cwd())
+ return 0
+ case "verify":
+ return verify()
+ case "check":
+ return focused("check", ["make", "check-steps"])
+ case "integration":
+ return integration()
+ case "focused":
+ return focused(extra[0], [sys.executable, *extra[1:]])
+ case _:
+ return execute(args.lane, extra)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/verification_contracts.py b/tools/verification_contracts.py
new file mode 100644
index 0000000..a739e7f
--- /dev/null
+++ b/tools/verification_contracts.py
@@ -0,0 +1,21 @@
+"""Non-mutating schema-pin and metadata-contract checks."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+def schema_hash(root: Path) -> int:
+ directory = root / "src/azure_functions_validation/schemas"
+ expected = (directory / "endpoint.schema.sha256").read_text().split()[0]
+ actual = hashlib.sha256((directory / "endpoint.schema.json").read_bytes()).hexdigest()
+ if actual != expected:
+ print(f"endpoint.schema.json digest drift: pinned={expected} actual={actual}")
+ return 1
+ print("endpoint.schema.json digest matches pin.")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(schema_hash(Path.cwd()))
diff --git a/tools/verification_environment.py b/tools/verification_environment.py
new file mode 100644
index 0000000..45d86f7
--- /dev/null
+++ b/tools/verification_environment.py
@@ -0,0 +1,53 @@
+"""Read-only prerequisites and allowlisted generated-artifact cleanup."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+import shutil
+import sys
+
+
+def doctor(live: bool = True) -> int:
+ healthy = (3, 11) <= sys.version_info[:2] < (3, 15)
+ print(f"python REQUIRED >=3.11,<3.15: {sys.version.split()[0]} ({sys.executable})")
+ for module in ("venv", "ruff", "mypy", "pytest", "bandit", "build", "mkdocs"):
+ found = importlib.util.find_spec(module) is not None
+ print(f"{module} REQUIRED: {'found' if found else 'MISSING'}")
+ healthy = healthy and found
+ if not healthy:
+ print("development environment not created: run make install")
+ for tool in ("func", "azurite"):
+ path = shutil.which(tool)
+ print(f"{tool} REQUIRED for integration/verify: {path or 'MISSING'}")
+ if live:
+ healthy = healthy and path is not None
+ return 0 if healthy else 1
+
+
+def clean(root: Path) -> None:
+ for name in (
+ "test-results",
+ "dist",
+ "build",
+ ".pytest_cache",
+ ".mypy_cache",
+ ".ruff_cache",
+ ".coverage",
+ "coverage.xml",
+ "htmlcov",
+ "site",
+ "__pycache__",
+ ):
+ path = root / name
+ if path.is_symlink() or path.is_file():
+ path.unlink()
+ elif path.is_dir():
+ shutil.rmtree(path)
+ for tree in ("src", "tests", "tools", "scripts", "examples"):
+ for path in (root / tree).rglob("__pycache__"):
+ if not any(part in {".venv", "venv", "env"} for part in path.parts):
+ if path.is_symlink():
+ path.unlink()
+ else:
+ shutil.rmtree(path)
diff --git a/tools/verification_evidence.py b/tools/verification_evidence.py
new file mode 100644
index 0000000..5ea3c9a
--- /dev/null
+++ b/tools/verification_evidence.py
@@ -0,0 +1,221 @@
+"""Versioned, redacted evidence shared by local commands and CI."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, replace
+from importlib.metadata import PackageNotFoundError, version
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import subprocess
+import sys
+from typing import Literal
+import xml.etree.ElementTree as ET
+
+Status = Literal["PASS", "FAIL", "INCOMPLETE", "NOT_APPLICABLE"]
+
+
+def read_xml_root(path: Path) -> ET.Element:
+ """Parse a locally generated report, refusing entity declarations."""
+ data = path.read_bytes()
+ if b"(?:proxy-)?authorization[\"']?\s*[:=]\s*)(?P" + _QUOTED + r"|[^\r\n\"<]+)"
+)
+_URL_CREDENTIALS = re.compile(r"(https?://)[^/@\s\"']+:[^/@\s\"']+@")
+_SECRET_FIELD = re.compile(
+ r"(?i)(?(?:[\w.-]*(?:token|password|passwd|secret|api[_-]?key|"
+ r"connection[_-]?string)|AccountKey|SharedAccessKey|SharedAccessSignature|"
+ r"AzureWebJobsStorage|InstrumentationKey|x-functions-key|code|sig|sas)"
+ r"[\"']?\s*[:=]\s*)(?P" + _QUOTED + r"|[^\s;&<>\"',]+)"
+)
+_SECRET_ENV_NAME = re.compile(
+ r"(?i)secret|token|password|connection|string|accountkey|functions.key|api.?key|sas"
+)
+_TEXT_SUFFIXES = frozenset({".log", ".txt", ".xml", ".json", ".md"})
+
+
+def _mask(match: re.Match[str]) -> str:
+ value = match.group("value")
+ quote = value[0] if value[:1] in {'"', "'"} else ""
+ return f"{match.group('key')}{quote}{_REDACTED}{quote}"
+
+
+def _known_secrets() -> list[str]:
+ values = {
+ value
+ for key, value in os.environ.items()
+ if len(value) >= 6 and _SECRET_ENV_NAME.search(key)
+ }
+ escaped = {json.dumps(value)[1:-1] for value in values}
+ return sorted(values | escaped, key=len, reverse=True)
+
+
+def redact(text: str) -> str:
+ for value in _known_secrets():
+ text = text.replace(value, _REDACTED)
+ text = _AUTHORIZATION.sub(_mask, text)
+ text = _URL_CREDENTIALS.sub(r"\1" + _REDACTED + "@", text)
+ text = re.sub(r"(?i)(DefaultEndpointsProtocol=)[^\s<\"']+", r"\1[REDACTED]", text)
+ return _SECRET_FIELD.sub(_mask, text)
+
+
+def redact_json(value: object) -> object:
+ if isinstance(value, str):
+ return redact(value)
+ if isinstance(value, (list, tuple)):
+ return [redact_json(item) for item in value]
+ if isinstance(value, dict):
+ return {redact(str(key)): redact_json(item) for key, item in value.items()}
+ return value
+
+
+def dump_json(value: object) -> str:
+ return json.dumps(redact_json(value), indent=2) + "\n"
+
+
+def environment(live: bool = False) -> dict[str, str]:
+ metadata = {
+ "python": platform.python_version(),
+ "executable": sys.executable,
+ "platform": platform.platform(),
+ "runtime": platform.python_implementation(),
+ }
+ for package in ("azure-functions", "azure-functions-validation", "pydantic", "hatch"):
+ try:
+ metadata[package] = version(package)
+ except PackageNotFoundError:
+ metadata[package] = "not installed"
+ for key in ("E2E_BASE_URL", "E2E_EXPECTED_VERSION", "GITHUB_SHA", "AZFUNC_2X_SPIKE"):
+ if key in os.environ:
+ metadata[key] = redact(os.environ[key])
+ for tool in ("func", "azurite") if live else ():
+ try:
+ result = subprocess.run([tool, "--version"], capture_output=True, text=True, timeout=10)
+ metadata[tool] = redact(result.stdout.strip())
+ except (OSError, subprocess.TimeoutExpired):
+ metadata[tool] = "unavailable"
+ return metadata
+
+
+def collect(directory: Path, command: list[str], exit_code: int) -> Summary:
+ cases: list[ET.Element] = []
+ for path in directory.rglob("*.xml"):
+ if "junit" in path.name:
+ cases.extend(read_xml_root(path).iter("testcase"))
+ skipped = sum(case.find("skipped") is not None for case in cases)
+ failed = sum(
+ case.find("failure") is not None or case.find("error") is not None for case in cases
+ )
+ passed = len(cases) - skipped - failed
+ status: Status = "PASS"
+ if failed or exit_code not in (0, 5):
+ status = "FAIL"
+ elif not cases or not passed or (directory.name != "test" and skipped):
+ status = "INCOMPLETE"
+ elif exit_code:
+ status = "FAIL"
+ coverage_path = directory / "coverage.xml"
+ coverage = None
+ if coverage_path.exists():
+ coverage = float(read_xml_root(coverage_path).get("line-rate", "0"))
+ return Summary(
+ 1,
+ directory.name,
+ tuple(command),
+ len(cases),
+ passed,
+ failed,
+ skipped,
+ status,
+ exit_code if exit_code else int(status != "PASS"),
+ coverage=coverage,
+ )
+
+
+def render(summary: Summary) -> str:
+ return (
+ f"{summary.lane}: {summary.status} | selected={summary.selected} "
+ f"passed={summary.passed} failed={summary.failed} skipped={summary.skipped} "
+ f"exit={summary.exit_code}" + (f" | {summary.reason}" if summary.reason else "")
+ )
+
+
+def publish(summary: Summary) -> None:
+ text = redact(render(summary))
+ print(text)
+ destination = os.environ.get("GITHUB_STEP_SUMMARY")
+ if destination:
+ with Path(destination).open("a", encoding="utf-8") as stream:
+ stream.write(f"\n- {text}\n")
+
+
+def _redact_file_text(suffix: str, text: str) -> str:
+ if suffix == ".json":
+ try:
+ return dump_json(json.loads(text))
+ except ValueError:
+ return redact(text)
+ return redact(text)
+
+
+def save(directory: Path, summary: Summary) -> Summary:
+ directory.mkdir(parents=True, exist_ok=True)
+ for path in directory.rglob("*"):
+ if path.is_symlink() or not path.is_file() or path.suffix not in _TEXT_SUFFIXES:
+ continue
+ try:
+ original = path.read_text(encoding="utf-8")
+ except UnicodeDecodeError:
+ continue
+ redacted = _redact_file_text(path.suffix, original)
+ if redacted != original:
+ path.write_text(redacted, encoding="utf-8")
+ (directory / "environment.json").write_text(
+ dump_json(environment(live=directory.name in {"integration", "host-smoke"})),
+ encoding="utf-8",
+ )
+ summary = replace(
+ summary,
+ supporting_files=tuple(
+ str(path)
+ for path in sorted(directory.rglob("*"))
+ if path.is_file() and path.name != "summary.json"
+ ),
+ )
+ (directory / "summary.json").write_text(dump_json(asdict(summary)), encoding="utf-8")
+ return summary
+
+
+def report(root: Path) -> int:
+ paths = sorted(root.glob("*/summary.json"))
+ if not paths:
+ print("No saved verification results")
+ return 1
+ for path in paths:
+ publish(Summary(**json.loads(path.read_text(encoding="utf-8"))))
+ return 0
diff --git a/tools/verification_host.py b/tools/verification_host.py
new file mode 100644
index 0000000..06ad5a7
--- /dev/null
+++ b/tools/verification_host.py
@@ -0,0 +1,131 @@
+"""Real validation host smoke with private Azurite storage and owned processes."""
+
+from __future__ import annotations
+
+import base64
+from contextlib import ExitStack
+from http.client import HTTPException
+import json
+import os
+from pathlib import Path
+import re
+import socket
+import subprocess
+import sys
+import tempfile
+import time
+from urllib.request import urlopen
+import xml.etree.ElementTree as ET
+
+from tools.verification_process import child
+
+
+def free_port() -> int:
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ return int(sock.getsockname()[1])
+
+
+def wait_ready(process: subprocess.Popen[str], url: str) -> None:
+ deadline = time.monotonic() + 90
+ while time.monotonic() < deadline:
+ if process.poll() is not None:
+ raise RuntimeError("service exited before readiness")
+ try:
+ with urlopen(url, timeout=1) as response:
+ if response.status < 500:
+ return
+ except (OSError, HTTPException):
+ time.sleep(0.25)
+ raise TimeoutError("service readiness deadline exceeded")
+
+
+def host_smoke(directory: Path) -> int:
+ directory = directory.resolve()
+ suite = ET.Element("testsuite", name="real-functions-host")
+ case = ET.SubElement(suite, "testcase", name="validated_context_and_response_filtering")
+ try:
+ with (
+ tempfile.TemporaryDirectory(prefix="validation-host-") as temporary,
+ ExitStack() as stack,
+ ):
+ work = Path(temporary)
+ for name in ("function_app.py", "host.json"):
+ (work / name).write_bytes((Path("tests/host_smoke") / name).read_bytes())
+ blob, queue, table, port = (free_port() for _ in range(4))
+ key = base64.b64encode(b"validation-local-emulator-only" * 2).decode()
+ emulator = stack.enter_context(
+ child(
+ [
+ "env",
+ f"AZURITE_ACCOUNTS=validationdev:{key}",
+ "azurite",
+ "--silent",
+ "--location",
+ str(work / "storage"),
+ "--blobPort",
+ str(blob),
+ "--queuePort",
+ str(queue),
+ "--tablePort",
+ str(table),
+ ],
+ directory / "logs/azurite.log",
+ )
+ )
+ deadline = time.monotonic() + 30
+ for service in (blob, queue, table):
+ while True:
+ if emulator.poll() is not None or time.monotonic() >= deadline:
+ raise TimeoutError("Azurite failed readiness")
+ try:
+ with socket.create_connection(("127.0.0.1", service), timeout=1):
+ break
+ except OSError:
+ time.sleep(0.25)
+ (work / "local.settings.json").write_text(
+ json.dumps(
+ {
+ "IsEncrypted": False,
+ "Values": {
+ "FUNCTIONS_WORKER_RUNTIME": "python",
+ "AzureWebJobsStorage": (
+ f"DefaultEndpointsProtocol=http;AccountName=validationdev;AccountKey={key};"
+ f"BlobEndpoint=http://127.0.0.1:{blob}/validationdev;"
+ f"QueueEndpoint=http://127.0.0.1:{queue}/validationdev;"
+ f"TableEndpoint=http://127.0.0.1:{table}/validationdev;"
+ ),
+ },
+ }
+ )
+ )
+ previous_path = os.environ.get("PATH", "")
+ os.environ["PATH"] = str(Path(sys.executable).parent) + os.pathsep + previous_path
+ try:
+ host = stack.enter_context(
+ child(
+ ["func", "start", "--python", "--verbose", "--port", str(port)],
+ directory / "logs/host.log",
+ work,
+ )
+ )
+ url = f"http://127.0.0.1:{port}/api/smoke"
+ wait_ready(host, url)
+ with urlopen(url, timeout=5) as response:
+ assert json.load(response) == {"function_name": "smoke"}
+ finally:
+ os.environ["PATH"] = previous_path
+ assert not re.search(
+ r"0 functions (loaded|found \(Worker\))|No job functions found|"
+ r"Executed 'Functions\..*' \(Failed",
+ (directory / "logs/host.log").read_text(),
+ )
+ code = 0
+ except (OSError, RuntimeError, TimeoutError, AssertionError, ValueError) as error:
+ ET.SubElement(case, "failure").text = str(error)
+ code = 1
+ finally:
+ destination = directory / "junit/junit.xml"
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ ET.ElementTree(suite).write(destination, encoding="unicode")
+ return code
diff --git a/tools/verification_process.py b/tools/verification_process.py
new file mode 100644
index 0000000..e7da4ee
--- /dev/null
+++ b/tools/verification_process.py
@@ -0,0 +1,81 @@
+"""Bounded subprocess ownership and redacted output capture."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+import os
+from pathlib import Path
+import signal
+import subprocess
+import tempfile
+from types import FrameType
+
+from tools.verification_evidence import redact
+
+
+def terminate(process: subprocess.Popen[str]) -> None:
+ if process.poll() is None:
+ try:
+ os.killpg(process.pid, signal.SIGTERM)
+ except ProcessLookupError:
+ process.wait()
+ return
+ try:
+ process.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ os.killpg(process.pid, signal.SIGKILL)
+ process.wait()
+ try:
+ os.killpg(process.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ return
+
+
+@contextmanager
+def child(
+ command: list[str], log: Path, cwd: Path | None = None
+) -> Iterator[subprocess.Popen[str]]:
+ log.parent.mkdir(parents=True, exist_ok=True)
+ # Keep raw output anonymous until credentials have been removed.
+ with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output:
+ with subprocess.Popen(
+ command,
+ cwd=cwd,
+ stdout=output,
+ stderr=subprocess.STDOUT,
+ text=True,
+ start_new_session=True,
+ ) as process:
+
+ def interrupted(number: int, frame: FrameType | None) -> None:
+ terminate(process)
+ raise KeyboardInterrupt(number)
+
+ previous = {
+ sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT)
+ }
+ try:
+ yield process
+ finally:
+ terminate(process)
+ for sig, handler in previous.items():
+ signal.signal(sig, handler)
+ output.seek(0)
+ text = redact(output.read())
+ log.write_text(text, encoding="utf-8")
+ print(text, end="")
+
+
+def run(command: list[str], log: Path, timeout: float = 600) -> int:
+ try:
+ with child(command, log) as process:
+ try:
+ code = process.wait(timeout=timeout)
+ return code if code >= 0 else 128 - code
+ except subprocess.TimeoutExpired:
+ return 124
+ except FileNotFoundError as error:
+ log.parent.mkdir(parents=True, exist_ok=True)
+ log.write_text(redact(str(error)), encoding="utf-8")
+ return 127
diff --git a/tools/verification_wheel.py b/tools/verification_wheel.py
new file mode 100644
index 0000000..065772f
--- /dev/null
+++ b/tools/verification_wheel.py
@@ -0,0 +1,72 @@
+"""Install the built wheel and check schema/import identity outside the checkout."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import subprocess
+import sys
+import sysconfig
+import tempfile
+
+from scripts.check_wheel_schemas import verify_wheel
+from tools.verification_evidence import redact
+from tools.verification_process import run
+
+
+def main() -> int:
+ directory = Path("test-results/wheel-smoke/logs")
+ with tempfile.TemporaryDirectory(prefix="validation-wheel-") as temporary:
+ root = Path(temporary)
+ code = run(
+ [sys.executable, "-m", "build", "--no-isolation", "--outdir", str(root / "dist")],
+ directory / "build.log",
+ )
+ if code:
+ return code
+ wheels = list((root / "dist").glob("*.whl"))
+ if len(wheels) != 1:
+ print("Expected exactly one freshly built wheel in isolated output directory")
+ return 1
+ verify_wheel(wheels[0])
+ subprocess.run(
+ [sys.executable, "-m", "venv", "--system-site-packages", str(root / "venv")], check=True
+ )
+ python = str(root / "venv/bin/python")
+ packages = (
+ root
+ / "venv/lib"
+ / f"python{sys.version_info.major}.{sys.version_info.minor}"
+ / "site-packages"
+ )
+ (packages / "verification-dependencies.pth").write_text(
+ sysconfig.get_path("purelib") + "\n"
+ )
+ code = run(
+ [python, "-m", "pip", "install", "--no-deps", "--no-index", str(wheels[0].resolve())],
+ directory / "wheel-install.log",
+ )
+ if code:
+ return code
+ script = (
+ "from pathlib import Path; from importlib import resources; import hashlib; "
+ "import azure_functions_validation as a; "
+ f"assert Path(a.__file__).resolve().is_relative_to(Path({str(root)!r})); "
+ "assert callable(a.validate_http); "
+ "p=resources.files('azure_functions_validation.schemas'); "
+ "assert hashlib.sha256(p.joinpath('endpoint.schema.json').read_bytes()).hexdigest() "
+ "== p.joinpath('endpoint.schema.sha256').read_text().split()[0]; "
+ "print('Installed wheel imports outside checkout; schema matches pin')"
+ )
+ with tempfile.TemporaryFile(mode="w+", encoding="utf-8") as output:
+ result = subprocess.run(
+ [python, "-I", "-c", script], cwd=root, stdout=output, stderr=subprocess.STDOUT
+ )
+ output.seek(0)
+ text = redact(output.read())
+ (directory / "wheel-smoke.log").write_text(text)
+ print(text)
+ return result.returncode
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())