ci: unify verification commands and test reports

.github/workflows/ci-test.yml

@@ -92,20 +92,10 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: Install package with dev extras
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+        run: make install-ci
 
-      - name: Lint workflows (pin hygiene + release-gate canonical SHAs)
-        run: |
-          python tools/lint_release_workflows.py
-          python tools/lint_workflow_pins.py
-          python tools/lint_hatch_matrix.py
-
-      - name: Ruff style check
-        run: |
-          ruff check src tests
-          ruff format --check src tests
+      - name: Quality gate (make check)
+        run: make check
 
       - name: Ruff format check (changed Python files)
         if: github.event_name == 'pull_request'
@@ -114,8 +104,12 @@ jobs:
           PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: python tools/check_pr_format.py "$PR_BASE_SHA" "$PR_HEAD_SHA"
 
-      - name: Type check (mypy, strict)
-        run: mypy src tests
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: quality-evidence
+          path: test-results/**
 
   test:
     name: Test (Py ${{ matrix.python-version }}, af ${{ matrix.azure-functions-version }})
@@ -169,9 +163,7 @@ jobs:
           PY
 
       - name: Install package with dev extras (on the matrix interpreter)
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+        run: make install-ci
 
       - name: Install azure-functions ${{ matrix.azure-functions-version }}
         run: |
@@ -206,26 +198,33 @@ jobs:
           IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' && matrix.azure-functions-version == 'latest' }}
         run: |
           if [ "$IS_COVERAGE_CELL" = "true" ]; then
-            pytest tests/
+            python -m tools.verification test
           else
-            pytest tests/ --no-cov
+            python -m tools.verification test --no-cov
           fi
 
       - name: Verify coverage output
         if: matrix.python-version == '3.11' && matrix.azure-functions-version == 'latest'
-        run: ls -lh coverage.xml
+        run: ls -lh test-results/test/coverage.xml
 
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.11' && matrix.azure-functions-version == 'latest' && github.actor != 'dependabot[bot]'
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage.xml
+          files: ./test-results/test/coverage.xml
           verbose: true
           fail_ci_if_error: false
         env:
           CODECOV_RETRIES: 3
 
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: test-evidence-py${{ matrix.python-version }}-af${{ matrix.azure-functions-version }}
+          path: test-results/**
+
   docs-check:
     name: Documentation checks
     runs-on: ubuntu-latest
@@ -255,7 +254,14 @@ jobs:
         run: pytest tests/test_screenshot_manifest.py --no-cov
 
       - name: Build docs (strict)
-        run: mkdocs build --strict
+        run: make docs
+
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: docs-evidence
+          path: test-results/**
 
   # Advisory regression lane: visible on PRs/main but intentionally omitted
   # from ci-required so noisy shared-runner timing cannot block a merge.
@@ -281,6 +287,13 @@ jobs:
       - name: Check spec-generation budgets
         run: make perf
 
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: perf-evidence
+          path: test-results/**
+
   # The single required status check for branch protection.
   #
   # Branch protection matches required checks by literal name, so requiring
@@ -390,7 +403,14 @@ jobs:
           PY
 
       - name: Run full suite against minimum runtime dependencies
-        run: pytest tests/ --no-cov
+        run: python -m tools.verification test --no-cov
+
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: minimum-dependencies-evidence
+          path: test-results/**
 
   artifact-build:
     name: Build compatibility artifacts
@@ -540,87 +560,18 @@ jobs:
 
       - name: Install Functions Core Tools v4 and smoke app
         run: |
-          npm install --global azure-functions-core-tools@4 --unsafe-perm true
-          python -m venv tests/host_smoke/.venv
-          tests/host_smoke/.venv/bin/python -m pip install --upgrade pip
-          cd tests/host_smoke
-          .venv/bin/python -m pip install -r requirements.txt
+          npm install --global azure-functions-core-tools@4 azurite --unsafe-perm true
+          make install-ci
 
       - name: Exercise OpenAPI routes on a real Functions host
-        run: |
-          npx --yes azurite --silent > "$RUNNER_TEMP/azurite.log" 2>&1 &
-          azurite_pid=$!
-          trap 'kill "$azurite_pid" "$host_pid" 2>/dev/null || true' EXIT
-
-          cd tests/host_smoke
-          export PATH="$PWD/.venv/bin:$PATH"
-          timeout 120s func start --verbose > "$RUNNER_TEMP/host-smoke.log" 2>&1 &
-          host_pid=$!
-
-          ready=false
-          for _ in {1..60}; do
-            if curl --fail --silent http://127.0.0.1:7071/api/custom_name \
-              --output "$RUNNER_TEMP/custom-route.json"; then
-              ready=true
-              break
-            fi
-            if ! kill -0 "$host_pid" 2>/dev/null; then
-              break
-            fi
-            sleep 2
-          done
-
-          if [ "$ready" != true ]; then
-            cat "$RUNNER_TEMP/host-smoke.log"
-            echo "::error::The custom-name route did not become ready"
-            exit 1
-          fi
-
-          python -c 'import json, pathlib; assert json.loads(pathlib.Path("'"$RUNNER_TEMP"'/custom-route.json").read_text()) == {"function_name": "custom_name"}'
-          curl --fail --silent http://127.0.0.1:7071/api/openapi.json \
-            --output "$RUNNER_TEMP/openapi.json"
-          python - <<'PY'
-          import json
-          import os
-          import pathlib
-          import urllib.request
-
-          spec = json.loads((pathlib.Path(os.environ["RUNNER_TEMP"]) / "openapi.json").read_text())
-          assert spec["openapi"], "spec is missing the 'openapi' version key"
-          paths = spec["paths"]
-
-          # #786: the fixture previously documented the pre-function_name route
-          # ('/api/python_handler_name') while the host actually served
-          # '/api/custom_name', and the host-smoke job passed anyway because it
-          # only checked a hard-coded URL, never the generated document itself.
-          assert "/api/python_handler_name" not in paths, (
-              f"spec still documents the stale pre-function_name route; paths={sorted(paths)}"
-          )
-          app_paths = [p for p in paths if p not in ("/api/openapi.json", "/api/docs")]
-          assert app_paths == ["/api/custom_name"], (
-              f"expected the documented app route to be exactly ['/api/custom_name'], got {app_paths}"
-          )
-
-          # Exercise the URL taken FROM the generated document -- not a second
-          # hard-coded '/api/custom_name' literal -- so a documentation/runtime
-          # mismatch cannot hide behind two copies of the same string.
-          documented_path = app_paths[0]
-          with urllib.request.urlopen(f"http://127.0.0.1:7071{documented_path}") as resp:
-              body = json.loads(resp.read())
-          assert body == {"function_name": "custom_name"}, body
-          PY
-          curl --fail --silent http://127.0.0.1:7071/api/docs \
-            --output "$RUNNER_TEMP/swagger.html"
-          grep -q 'SwaggerUIBundle' "$RUNNER_TEMP/swagger.html"
+        run: python -m tools.verification integration
 
-          kill "$host_pid" 2>/dev/null || true
-          wait "$host_pid" 2>/dev/null || true
-          cat "$RUNNER_TEMP/host-smoke.log"
-
-          if grep -Eq "0 functions (loaded|found \(Worker\))|No job functions found|Executed 'Functions\..*' \(Failed" "$RUNNER_TEMP/host-smoke.log"; then
-            echo "::error::The Functions host reported indexing or invocation errors"
-            exit 1
-          fi
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: host-evidence-py${{ matrix.python-version }}
+          path: test-results/**
 
   # azure-functions 2.x compatibility proof lane (issues #488 / #528),
   # following the sibling proof-lane pattern (azure-functions-validation #351 /
@@ -697,4 +648,11 @@ jobs:
         # `FunctionBuilder._function` private-attribute tripwires also run,
         # proving the adapter surface holds under azure-functions 2.x.
         run: |
-          pytest tests/ --no-cov -q -o "pythonpath=."
+          python -m tools.verification test --no-cov -q -o "pythonpath=."
+
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: wheel-2x-evidence-py${{ matrix.python-version }}
+          path: test-results/**

.github/workflows/e2e-azure.yml

@@ -194,8 +194,7 @@ jobs:
 
       - name: Run e2e tests
         run: |
-          pytest tests/e2e -v -m e2e --no-cov \
-            --junitxml=e2e-report/junit.xml \
+          python -m tools.verification azure-e2e -v \
             --html=e2e-report/report.html --self-contained-html
         env:
           E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net
@@ -211,7 +210,7 @@ jobs:
           import sys
           import xml.etree.ElementTree as ET
 
-          root = ET.parse("e2e-report/junit.xml").getroot()
+          root = ET.parse("test-results/azure-e2e/junit.xml").getroot()
           suite = root if root.tag == "testsuite" else root.find("testsuite")
           tests = int(suite.get("tests", 0))
           skipped = int(suite.get("skipped", 0))
@@ -304,7 +303,9 @@ jobs:
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
-          path: e2e-report/
+          path: |
+            e2e-report/
+            test-results/**
           retention-days: 30
 
   cleanup:

.github/workflows/publish-pypi.yml

@@ -103,11 +103,18 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install Hatch
-        run: pip install hatch
+      - name: Install test dependencies on the selected interpreter
+        run: make install-ci
 
       - name: Run test suite
-        run: hatch run test
+        run: python -m tools.verification test
+
+      - name: Upload verification evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: release-library-evidence
+          path: test-results/**
 
   cookbook-smoke:
     name: Cookbook dogfood smoke (candidate wheel)

.github/workflows/security.yml

@@ -23,10 +23,8 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install Hatch and dependencies via Makefile
-        run: |
-          pip install hatch
-          make install
+      - name: Install dependencies on the selected interpreter
+        run: make install-ci
 
       - name: Run Bandit
         run: make security

.gitignore

@@ -18,6 +18,7 @@ env/
 .coverage
 coverage.xml
 htmlcov/
+test-results/
 
 # Build artifacts
 build/

AGENTS.md

@@ -190,10 +190,11 @@ This repository is **issue-based, not milestone-based**. Track and group work us
 - Before adding any organizational structure, check whether the repository already has an established convention. A category being empty or unused (zero milestones, no `epic:*` labels) is evidence to follow the existing pattern, not to introduce a new one.
 
 ## Validation
-- `make test`
-- `make lint`
-- `make typecheck`
-- `make build`
+- `make doctor`
+- `make check`
+- `make test` (offline, 95% coverage)
+- `make integration` (requires func + Azurite)
+- `make verify` (`make check-all` is an exact backward-compatible alias)
 
 ## Release Process
 
@@ -275,15 +276,19 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor
 | Purpose | Command |
 | --- | --- |
 | Environment setup | `make install` |
+| CI-selected interpreter setup | `make install-ci` |
+| Read-only prerequisite diagnostics | `make doctor` |
 | Format code | `make format` |
 | Check formatting (`src`, `tests`) | `make format-check` |
-| Lint | `make lint` |
-| Type check | `make typecheck` |
-| Tests | `make test` |
-| Coverage | `make cov` |
-| Full validation | `make check-all` |
-| Docs build | `make docs` |
-| Package build | `make build` |
+| Complete non-mutating quality gate | `make check` |
+| Offline tests with 95% coverage | `make test` |
+| Managed local host/emulator tests | `make integration` |
+| Full local validation | `make verify` (`make check-all` alias) |
+| Configured Azure candidate certification | `make e2e-azure` |
+| Render saved evidence | `make report` |
+| Generated artifact cleanup only | `make clean` |
+| Focused lint/types/coverage | `make lint`, `make typecheck`, `make cov` |
+| Focused docs/package/performance | `make docs`, `make build`, `make perf` |
 
 ## Compatibility Rules
 
@@ -297,7 +302,7 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor
 - Public APIs require tests.
 - Bug fixes require regression tests.
 - Representative and complex examples must remain smoke-tested.
-- `make check-all` is the minimum merge gate.
+- `make verify` is the minimum local merge gate; `make check-all` aliases it exactly.
 
 ## Commit Rules