ci: unify validation commands and test reports

.github/workflows/ci-test.yml

@@ -93,28 +93,17 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: Install package with dev extras
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
-
-      - name: Check endpoint schema hash pin
-        run: pytest --no-cov tests/test_endpoint_schema.py::TestSchemaHashPin
-
-      - name: Lint workflows (pin hygiene + release-gate canonical SHAs)
-        run: |
-          python tools/lint_release_workflows.py
-          python tools/lint_workflow_pins.py
-
-      - name: Hatch/matrix hygiene lint
-        run: python tools/lint_hatch_matrix.py
+        run: make install-ci
 
-      - name: Ruff style check
-        run: |
-          ruff check src tests
-          ruff format --check src tests
+      - name: Complete non-mutating quality and contract gate
+        run: make check
 
-      - name: Type check (mypy, strict)
-        run: mypy src tests
+      - name: Upload quality evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: quality-evidence
+          path: test-results/**
 
   test:
     name: Test (Py ${{ matrix.python-version }})
@@ -157,9 +146,7 @@ jobs:
           PY
 
       - name: Install package with dev extras (on the matrix interpreter)
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+        run: make install-ci
 
       - name: Run tests on the matrix interpreter
         # Raw pytest on the interpreter provisioned above (NOT through the
@@ -171,26 +158,36 @@ jobs:
           IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' }}
         run: |
           if [ "$IS_COVERAGE_CELL" = "true" ]; then
-            pytest tests/
+            python -m tools.verification test
           else
-            pytest tests/ --no-cov
+            python -m tools.verification test --no-cov
           fi
 
       - name: Verify coverage output
         if: matrix.python-version == '3.11'
-        run: ls -lh coverage.xml
+        run: ls -lh test-results/test/coverage.xml
 
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.11' && github.actor != 'dependabot[bot]'
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage.xml
+          files: ./test-results/test/coverage.xml
           verbose: true
           fail_ci_if_error: false
         env:
           CODECOV_RETRIES: 3
 
+      - name: Required worker signature and registration evidence
+        run: python -m tools.verification worker-compat
+
+      - name: Upload test and worker evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: test-evidence-py${{ matrix.python-version }}
+          path: test-results/**
+
   docs-check:
     name: Documentation checks
     needs: changes
@@ -211,7 +208,14 @@ jobs:
         run: pip install -e ".[docs]" pytest pytest-cov PyYAML
 
       - name: Build docs (strict)
-        run: mkdocs build --strict
+        run: make docs
+
+      - name: Upload docs evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: docs-evidence
+          path: test-results/**
 
       - name: Check README examples and screenshot manifest
         run: |
@@ -297,7 +301,14 @@ jobs:
           PY
 
       - name: Run full suite against minimum runtime dependencies
-        run: pytest tests/ --no-cov
+        run: python -m tools.verification test --no-cov
+
+      - name: Upload minimum-dependency evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: minimum-dependencies-evidence
+          path: test-results/**
 
   artifact-build:
     name: Build compatibility artifacts
@@ -444,26 +455,17 @@ jobs:
           npm install --global azurite
 
       - name: Install smoke app dependencies
-        working-directory: tests/host_smoke
-        run: pip install -r requirements.txt
+        run: make install-ci
 
-      - name: Start Azurite and Functions host
-        run: |
-          azurite --silent > /tmp/azurite.log 2>&1 &
-          cd tests/host_smoke
-          func start --python --port 7071 > /tmp/functions-host.log 2>&1 &
+      - name: Managed real Functions host and worker signature smoke
+        run: python -m tools.verification integration
 
-      - name: Call validated handler
-        run: |
-          for attempt in {1..30}; do
-            if response="$(curl --fail --silent http://127.0.0.1:7071/api/smoke)"; then
-              python -c 'import json, sys; assert json.loads(sys.argv[1]) == {"function_name": "smoke"}' "$response"
-              exit 0
-            fi
-            sleep 2
-          done
-          cat /tmp/functions-host.log
-          exit 1
+      - name: Upload host and worker evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: host-evidence-py${{ matrix.python-version }}
+          path: test-results/**
 
   # azure-functions 2.x compatibility proof lane (issue #351), modeled on
   # langgraph d433b63. This is a SINGLE representative 2.x lane, not a matrix:
@@ -528,6 +530,11 @@ jobs:
         # pythonpath so the installed WHEEL is exercised, not the source tree.
         env:
           AZFUNC_2X_SPIKE: "1"
-        run: |
-          pytest -m compat2x -o addopts= -o pythonpath= \
-            tests/test_worker_compat_2x_spike.py -v
+        run: python -m tools.verification sdk-compat -v
+
+      - name: Upload SDK compatibility spike evidence (not Azure certification)
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: sdk-2x-evidence
+          path: test-results/**

.github/workflows/e2e-azure.yml

@@ -218,7 +218,7 @@ jobs:
           echo "=== Live logstream (20s) ==="
           timeout 20 az webapp log tail --name "$APP" --resource-group "$RG" 2>&1 | head -200 || true
       - name: Run e2e tests
-        run: pytest -v -m e2e tests/e2e/ --no-cov --html=e2e-report/report.html --self-contained-html
+        run: python -m tools.verification azure-e2e -v --html=test-results/azure-e2e/report.html --self-contained-html
         env:
           E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net
 
@@ -287,7 +287,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
-          path: e2e-report/
+          path: test-results/**
           retention-days: 30
 
   cleanup:

.github/workflows/publish-pypi.yml

@@ -108,11 +108,18 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install Hatch
-        run: pip install hatch
+      - name: Install test dependencies on the selected interpreter
+        run: make install-ci
 
       - name: Run test suite
-        run: hatch run test
+        run: python -m tools.verification test
+
+      - name: Upload release library evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: release-library-evidence
+          path: test-results/**
 
   cookbook-smoke:
     name: Cookbook dogfood smoke (candidate wheel)

.github/workflows/security.yml

@@ -23,10 +23,8 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install Hatch and dependencies via Makefile
-        run: |
-          pip install hatch
-          make install
+      - name: Install dependencies on the selected interpreter
+        run: make install-ci
 
       - name: Run Bandit
         run: make security

.github/workflows/worker-nightly.yml

@@ -61,9 +61,14 @@ jobs:
           python -c "import azure.functions as f; print('azure-functions', f.__version__)"
 
       - name: Run WorkerCompat suite against pre-release worker library
-        run: |
-          python -m pytest tests/test_decorator.py tests/test_integration.py \
-            -v --no-header -p no:cacheprovider --no-cov
+        run: python -m tools.verification worker-compat -v --no-header -p no:cacheprovider
+
+      - name: Upload pre-release worker evidence (not Azure certification)
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: worker-prerelease-py${{ matrix.python-version }}
+          path: test-results/**
 
       - name: Explain a WorkerCompat break
         if: failure()

.gitignore

@@ -15,6 +15,7 @@ env/
 .coverage
 coverage.xml
 htmlcov/
+test-results/
 
 # Build artifacts
 build/

AGENTS.md

@@ -18,7 +18,7 @@
 
 ### Test Coverage
 - Maintain test coverage at **95% or above** for committed changes and PRs.
-- Run `hatch run pytest --cov --cov-report=term-missing -q` to verify before submitting changes.
+- Run `make test` to verify the offline suite and coverage before submitting changes.
 - Any PR that drops coverage below 95% must include additional tests to compensate.
 - Runtime code must remain compatible with Python 3.11-3.14 (`requires-python = ">=3.11,<3.15"`).
 - Public APIs must be fully typed.
@@ -106,10 +106,11 @@ This repository is **issue-based, not milestone-based**. Track and group work us
 - Before adding any organizational structure, check whether the repository already has an established convention. A category being empty or unused (zero milestones, no `epic:*` labels) is evidence to follow the existing pattern, not to introduce a new one.
 
 ## Validation
-- `make test`
-- `make lint`
-- `make typecheck`
-- `make build`
+- `make doctor` (read-only prerequisites, actual interpreter)
+- `make check` (complete non-mutating quality and schema/metadata contracts)
+- `make test` (offline, coverage >=95%)
+- `make integration` (worker signatures and real host/private Azurite)
+- `make verify` (`make check-all` is an exact alias)
 
 ## Release Process
 
@@ -186,22 +187,31 @@ Use Makefile entry points only. Do not bypass the Makefile in CI or contributor
 | Purpose | Command |
 | --- | --- |
 | Environment setup | `make install` |
+| CI-selected interpreter setup | `make install-ci` |
+| Read-only prerequisite diagnostics | `make doctor` |
 | Format code | `make format` |
 | Check formatting (`src`, `tests`) | `make format-check` |
-| Lint | `make lint` |
-| Type check | `make typecheck` |
-| Tests | `make test` |
-| Coverage | `make cov` |
-| Full validation | `make check-all` |
-| Docs build | `make docs` |
-| Package build | `make build` |
+| Complete non-mutating quality/contracts | `make check` |
+| Offline tests, coverage >=95% | `make test` |
+| Managed worker/host/emulator lanes | `make integration` |
+| Full local validation | `make verify` (`make check-all` exact alias) |
+| Configured real Azure certification | `make e2e-azure` |
+| Render saved evidence only | `make report` |
+| Generated artifacts only | `make clean` |
+| Partial/focused lint/types/coverage | `make lint`, `make typecheck`, `make cov` |
+| Partial/focused docs/package/wheel | `make docs`, `make build`, `make wheel-smoke` |
+
+See CONTRIBUTING.md for evidence/status semantics. Required worker and host lanes
+have separate summaries under gitignored `test-results/`. SDK spikes are not Azure
+certification. Verification must never rewrite schema/hash pins. `clean-all` is
+maintenance-only deep cleanup, including bytecode and the project-local `.venv`.
 
 ## Testing Rules
 
 - Public APIs require tests.
 - Bug fixes require regression tests.
 - Representative and complex examples must remain smoke-tested.
-- `make check-all` is the minimum merge gate.
+- `make verify` is the minimum local merge gate; `make check-all` aliases it exactly.
 
 ## Commit Rules