ci: unify logging verification commands and reports

.github/workflows/ci-test.yml

@@ -91,22 +91,18 @@ jobs:
           cache-dependency-path: pyproject.toml
 
       - name: Install package with dev extras
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+        run: make install-ci
 
-      - name: Lint workflows (pin hygiene + release-gate canonical SHAs)
-        run: |
-          python tools/lint_release_workflows.py
-          python tools/lint_workflow_pins.py
+      - name: Set up Node.js for Mermaid policy checks
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "20"
 
-      - name: Hatch/matrix hygiene lint
-        run: python tools/lint_hatch_matrix.py
+      - name: Install Mermaid render checker
+        run: npm install -g @mermaid-js/mermaid-cli
 
-      - name: Ruff style check
-        run: |
-          ruff check src tests tools examples benchmarks scripts
-          ruff format --check src tests tools examples benchmarks scripts
+      - name: Complete non-mutating quality gate
+        run: make check
 
       - name: Ruff format check (changed Python files)
         if: github.event_name == 'pull_request'
@@ -115,8 +111,12 @@ jobs:
           PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: python tools/check_pr_format.py "$PR_BASE_SHA" "$PR_HEAD_SHA"
 
-      - name: Type check (mypy, strict)
-        run: mypy src tests
+      - name: Upload quality evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: quality-evidence
+          path: test-results/**
 
   test:
     name: Test (Py ${{ matrix.python-version }})
@@ -160,9 +160,7 @@ jobs:
           PY
 
       - name: Install package with dev extras (on the matrix interpreter)
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e ".[dev]"
+        run: make install-ci
 
       - name: Run tests on the matrix interpreter
         # Raw pytest on the interpreter provisioned above (NOT through the
@@ -174,26 +172,36 @@ jobs:
           IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' }}
         run: |
           if [ "$IS_COVERAGE_CELL" = "true" ]; then
-            pytest tests/
+            python -m tools.verification test
           else
-            pytest tests/ --no-cov
+            python -m tools.verification test --no-cov
           fi
 
       - name: Verify coverage output
         if: matrix.python-version == '3.11'
-        run: ls -lh coverage.xml
+        run: ls -lh test-results/test/coverage.xml
 
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.11' && github.actor != 'dependabot[bot]'
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage.xml
+          files: ./test-results/test/coverage.xml
           verbose: true
           fail_ci_if_error: false
         env:
           CODECOV_RETRIES: 3
 
+      - name: Record OTel instrumentation separately from ingestion
+        run: python -m tools.verification otel
+
+      - name: Upload offline and OTel evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: test-evidence-py${{ matrix.python-version }}
+          path: test-results/**
+
   minimum-dependencies:
     name: Minimum dependencies (Py 3.11)
     runs-on: ubuntu-latest
@@ -228,7 +236,14 @@ jobs:
           PY
 
       - name: Run full suite against minimum runtime dependencies
-        run: pytest tests/ --no-cov
+        run: python -m tools.verification test --no-cov
+
+      - name: Upload minimum-dependency evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: minimum-dependencies-evidence
+          path: test-results/**
 
   artifact-build:
     name: Build compatibility artifacts
@@ -371,28 +386,17 @@ jobs:
           npm install --global azurite
 
       - name: Install smoke app dependencies
-        working-directory: tests/host_smoke
-        run: pip install -r requirements.txt
+        run: make install-ci
 
-      - name: Start Azurite and Functions host
-        run: |
-          azurite --silent > /tmp/azurite.log 2>&1 &
-          cd tests/host_smoke
-          func start --python --port 7071 > /tmp/functions-host.log 2>&1 &
+      - name: Managed decorated-handler and correlation host verification
+        run: python -m tools.verification integration
 
-      - name: Call decorated handlers
-        run: |
-          for attempt in {1..30}; do
-            if response="$(curl --fail --silent http://127.0.0.1:7071/api/smoke)" \
-              && default_response="$(curl --fail --silent http://127.0.0.1:7071/api/smoke-default)"; then
-              test "$response" = "smoke"
-              test "$default_response" = "smoke_default"
-              exit 0
-            fi
-            sleep 2
-          done
-          cat /tmp/functions-host.log
-          exit 1
+      - name: Upload local host evidence (not App Insights certification)
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: host-evidence-py${{ matrix.python-version }}
+          path: test-results/**
 
   docs-check:
     runs-on: ubuntu-latest
@@ -421,7 +425,14 @@ jobs:
           npm install -g @mermaid-js/mermaid-cli
 
       - name: Build documentation strictly
-        run: mkdocs build --strict
+        run: make docs
+
+      - name: Upload documentation evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: docs-evidence
+          path: test-results/**
 
       - name: Lint documentation links
         run: python tools/lint_doc_links.py

.github/workflows/e2e-azure.yml

@@ -173,7 +173,7 @@ jobs:
           func azure functionapp publish "${{ steps.names.outputs.app }}" --python --build remote
 
       - name: Run e2e tests (HTTP + App Insights)
-        run: pytest -v -m e2e tests/e2e/ --no-cov --html=e2e-report/report.html --self-contained-html
+        run: python -m tools.verification azure-e2e -v --html=test-results/azure-e2e/report.html --self-contained-html
         env:
           E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net
           E2E_EXPECTED_VERSION: ${{ env.RESOLVED_VERSION }}
@@ -218,7 +218,7 @@ jobs:
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
-          path: e2e-report/
+          path: test-results/**
           retention-days: 30
 
   cleanup:

.github/workflows/e2e-host-matrix.yml

@@ -1,20 +1,7 @@
 name: e2e-host-matrix
 
-# Scheduled Core Tools / host version matrix smoke (#394).
-#
-# Boots the local ``examples/e2e_app`` under a real ``func`` host + Azurite
-# across multiple Azure Functions Core Tools versions and asserts the host
-# starts and serves HTTP. This catches host-behavior drift across Core
-# Tools / host releases early, WITHOUT deploying to Azure (no cloud, no
-# secrets). It is intentionally **scheduled-only** (plus manual dispatch) so
-# it never blocks a PR and its CI cost stays bounded.
-#
-# NOTE: this is a *host-boot* smoke — it proves the app registers and serves
-# under each Core Tools version. It is not a substitute for the real-Azure
-# certification in e2e-azure.yml.
 on:
   schedule:
-    # Weekly, Mondays 05:00 UTC. Off the PR path.
     - cron: "0 5 * * 1"
   workflow_dispatch:
 
@@ -26,19 +13,8 @@ jobs:
     name: Host-boot smoke (func ${{ matrix.core_tools }})
     runs-on: ubuntu-latest
     strategy:
-      # Do not cancel the whole matrix when one Core Tools version fails —
-      # each entry is an independent drift signal.
       fail-fast: false
       matrix:
-        # Chosen set (documented per the issue acceptance checklist):
-        #   * "4"      — the latest published 4.x Core Tools (drift detector:
-        #                surfaces breakage in the newest release before it is
-        #                pinned anywhere).
-        #   * "4.12.0" — the version currently pinned by e2e-azure.yml and
-        #                cookbook-host-smoke (control / known-good baseline).
-        # Both are guaranteed installable via npm. Expand this list with
-        # additional explicit versions only after confirming they install and
-        # boot cleanly here.
         core_tools: ["4", "4.12.0"]
     steps:
       - name: Checkout repository
@@ -54,104 +30,21 @@ jobs:
         with:
           node-version: "22"
 
-      - name: Install Azure Functions Core Tools + Azurite
-        run: |
-          npm install -g "azure-functions-core-tools@${{ matrix.core_tools }}" azurite
-          echo "func version: $(func --version)"
-          echo "azurite version: $(azurite --version)"
-
-      - name: Install the library and the app's runtime deps
-        run: |
-          python -m pip install --upgrade pip
-          # The worker that `func start` launches uses this Python env, so the
-          # app's import of azure_functions_logging must resolve here.
-          pip install -e .
-          pip install -r examples/e2e_app/requirements.txt
+      - name: Install Azure Functions Core Tools and Azurite
+        run: npm install -g "azure-functions-core-tools@${{ matrix.core_tools }}" azurite
 
-      - name: Start Azurite
+      - name: Install selected-interpreter development and app dependencies
         run: |
-          mkdir -p /tmp/azurite
-          azurite --silent --location /tmp/azurite \
-            --blobHost 127.0.0.1 --queueHost 127.0.0.1 --tableHost 127.0.0.1 &
-          echo "AZURITE_PID=$!" >> "$GITHUB_ENV"
-
-      - name: Boot func host and smoke the HTTP endpoints
-        working-directory: examples/e2e_app
-        env:
-          AzureWebJobsStorage: UseDevelopmentStorage=true
-          FUNCTIONS_WORKER_RUNTIME: python
-        run: |
-          set -euo pipefail
-          # Launch the host in the background, capturing logs for diagnostics.
-          func start --port 7071 --python > "$RUNNER_TEMP/func-host.log" 2>&1 &
-          echo "FUNC_PID=$!" >> "$GITHUB_ENV"
-
-          # Poll for readiness (host + worker registration can take a while on
-          # a cold runner). Fail loudly with the host log if it never comes up.
-          ready=""
-          for i in $(seq 1 60); do
-            if curl -fsS "http://127.0.0.1:7071/api/health" >/dev/null 2>&1; then
-              ready="yes"
-              break
-            fi
-            sleep 5
-          done
-          if [ -z "$ready" ]; then
-            echo "::error ::func host did not become ready within timeout (Core Tools ${{ matrix.core_tools }})"
-            echo "----- func host log -----"
-            cat "$RUNNER_TEMP/func-host.log" || true
-            exit 1
-          fi
-
-          echo "Host is up. Asserting /api/health payload..."
-          HEALTH="$(curl -fsS http://127.0.0.1:7071/api/health)"
-          echo "health response: $HEALTH"
-          echo "$HEALTH" | grep -q '"status": "ok"'
+          make install-ci
+          python -m pip install -r examples/e2e_app/requirements.txt
 
-          echo "Asserting /api/version reports the installed package..."
-          VERSION="$(curl -fsS http://127.0.0.1:7071/api/version)"
-          echo "version response: $VERSION"
-          echo "$VERSION" | grep -q '"version"'
-
-          echo "Exercising /api/correlation to certify correlation claims..."
-          CORR="$(curl -fsS http://127.0.0.1:7071/api/correlation)"
-          echo "correlation response: $CORR"
-          printf '%s' "$CORR" > "$RUNNER_TEMP/correlation-response.json"
-
-          # Worker log records reach the host log asynchronously; wait until the
-          # three in-context markers have flushed (or fail loudly with the log).
-          markers=""
-          for i in $(seq 1 24); do
-            if grep -q 'corr-main-1' "$RUNNER_TEMP/func-host.log" \
-               && grep -q 'corr-main-2' "$RUNNER_TEMP/func-host.log" \
-               && grep -q 'corr-thread-propagated' "$RUNNER_TEMP/func-host.log"; then
-              markers="yes"
-              break
-            fi
-            sleep 5
-          done
-          if [ -z "$markers" ]; then
-            echo "::error ::correlation markers never appeared in the host log (Core Tools ${{ matrix.core_tools }})"
-            cat "$RUNNER_TEMP/func-host.log" || true
-            exit 1
-          fi
-
-          echo "Asserting correlation claims against the host log and HTTP response..."
-          python ../../scripts/assert_correlation.py \
-            "$RUNNER_TEMP/func-host.log" \
-            "$RUNNER_TEMP/correlation-response.json"
-
-      - name: Stop background processes
-        if: always()
-        run: |
-          [ -n "${FUNC_PID:-}" ] && kill "${FUNC_PID}" 2>/dev/null || true
-          [ -n "${AZURITE_PID:-}" ] && kill "${AZURITE_PID}" 2>/dev/null || true
+      - name: Managed host HTTP and correlation checks (not cloud ingestion)
+        run: python -m tools.verification host-matrix
 
-      - name: Upload func host log
+      - name: Upload redacted host evidence
         if: always()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: func-host-log-${{ matrix.core_tools }}
-          path: ${{ runner.temp }}/func-host.log
-          if-no-files-found: ignore
+          path: test-results/**
           retention-days: 14

.github/workflows/otel-nightly.yml

@@ -39,13 +39,11 @@ jobs:
           pip show opentelemetry-api opentelemetry-sdk | grep -E "^(Name|Version):"
 
       - name: Run OTel-focused test suite
-        run: |
-          # This canary runs a four-file subset, but pyproject's addopts force
-          # --cov over the whole package and fail_under=95 applies. Coverage of
-          # a subset is meaningless here and was failing the job at ~49%,
-          # masking the upstream breakage this canary exists to detect.
-          python -m pytest -q --cov-fail-under=0 \
-            tests/test_otel.py \
-            tests/test_otel_contract.py \
-            tests/test_otel_filters.py \
-            tests/test_host_config_otel.py
+        run: python -m tools.verification otel
+
+      - name: Upload OTel instrumentation evidence (not ingestion certification)
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: otel-latest-evidence
+          path: test-results/**

.github/workflows/publish-pypi.yml

@@ -101,11 +101,18 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Install Hatch
-        run: pip install hatch
+      - name: Install test dependencies on the selected interpreter
+        run: make install-ci
 
       - name: Run test suite
-        run: hatch run test
+        run: python -m tools.verification test
+
+      - name: Upload release library evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: release-library-evidence
+          path: test-results/**
 
   cookbook-smoke:
     name: Cookbook dogfood smoke (candidate wheel)
@@ -278,7 +285,7 @@ jobs:
           path: dist
 
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1 # pin-exempt: PyPA official publisher (trusted-publisher OIDC); tracks release/v1 by design
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
 
   notify-cookbook:
     name: Trigger cookbook e2e re-verification