diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index ae2d466..ed6f020 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -481,6 +481,21 @@ jobs: cat /tmp/functions-host.log exit 1 + - name: Prove candidate wheel behavior and broken-wrapper rejection + env: + HOST_PYTHON: python + run: | + pip install hatch + make build + make host-candidate CANDIDATE_WHEEL="$(realpath dist/*.whl)" + + - name: Upload candidate host evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: candidate-host-${{ matrix.python-version }} + path: test-results/candidate-host/ + # azure-functions 2.x compatibility proof lane (issue #351), modeled on # langgraph d433b63. This is a SINGLE representative 2.x lane, not a matrix: # it builds the wheel, installs it, overrides the pyproject cap to pull diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index f513d85..cf2d302 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -19,12 +19,8 @@ permissions: # Tiered runtime verification before any PyPI upload: # * lib-tests : the library's own unit suite (against the built wheel). # * cookbook-smoke : downstream module-import checks (catches registration/import drift). -# * cookbook-host-smoke : install the candidate wheel into the cookbook, then boot a real func -# host + Azurite and run the cookbook HTTP e2e. This proves the candidate -# installs cleanly and the Functions host starts with it present in the -# environment. NOTE: the cookbook HTTP examples do not import this package, -# so this tier is a host-boot smoke, NOT proof of this package's own runtime -# behavior. Package-native runtime coverage is tracked separately. +# * cookbook-host-smoke : retained ID; repository-local wheel-installed validation behavior +# on a real Functions host, including a broken-wrapper negative control. # * azure-e2e : calls e2e-azure.yml to deploy to real Azure and run the live e2e suite # against the tag being published. Because it runs inside this chain at # the same ref, certification covers the exact published commit by @@ -162,7 +158,7 @@ jobs: run: hatch run smoke cookbook-host-smoke: - name: Cookbook host-boot smoke (func + Azurite, candidate wheel) + name: Validation candidate host behavior and negative control needs: build runs-on: ubuntu-latest env: @@ -170,11 +166,10 @@ jobs: # documented v4 install path (see e2e-azure.yml for rationale). FUNCTIONS_CORE_TOOLS_VERSION: "4.12.0" steps: - - name: Checkout cookbook (downstream consumer) + - name: Checkout candidate host fixture uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: yeongseon/azure-functions-cookbook-python - ref: main + ref: ${{ github.ref }} - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -198,31 +193,16 @@ jobs: name: dist path: candidate-dist - - name: Install cookbook environment - run: | - pip install hatch - make install - - - name: Install candidate wheel over the PyPI floor - run: | - WHEEL=$(ls candidate-dist/*.whl | head -n1) - echo "Installing candidate wheel: $WHEEL" - hatch run pip install --force-reinstall --no-deps "$WHEEL" + - name: Execute candidate validation and broken-wrapper control + run: make host-candidate CANDIDATE_WHEEL="$(realpath candidate-dist/*.whl)" + timeout-minutes: 5 - - name: Verify candidate version is under test - run: | - EXPECTED="${{ needs.build.outputs.version }}" - ACTUAL=$(hatch run python -c "import importlib.metadata as m; print(m.version('azure-functions-validation'))") - echo "Expected candidate version: $EXPECTED" - echo "Installed version: $ACTUAL" - if [ "$EXPECTED" != "$ACTUAL" ]; then - echo "::error ::Cookbook is not exercising the candidate wheel ($ACTUAL != $EXPECTED)" - exit 1 - fi - - - name: Boot cookbook func host with candidate wheel (host-smoke) - run: hatch run e2e tests/e2e/test_http_examples.py - timeout-minutes: 15 + - name: Upload candidate host evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: candidate-host-evidence + path: test-results/candidate-host/ azure-e2e: name: Azure release certification diff --git a/AGENTS.md b/AGENTS.md index 2c78544..94b8f32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -184,7 +184,7 @@ Every tier runs inside `publish-pypi.yml` on the tag, and **all of them gate the | `build` | tag/`__version__` mismatch; produces the one artifact that is later uploaded | | `lib-tests` | library unit regressions | | `cookbook-smoke` | downstream import/registration drift (0.21.0 class) | -| `cookbook-host-smoke` | candidate wheel installs cleanly and a real `func` host + Azurite boots with it present, no cloud. NOTE: the cookbook HTTP examples do not import this package, so this is a host-boot smoke, **not** proof of this package's own runtime behavior (tracked separately) | +| `cookbook-host-smoke` | retained job ID; repository-local candidate-wheel fixture proves indexing, custom trigger binding, valid/invalid JSON, response validation and subclass-field filtering on a real `func` host; a broken-wrapper negative control must fail indexing | | `azure-e2e` | cloud-only drift — deploys to real Azure, runs the live e2e suite, uploads an `azure-cert` record | | `publish` | uploads the exact artifact `build` produced; it never rebuilds | @@ -194,7 +194,7 @@ Reproduce the automated gate locally before merging the Release PR, so failures - Build the candidate: `make build` (produces `dist/*.whl`). - In [`azure-functions-cookbook-python`](https://github.com/yeongseon/azure-functions-cookbook-python): `make install`, then install the candidate over the PyPI floor with `hatch run pip install --force-reinstall --no-deps `, confirm `importlib.metadata.version("azure-functions-validation")` equals the tag, and run `hatch run smoke`. -- Reproduce the host-boot smoke tier: in the cookbook, with the candidate wheel installed, run `hatch run e2e tests/e2e/test_http_examples.py` (starts a real `func` host + Azurite). This confirms the candidate installs and the host boots with it present; it is the local stand-in for `cookbook-host-smoke` and must pass before tagging. NOTE: these HTTP examples do not import this package, so this is not a decorator runtime test — rely on `lib-tests` for `@validate_http` behavior. +- Reproduce the candidate host tier here with `make host-candidate CANDIDATE_WHEEL="$(realpath dist/*.whl)"`. It proves decorator runtime behavior and rejects a deliberately broken wrapper; see [candidate host verification](docs/candidate-host.md). This is not deployed Azure certification. - Treat any new `RuntimeWarning`/`DeprecationWarning` from `@validate_http` as release-blocking — the library surfaces decorator-order and API-drift problems as warnings, so a clean run (zero validation warnings) is part of the gate. - If the cookbook pins a lower bound (`azure-functions-validation>=X.Y,<1`), bump it to the new minor in the same release PR so examples are tested against the version they advertise. diff --git a/Makefile b/Makefile index 512634d..e99e7b2 100644 --- a/Makefile +++ b/Makefile @@ -149,6 +149,10 @@ e2e-azure: ensure-hatch build: ensure-hatch @$(HATCH) build +.PHONY: host-candidate +host-candidate: + @bash scripts/check_candidate_host.sh "$(CANDIDATE_WHEEL)" + .PHONY: publish-test publish-test: ensure-hatch @$(HATCH) publish --repo test diff --git a/docs/candidate-host.md b/docs/candidate-host.md new file mode 100644 index 0000000..16b1049 --- /dev/null +++ b/docs/candidate-host.md @@ -0,0 +1,32 @@ +# Candidate-wheel host verification + +Build and exercise the wheel locally on Python 3.11 with Functions Core Tools v4: + +```bash +make build +make host-candidate CANDIDATE_WHEEL="$(realpath dist/*.whl)" +``` + +This supersedes the historical Cookbook HTTP host-boot instructions in the release +guide. The release job retains the ID `cookbook-host-smoke` for gate compatibility, +but checks out this repository and executes its own HTTP-only fixture. Cookbook +import smoke remains a separate downstream integration gate. Real Azure candidate +certification remains mandatory and separate; local success is not Azure evidence. + +The runner installs the candidate in a fresh non-editable virtual environment, +checks distribution version against wheel metadata and import origin under that +environment, and starts a real Functions host outside the checkout. It uses a +private directory and dynamically selected loopback ports, never shared services. +It checks default request/context binding, custom `trigger_arg_name`, typed body +injection, success serialization without subclass-only fields, malformed JSON 400, +invalid-model 422 and sanitized invalid-response 500. + +A second host uses the same installed candidate but deliberately replaces the +fixture's decorator with a no-op. The same behavior probe must fail, with the +worker reporting a return-annotation indexing error. A timeout or arbitrary +process failure alone is not accepted as the negative-control proof. + +`test-results/candidate-host/` retains import identity, synthetic HTTP responses +and host logs. CI uploads these even on failure. The fixture contains no real +credentials or user payloads. The runner fails rather than skips when tools, +installation, identity, indexing or behavior checks fail. diff --git a/scripts/check_candidate_host.sh b/scripts/check_candidate_host.sh new file mode 100644 index 0000000..1056b9c --- /dev/null +++ b/scripts/check_candidate_host.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail +wheel=$(realpath "${1:?candidate wheel required}") +root=$(pwd) +output="$root/test-results/candidate-host" +mkdir -p "$output" +command -v func >/dev/null +command -v curl >/dev/null +scratch=$(mktemp -d) +host_pid="" +cleanup() { + if [ -n "$host_pid" ]; then kill -KILL -- "-$host_pid" 2>/dev/null || true; wait "$host_pid" 2>/dev/null || true; fi + rm -rf "$scratch" +} +trap cleanup EXIT +"${HOST_PYTHON:-python3.11}" -m venv "$scratch/venv" +"$scratch/venv/bin/python" -m pip install --disable-pip-version-check "$wheel" +cp tests/host_smoke/function_app.py "$scratch/function_app.py" +printf '{"version":"2.0"}\n' > "$scratch/host.json" +"$scratch/venv/bin/python" -I - "$wheel" > "$output/identity.txt" <<'PY' +import importlib.metadata +from pathlib import Path +import sys +import zipfile +import azure_functions_validation as package +with zipfile.ZipFile(sys.argv[1]) as archive: + metadata = archive.read(next(n for n in archive.namelist() if n.endswith('.dist-info/METADATA'))).decode() +expected = next(line.removeprefix('Version: ') for line in metadata.splitlines() if line.startswith('Version: ')) +assert importlib.metadata.version('azure-functions-validation') == expected +assert Path(package.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()) +print(expected, package.__file__) +PY +run_host() { + local mode="$1" + port=$("$scratch/venv/bin/python" -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()') + (cd "$scratch"; exec env PATH="$scratch/venv/bin:$PATH" VIRTUAL_ENV="$scratch/venv" \ + FUNCTIONS_WORKER_RUNTIME=python AzureWebJobsStorage="" BROKEN_VALIDATION_WRAPPER="$mode" \ + setsid func start --python --port "$port") > "$output/host-$mode.log" 2>&1 & + host_pid=$! +} +probe() { + local attempt status + for attempt in {1..45}; do + if ! kill -0 "$host_pid" 2>/dev/null; then return 1; fi + if curl --max-time 3 --silent --fail "http://127.0.0.1:$port/api/smoke" > "$output/smoke.json"; then break; fi + sleep 1 + done + [ "$(< "$output/smoke.json")" = '{"function_name":"smoke"}' ] || return 1 + status=$(curl --silent -o "$output/valid.json" -w '%{http_code}' \ + -H 'Content-Type: application/json' -d '{"name":"candidate"}' "http://127.0.0.1:$port/api/validated") + [ "$status" = 200 ] && [ "$(< "$output/valid.json")" = '{"function_name":"candidate"}' ] || return 1 + status=$(curl --silent -o "$output/malformed.json" -w '%{http_code}' \ + -H 'Content-Type: application/json' -d 'broken' "http://127.0.0.1:$port/api/validated") + [ "$status" = 400 ] || return 1 + status=$(curl --silent -o "$output/invalid.json" -w '%{http_code}' \ + -H 'Content-Type: application/json' -d '{}' "http://127.0.0.1:$port/api/validated") + [ "$status" = 422 ] || return 1 + status=$(curl --silent -o "$output/response.json" -w '%{http_code}' "http://127.0.0.1:$port/api/invalid-response") + [ "$status" = 500 ] || return 1 + ! grep -q 'must not leak' "$output/response.json" +} +run_host 0 +probe +kill -KILL -- "-$host_pid"; wait "$host_pid" || true +host_pid="" +run_host 1 +if probe; then + echo 'Broken wrapper unexpectedly passed the host behavior proof' >&2 + exit 1 +fi +grep -q 'Worker failed to index functions' "$output/host-1.log" +grep -q 'InternalResponse.*does not match binding type' "$output/host-1.log" +echo 'Candidate host behavior passed; broken-wrapper negative control failed as required.' diff --git a/tests/host_smoke/function_app.py b/tests/host_smoke/function_app.py index 7879d06..0232956 100644 --- a/tests/host_smoke/function_app.py +++ b/tests/host_smoke/function_app.py @@ -1,3 +1,5 @@ +import os + import azure.functions as func from pydantic import BaseModel @@ -14,7 +16,29 @@ class InternalResponse(PublicResponse): secret: str +class InputBody(BaseModel): + name: str + + +if os.environ.get("BROKEN_VALIDATION_WRAPPER") == "1": + + def validate_http(**options): + return lambda handler: handler + + @app.route(route="smoke", auth_level=func.AuthLevel.ANONYMOUS) @validate_http(response_model=PublicResponse) def smoke(req: func.HttpRequest, context: func.Context) -> InternalResponse: return InternalResponse(function_name=context.function_name, secret="hidden") + + +@app.route(route="validated", methods=["POST"], trigger_arg_name="request") +@validate_http(body=InputBody, response_model=PublicResponse) +def validated(request: func.HttpRequest, body: InputBody) -> InternalResponse: + return InternalResponse(function_name=body.name, secret="hidden") + + +@app.route(route="invalid-response") +@validate_http(response_model=PublicResponse) +def invalid_response(req: func.HttpRequest) -> dict: + return {"unexpected": "must not leak"} diff --git a/tests/test_host_release_contract.py b/tests/test_host_release_contract.py new file mode 100644 index 0000000..05242a2 --- /dev/null +++ b/tests/test_host_release_contract.py @@ -0,0 +1,13 @@ +from pathlib import Path + +import yaml + + +def test_release_host_gate_executes_candidate_behavior_and_negative_control() -> None: + root = Path(__file__).resolve().parents[1] + workflow = yaml.safe_load((root / ".github/workflows/publish-pypi.yml").read_text()) + job = workflow["jobs"]["cookbook-host-smoke"] + commands = "\n".join(step.get("run", "") for step in job["steps"]) + assert "make host-candidate" in commands + assert any(step.get("if") == "always()" for step in job["steps"]) + assert "cookbook-host-smoke" in workflow["jobs"]["publish"]["needs"]