From 9892fb120d13bb5bf5af78752eeb9cb72c2bfbba Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Sat, 24 Oct 2026 14:02:05 +0900 Subject: [PATCH 1/2] test(host-smoke): select the real-host route tests with a host_smoke marker Request every route the fixture registers through register_openapi_routes (JSON, YAML and docs; anonymous and protected) on a running host and assert 200 plus a body marker. The real-host tests are now selected by a host_smoke marker instead of a HOST_SMOKE_BASE_URL skipif, so they fail rather than skip when no host is given, and pyproject addopts deselect them from the offline suite (0 new skips). Refs #824 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- pyproject.toml | 3 ++- tests/host_smoke/test_fixture_routes.py | 31 ++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0417ba40..a241bf36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,12 +142,13 @@ pretty = true module = "examples.*" follow_imports = "skip" [tool.pytest.ini_options] -addopts = "--cov=src/azure_functions_openapi --cov-report=xml --cov-report=term-missing -ra -q -m 'not e2e and not perf and not browser'" +addopts = "--cov=src/azure_functions_openapi --cov-report=xml --cov-report=term-missing -ra -q -m 'not e2e and not perf and not browser and not host_smoke'" testpaths = ["tests"] pythonpath = ["src", "."] markers = [ "e2e: real Azure end-to-end tests (require E2E_BASE_URL)", "perf: spec-generation regression budgets (run with make perf)", + "host_smoke: requests a running tests/host_smoke Functions host (set HOST_SMOKE_BASE_URL)", "browser: offline Chromium script checks (require cached Playwright 1.54.1 and Chromium)", "browser_cdn: real SRI-enforced Swagger UI CDN checks (skip when CDN network is unavailable)", ] diff --git a/tests/host_smoke/test_fixture_routes.py b/tests/host_smoke/test_fixture_routes.py index e109e207..fa15f8d5 100644 --- a/tests/host_smoke/test_fixture_routes.py +++ b/tests/host_smoke/test_fixture_routes.py @@ -14,6 +14,12 @@ Core Tools that the `host-smoke` CI job spins up. Reverting the fix (dropping app=app from get_openapi_json() in function_app.py) makes test_fixture_handler_documents_the_route_it_actually_serves fail. + +The tests marked `host_smoke` are the opposite: they request the routes of an +already running host (HOST_SMOKE_BASE_URL) and are deselected from the offline +run by pyproject.toml's addopts. The `host-smoke` CI job selects them with +`-m host_smoke`, and a missing HOST_SMOKE_BASE_URL fails them instead of +skipping them (#824). """ from __future__ import annotations @@ -107,10 +113,29 @@ def test_protected_docs_fixture_never_echoes_key(fixture_module: ModuleType) -> assert b'url: "/api/protected/openapi.json"' in response.get_body() -@pytest.mark.skipif( - not os.environ.get("HOST_SMOKE_BASE_URL"), - reason="Set HOST_SMOKE_BASE_URL to a running tests/host_smoke Functions host", +@pytest.mark.host_smoke +@pytest.mark.parametrize( + ("route", "needle"), + [ + ("registered/openapi.json", '"openapi"'), + ("registered/openapi.yaml", "openapi:"), + ("registered/docs", "SwaggerUIBundle"), + ("protected/openapi.json", '"openapi"'), + ("protected/openapi.yaml", "openapi:"), + ("protected/docs", "SwaggerUIBundle"), + ], ) +def test_real_host_serves_every_registered_route(route: str, needle: str) -> None: + # Given: the app indexed by the host (#814 acceptance: request each helper route) + base = os.environ["HOST_SMOKE_BASE_URL"] + # When + response = requests.get(f"{base}/api/{route}", timeout=10, allow_redirects=False) + # Then + assert response.status_code == 200 + assert needle in response.text + + +@pytest.mark.host_smoke def test_real_host_protected_docs_never_echoes_key() -> None: # Given: Core Tools does not enforce keys locally; this proves no-echo, NOT key enforcement. base = os.environ["HOST_SMOKE_BASE_URL"] From 6432b2ea78c019494e787d0e29d2cfa70f059368 Mon Sep 17 00:00:00 2001 From: Yeongseon Choe Date: Sat, 24 Oct 2026 14:02:15 +0900 Subject: [PATCH 2/2] ci(host-smoke): request every registered route on the real Functions host The host-smoke job only proved that the Functions host indexes the fixture and answered three hand-written routes, so a helper handler that fails when invoked passed CI. Install pytest, pytest-cov and requests into the isolated smoke venv and run the host_smoke tests against the live host before it is stopped. The pytest status is kept until the host log has been printed and scanned, and the step still fails with pytest's own exit code (5 for a wrong marker selector). A workflow hardening test keeps HOST_SMOKE_BASE_URL, the -m host_smoke selector and the status propagation wired, so the real-host tests cannot go dead again. Refs #824 Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- .github/workflows/ci-test.yml | 15 ++++++++++++++- tests/test_ci_workflow_hardening.py | 9 +++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 68aa942b..ac6db7ef 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -544,7 +544,7 @@ jobs: python -m venv tests/host_smoke/.venv tests/host_smoke/.venv/bin/python -m pip install --upgrade pip cd tests/host_smoke - .venv/bin/python -m pip install -r requirements.txt + .venv/bin/python -m pip install -r requirements.txt pytest pytest-cov requests - name: Exercise OpenAPI routes on a real Functions host run: | @@ -613,6 +613,14 @@ jobs: --output "$RUNNER_TEMP/swagger.html" grep -q 'SwaggerUIBundle' "$RUNNER_TEMP/swagger.html" + # Request every route register_openapi_routes registered (JSON, YAML, docs; + # anonymous and protected): indexing alone never proved they respond (#824). + # The status is kept so the host log is still printed and scanned before failing. + pytest_status=0 + (cd ../.. && HOST_SMOKE_BASE_URL=http://127.0.0.1:7071 \ + tests/host_smoke/.venv/bin/python -m pytest tests/host_smoke -m host_smoke --no-cov -rs) \ + || pytest_status=$? + kill "$host_pid" 2>/dev/null || true wait "$host_pid" 2>/dev/null || true cat "$RUNNER_TEMP/host-smoke.log" @@ -622,6 +630,11 @@ jobs: exit 1 fi + if [ "$pytest_status" -ne 0 ]; then + echo "::error::The real-host route tests failed (pytest exit $pytest_status)" + exit "$pytest_status" + fi + # azure-functions 2.x compatibility proof lane (issues #488 / #528), # following the sibling proof-lane pattern (azure-functions-validation #351 / # azure-functions-langgraph #350). It installs the package from a WHEEL, then diff --git a/tests/test_ci_workflow_hardening.py b/tests/test_ci_workflow_hardening.py index c8e1d277..119a21e9 100644 --- a/tests/test_ci_workflow_hardening.py +++ b/tests/test_ci_workflow_hardening.py @@ -92,3 +92,12 @@ def test_gate_accepts_a_successful_unknown_dependency() -> None: jobs["future-job"] = {"result": "success", "outputs": {}} assert run_gate(jobs).returncode == 0 + + +def test_host_smoke_job_runs_the_real_host_tests() -> None: + steps = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"]["host-smoke"]["steps"] + script = "\n".join(step.get("run", "") for step in steps) + + assert "HOST_SMOKE_BASE_URL=" in script + assert "-m host_smoke" in script + assert 'exit "$pytest_status"' in script