diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index ed6f020..9ec030a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -171,9 +171,9 @@ jobs: IS_COVERAGE_CELL: ${{ matrix.python-version == '3.11' }} run: | if [ "$IS_COVERAGE_CELL" = "true" ]; then - pytest tests/ --junitxml=test-results/junit.xml + make test-ci else - pytest tests/ --no-cov --junitxml=test-results/junit.xml + make test-ci TEST_ARGS=--no-cov fi - name: Upload test reports @@ -187,14 +187,14 @@ jobs: - name: Verify coverage output if: matrix.python-version == '3.11' - run: ls -lh coverage.xml + run: ls -lh test-results/coverage.xml - name: Upload coverage to Codecov if: matrix.python-version == '3.11' && github.actor != 'dependabot[bot]' uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage.xml + files: ./test-results/coverage.xml verbose: true fail_ci_if_error: false env: @@ -306,7 +306,7 @@ jobs: PY - name: Run full suite against minimum runtime dependencies - run: pytest tests/ --no-cov --junitxml=test-results/minimum-junit.xml + run: make test-ci TEST_ARGS=--no-cov - name: Upload minimum dependency reports if: always() @@ -563,10 +563,7 @@ jobs: AZFUNC_2X_SPIKE: "1" FAIL_ON_SKIP: "1" run: | - pytest -m compat2x -o addopts= -o pythonpath= \ - tests/test_worker_compat_2x_spike.py -v \ - --junitxml=test-results/compat2x-junit.xml \ - --cov=azure_functions_validation --cov-report=xml:coverage.xml --cov-fail-under=0 + make test-spike-ci - name: Upload SDK compatibility reports if: always() diff --git a/.github/workflows/e2e-azure.yml b/.github/workflows/e2e-azure.yml index 21e12b8..4e8eaee 100644 --- a/.github/workflows/e2e-azure.yml +++ b/.github/workflows/e2e-azure.yml @@ -218,7 +218,7 @@ jobs: echo "=== Live logstream (20s) ===" timeout 20 az webapp log tail --name "$APP" --resource-group "$RG" 2>&1 | head -200 || true - name: Run e2e tests - run: pytest -v -m e2e tests/e2e/ --no-cov --html=e2e-report/report.html --self-contained-html + run: make test-e2e-ci TEST_ARGS="--html=e2e-report/report.html --self-contained-html" env: E2E_BASE_URL: https://${{ steps.names.outputs.app }}.azurewebsites.net @@ -290,6 +290,13 @@ jobs: path: e2e-report/ retention-days: 30 + - name: Upload e2e JUnit + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: e2e-junit + path: test-results/ + cleanup: name: Cleanup Azure Resources runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bee73e8..c7cc667 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,13 @@ make build `make check-all` runs the complete local quality gate used for contributions. +`make test` is service-free and writes JUnit and coverage XML in ignored +`test-results/`, enforcing the 95% floor. CI uses `make test-ci` on the selected +matrix interpreter (`CI_PYTHON`, default `python`) rather than the pinned local +Hatch environment. `make test-spike-ci` requires `AZFUNC_2X_SPIKE=1` and the SDK +2.x wheel environment. `make test-e2e-ci` requires `E2E_BASE_URL`. Both reject skips +and empty selections; neither local tests nor a host smoke certifies Azure. + ## Branch Strategy Use GitHub Flow and branch from `main`. diff --git a/Makefile b/Makefile index e99e7b2..e9931ba 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,16 @@ test: ensure-hatch @echo "Running tests..." @$(HATCH) run test +CI_PYTHON ?= python +TEST_ARGS ?= +.PHONY: test-ci test-spike-ci test-e2e-ci +test-ci: + @FAIL_ON_SKIP=1 $(CI_PYTHON) -m pytest tests --junitxml=test-results/junit.xml --cov-report=xml:test-results/coverage.xml $(TEST_ARGS) +test-spike-ci: + @FAIL_ON_SKIP=1 $(CI_PYTHON) -m pytest -m compat2x -o addopts= -o pythonpath= tests/test_worker_compat_2x_spike.py -v --junitxml=test-results/compat2x-junit.xml --cov=azure_functions_validation --cov-report=xml:test-results/coverage.xml --cov-fail-under=0 +test-e2e-ci: + @FAIL_ON_SKIP=1 $(CI_PYTHON) -m pytest -v -m e2e tests/e2e --no-cov --junitxml=test-results/e2e-junit.xml $(TEST_ARGS) + .PHONY: cov cov: ensure-hatch @$(HATCH) run cov @@ -143,7 +153,7 @@ e2e-local: ensure-hatch .PHONY: e2e-azure e2e-azure: ensure-hatch @echo "Running e2e tests against Azure (E2E_BASE_URL must be set)..." - @$(HATCH) run e2e-azure + @FAIL_ON_SKIP=1 $(HATCH) run e2e-azure .PHONY: build build: ensure-hatch diff --git a/pyproject.toml b/pyproject.toml index b987f3a..b480aed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,7 +113,7 @@ style = "ruff check src tests tools scripts && ruff format --check src tests too # --explicit-package-bases names their files by directory. typecheck = "mypy src tests && mypy --explicit-package-bases tools scripts" lint = "hatch run style && hatch run typecheck" -test = "pytest -v tests" +test = "pytest -v tests --junitxml=test-results/junit.xml --cov-report=xml:test-results/coverage.xml" cov = "pytest --cov-report=term-missing --cov-report=html --cov-report=xml tests/" e2e-azure = "pytest -v -m e2e tests/e2e/ --no-cov" docs = "mkdocs serve" diff --git a/tests/test_ci_workflow_contract.py b/tests/test_ci_workflow_contract.py index 0773e43..591dcbc 100644 --- a/tests/test_ci_workflow_contract.py +++ b/tests/test_ci_workflow_contract.py @@ -16,8 +16,8 @@ def test_sdk_2x_spike_lane_sets_both_fail_closed_variables() -> None: workflow = yaml.safe_load(_WORKFLOW.read_text(encoding="utf-8")) job = workflow["jobs"]["azure-functions-2x"] - spike_steps = [step for step in job["steps"] if "-m compat2x" in step.get("run", "")] - assert spike_steps, "azure-functions-2x no longer runs `pytest -m compat2x`" + spike_steps = [step for step in job["steps"] if "make test-spike-ci" in step.get("run", "")] + assert spike_steps, "azure-functions-2x no longer runs the dedicated spike target" for step in spike_steps: env = {**workflow.get("env", {}), **job.get("env", {}), **step.get("env", {})} name = step.get("name") @@ -31,7 +31,7 @@ def test_required_test_lanes_preserve_reports_after_failure() -> None: for name in ("test", "minimum-dependencies", "azure-functions-2x"): steps = workflow["jobs"][name]["steps"] commands = "\n".join(step.get("run", "") for step in steps) - assert "--junitxml=" in commands, name + assert "make test-" in commands, name uploads = [step for step in steps if "actions/upload-artifact@" in step.get("uses", "")] assert uploads, name assert all(step.get("if") == "always()" for step in uploads), name diff --git a/tests/test_thin_lane_contract.py b/tests/test_thin_lane_contract.py new file mode 100644 index 0000000..74d6b8e --- /dev/null +++ b/tests/test_thin_lane_contract.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import yaml + + +def test_matrix_uses_makefile_on_selected_interpreter() -> None: + root = Path(__file__).resolve().parents[1] + workflow = yaml.safe_load((root / ".github/workflows/ci-test.yml").read_text()) + steps = workflow["jobs"]["test"]["steps"] + commands = "\n".join(step.get("run", "") for step in steps) + assert "make test-ci" in commands + makefile = (root / "Makefile").read_text() + assert "--junitxml=test-results/junit.xml" in makefile + assert "FAIL_ON_SKIP=1" in makefile + + +def test_azure_certification_uses_fail_closed_make_entrypoint() -> None: + root = Path(__file__).resolve().parents[1] + workflow = yaml.safe_load((root / ".github/workflows/e2e-azure.yml").read_text()) + steps = workflow["jobs"]["deploy_and_test"]["steps"] + test_step = next(step for step in steps if step.get("name") == "Run e2e tests") + assert "make test-e2e-ci" in test_step["run"]