diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 506211d4..450c945f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,17 +81,14 @@ jobs: - name: Build the workspace run: cargo build --locked --workspace --all-features - - name: Prepare the conformance binary artifact - run: | - mkdir -p target/conformance - cp target/debug/contextforge-data-plane target/conformance/contextforge-data-plane - strip target/conformance/contextforge-data-plane + - name: Strip the conformance binary + run: strip target/debug/contextforge-data-plane - name: Upload the conformance binary uses: actions/upload-artifact@v7.0.1 with: - name: contextforge-data-plane-conformance - path: target/conformance/contextforge-data-plane + name: contextforge-data-plane-conformance-${{ github.sha }} + path: target/debug/contextforge-data-plane if-no-files-found: error retention-days: 1 @@ -102,7 +99,3 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2.9.1 - run: cargo bench --locked --workspace --no-run - - conformance: - needs: build - uses: ./.github/workflows/mcp_conformance.yml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 00000000..6bae224b --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,118 @@ +name: Conformance +run-name: Conformance for PR #${{ github.event.issue.number }} + +on: + issue_comment: + types: [created] + +permissions: + actions: read + contents: read + +concurrency: + group: conformance-pr-${{ github.event.issue.number }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + acknowledge: + name: acknowledge command + if: >- + github.event.issue.pull_request && + github.event.comment.body == '/conformance' && + contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - name: Acknowledge the command + env: + COMMENT_ID: ${{ github.event.comment.id }} + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + run: gh api --method POST "repos/$REPOSITORY/issues/comments/$COMMENT_ID/reactions" -f content=eyes + + conformance: + name: conformance + if: >- + github.event.issue.pull_request && + github.event.comment.body == '/conformance' && + contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check out data plane + uses: actions/checkout@v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + ref: refs/pull/${{ github.event.issue.number }}/merge + + - uses: taiki-e/install-action@v2.75.27 + with: + tool: cargo-binstall + + - name: Install cf-integration 0.2.0 binary + run: cargo binstall cf-integration@0.2.0 --no-confirm + + - name: Prepare the exact data-plane image + env: + GH_TOKEN: ${{ github.token }} + run: >- + cf-integration ci prepare-image + --artifact contextforge-data-plane-conformance + --binary contextforge-data-plane + --image contextforge-data-plane:conformance + + - name: Install Node.js 22 + uses: actions/setup-node@v6.5.0 + with: + node-version: 22 + + - name: Run strict modern conformance + env: + CF_DATAPLANE_IMAGE: contextforge-data-plane:conformance + CF_DATAPLANE_PULL_POLICY: never + CF_DATAPLANE_REF: "" + run: make conformance + + - name: Upload conformance diagnostics + if: failure() + uses: actions/upload-artifact@v7.0.1 + with: + name: conformance-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .integration/conformance/ + .integration/reports/ + if-no-files-found: warn + include-hidden-files: true + retention-days: 7 + + report: + name: report conformance result + if: >- + always() && + needs.conformance.result != 'skipped' + needs: conformance + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - name: Report the result + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.issue.number }} + REPOSITORY: ${{ github.repository }} + RESULT: ${{ needs.conformance.result }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + case "$RESULT" in + success) marker="✅" ;; + failure) marker="❌" ;; + cancelled) marker="⚪" ;; + *) marker="⚠️" ;; + esac + body="$marker Conformance **$RESULT** — [view workflow run]($RUN_URL)." + gh api --method POST "repos/$REPOSITORY/issues/$PR_NUMBER/comments" -f body="$body" diff --git a/.github/workflows/mcp_conformance.yml b/.github/workflows/mcp_conformance.yml deleted file mode 100644 index 6b114bf8..00000000 --- a/.github/workflows/mcp_conformance.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: MCP Conformance - -on: - workflow_call: - -permissions: - contents: read - -env: - MCP_CONFORMANCE_COLOR: always - MCP_CONFORMANCE_VERSION: 0.2.0-alpha.11 - MCP_CONFORMANCE_SOURCE_SHA: c321dd32035556e6769d3724a8ee97d87c3faaac # pragma: allowlist secret - MCP_CONFORMANCE_SPEC_VERSION: 2026-07-28 - MCP_CONFORMANCE_SERVER_ID: 3f33286667d34b65a31c3bafd30e4c21 - CF_DATAPLANE_IMAGE: contextforge-data-plane:conformance - -jobs: - conformance: - name: MCP 2026-07-28 through nginx and dataplane - runs-on: ubuntu-latest - continue-on-error: true - timeout-minutes: 30 - - steps: - - name: Check out data plane - uses: actions/checkout@v6.0.2 - - - name: Test conformance reporter - run: tests/conformance/report-baseline-diff-test.sh - - - name: Test client conformance adapter - run: | - python3 -m py_compile tests/conformance/write_client_config.py - tests/conformance/client-under-test-test.sh - tests/conformance/bless-client-baseline-test.sh - - - name: Download the conformance binary - uses: actions/download-artifact@v8.0.1 - with: - name: contextforge-data-plane-conformance - path: target/conformance - - - name: Make the conformance binary executable - run: chmod +x target/conformance/contextforge-data-plane - - - name: Package the data plane conformance image - run: >- - docker buildx build --load - --target conformance-prebuilt - --build-context prebuilt=target/conformance - --tag "${CF_DATAPLANE_IMAGE}" - --file docker/Dockerfile . - - - name: Check out the alpha.11 conformance fixture - uses: actions/checkout@v6.0.2 - with: - repository: modelcontextprotocol/conformance - ref: ${{ env.MCP_CONFORMANCE_SOURCE_SHA }} - path: .conformance-suite - persist-credentials: false - - - name: Install Node.js 22 - uses: actions/setup-node@v6.5.0 - with: - node-version: 22 - cache: npm - cache-dependency-path: .conformance-suite/package-lock.json - - - name: Install official conformance dependencies - working-directory: .conformance-suite - run: | - test "$(node -p "require('./package.json').version")" = "${MCP_CONFORMANCE_VERSION}" - npm ci --ignore-scripts - - # The pinned conformance CLI has no scenario-exclusion option. Apply an - # exact patch so this flaky scenario is not run until forwarding is ordered. - - name: Temporarily disable the flaky progress scenario - working-directory: .conformance-suite - run: git apply ../tests/conformance/disable-flaky-progress.patch - - - name: Resolve latest control-plane main image - env: - GITHUB_TOKEN: ${{ github.token }} - run: echo "CF_CONTROLPLANE_IMAGE=$(tests/conformance/resolve-control-plane-image.sh)" >> "${GITHUB_ENV}" - - - name: Pull external stack images - env: - MCP_CONFORMANCE_TOKEN: pull-only - run: >- - docker compose -f tests/conformance/docker-compose.yml - pull redis fixture-proxy control-plane nginx - - - name: Start the fixture and control plane - env: - MCP_CONFORMANCE_TOKEN: bootstrap-only - run: tests/conformance/start-fixture-and-control-plane.sh - - - name: Register the official fixture through the control plane - env: - MCP_CONFORMANCE_TOKEN: bootstrap-only - run: tests/conformance/register-fixture.sh - - - name: Start dataplane and nginx - run: tests/conformance/start-dataplane-and-nginx.sh - - - name: Run MCP 2026-07-28 requirements - id: runner - run: tests/conformance/run-conformance.sh - - - name: Run scoped MCP 2026-07-28 client conformance - run: tests/conformance/run-client-conformance.sh - - - name: Report conformance baseline diff - if: always() - run: tests/conformance/report-baseline-diff.sh conformance-results/server - - - name: Print live stack logs - if: always() - env: - MCP_CONFORMANCE_TOKEN: diagnostics-only - run: >- - docker compose -f tests/conformance/docker-compose.yml - logs --no-color || true - - - name: Print official fixture log - if: always() - run: | - if [ -f conformance-logs/reference-server.log ]; then - sed -n '1,240p' conformance-logs/reference-server.log - else - echo "No official fixture log was produced." - fi - - - name: Stop the live stack - if: always() - env: - MCP_CONFORMANCE_TOKEN: cleanup-only - run: tests/conformance/stop-live-stack.sh diff --git a/.gitignore b/.gitignore index 5c8c16ae..63bf79d6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,10 +15,11 @@ target # Generated by gateway local runs contextforge-data-plane.log.* -# Generated by local MCP conformance runs -.conformance-suite/ -conformance-logs/ -conformance-results/ +# Generated by cf-integration +.integration/ +/.conformance-suite/ +/conformance-logs/ +/conformance-results/ # Generated by mdBook (wiki) _context/wiki/book/ @@ -34,4 +35,4 @@ _context/wiki/book/ # Personal dev tooling (wt worktree manager, etc.) -.config/ \ No newline at end of file +.config/ diff --git a/Makefile b/Makefile index 78ea1645..240dc62d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ IMAGE_NAME := contextforge-data-plane:latest -CF_DATAPLANE_IMAGE ?= contextforge-data-plane:conformance SERVICES ?= nginx control-plane redis postgres pgbouncer data-plane fast_time_server register_fast_time ARGS ?= +CF_INTEGRATION ?= cf-integration +CF_INTEGRATION_DIR ?= $(CURDIR)/.integration +CF_DATAPLANE_REPO ?= $(CURDIR) +CF_DATAPLANE_REF ?= $(shell git -C "$(CF_DATAPLANE_REPO)" rev-parse HEAD) +CONFORMANCE_BASELINE_DIR := $(CURDIR)/tests/conformance/baselines # IBM detect-secrets hardened fork — pinned to the same commit used in mcp-context-forge. DETECT_SECRETS_SPEC ?= git+https://github.com/ibm/detect-secrets.git@076672a9a01abdfc7ecee2e7d14f08cdccb73976 @@ -25,12 +29,44 @@ compose-up: ## Launch stack: nginx, control plane, redis, postgres, pgbouncer, d compose-down: ## Tear down the stack docker compose -f docker/docker-compose.yml stop $(SERVICES) $(ARGS) -conformance: ## Build the data plane and run official MCP 2026-07-28 conformance locally - docker build -t "$(CF_DATAPLANE_IMAGE)" -f docker/conformance.Dockerfile . - CF_DATAPLANE_IMAGE="$(CF_DATAPLANE_IMAGE)" tests/conformance/run-local.sh +conformance: ## Run strict modern MCP conformance against the committed data-plane HEAD + @if ! command -v "$(CF_INTEGRATION)" >/dev/null 2>&1; then \ + echo "cf-integration not found: install its published binary with cargo binstall or set CF_INTEGRATION to its path."; \ + exit 1; \ + fi + @if [ -n "$$(git -C "$(CF_DATAPLANE_REPO)" status --porcelain --untracked-files=no)" ]; then \ + echo "Tracked data-plane changes are not committed; commit or stash them before conformance."; \ + exit 1; \ + fi + @CF_INTEGRATION_DIR="$(CF_INTEGRATION_DIR)" \ + CF_DATAPLANE_REPO="$(CF_DATAPLANE_REPO)" \ + CF_DATAPLANE_REF="$(CF_DATAPLANE_REF)" \ + "$(CF_INTEGRATION)" conformance run \ + --client-era modern \ + --server-era modern \ + --lane external-data-plane \ + --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ + --output-dir "$(CF_INTEGRATION_DIR)/reports" -conformance-bless: ## Run conformance and update the server and client expected-failure baselines - MCP_CONFORMANCE_BLESS=true $(MAKE) conformance +conformance-bless: ## Run strict modern conformance and atomically update its baselines + @if ! command -v "$(CF_INTEGRATION)" >/dev/null 2>&1; then \ + echo "cf-integration not found: install its published binary with cargo binstall or set CF_INTEGRATION to its path."; \ + exit 1; \ + fi + @if [ -n "$$(git -C "$(CF_DATAPLANE_REPO)" status --porcelain --untracked-files=no)" ]; then \ + echo "Tracked data-plane changes are not committed; commit or stash them before conformance."; \ + exit 1; \ + fi + @CF_INTEGRATION_DIR="$(CF_INTEGRATION_DIR)" \ + CF_DATAPLANE_REPO="$(CF_DATAPLANE_REPO)" \ + CF_DATAPLANE_REF="$(CF_DATAPLANE_REF)" \ + "$(CF_INTEGRATION)" conformance run \ + --client-era modern \ + --server-era modern \ + --lane external-data-plane \ + --baseline-dir "$(CONFORMANCE_BASELINE_DIR)" \ + --output-dir "$(CF_INTEGRATION_DIR)/reports" \ + --bless docs-serve: ## Serve the wiki book locally at http://127.0.0.1:3000 mdbook serve _context/wiki --hostname 127.0.0.1 --port 3000 --open diff --git a/_context/wiki/getting-started.md b/_context/wiki/getting-started.md index 31bf95dd..47983e2f 100644 --- a/_context/wiki/getting-started.md +++ b/_context/wiki/getting-started.md @@ -26,18 +26,18 @@ docker compose -f docker/docker-compose.yml logs -f register_fast_time Teardown: `make compose-down` (stops containers; volumes kept). -## cf-integration Harness (full end-to-end) +## cf-integration Conformance ```bash -scripts/cf-integration.sh up # checkout Python control/built-in repo, pull external-dataplane image, start full stack -scripts/cf-integration.sh probe # smoke: 401 check → initialize → tools/list → tools/call -scripts/cf-integration.sh test-all # all lanes: live-mcp, live-rbac, live-protocol -scripts/cf-integration.sh down +cargo binstall cf-integration@0.1.0 --no-confirm +make conformance ``` -Admin UI (control-plane): `http://localhost:8080/admin` — `admin@example.com` / `changeme` - -Key env overrides: `CF_DATAPLANE_IMAGE`, `CF_DATAPLANE_VERSION`, `NGINX_PORT` (default `8080`). +This runs the modern client and modern server eras through the committed +external-dataplane `HEAD`, including fixture-direct server comparison and the +scoped client suite. Use `make conformance-bless` to replace all selected +baselines transactionally after a fully successful run. Generated checkouts, +results, reports, and logs stay under `.integration/`. ## Local Cargo Dev Workflow diff --git a/_context/wiki/performance.md b/_context/wiki/performance.md index 700584f8..fe782bf0 100644 --- a/_context/wiki/performance.md +++ b/_context/wiki/performance.md @@ -3,7 +3,7 @@ ## Full-Stack Load (Locust via cf-integration) Performance testing uses the control-plane Locust suite through -[`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools). +[`cf-integration`](https://crates.io/crates/cf-integration). It measures the nginx → external dataplane → backend request path while the ContextForge control plane publishes configuration. diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 3d811d40..cf17c7d9 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -34,115 +34,44 @@ Protocol-sensitive tests and fixtures must cover MCP `2026-07-28` and `2025-11-2 These run in `cargo nextest run` with no Docker dependencies. -## MCP Conformance CI - -`.github/workflows/mcp_conformance.yml` runs the pinned official conformance -suite `0.2.0-alpha.11` for MCP `2026-07-28` in both directions. The server leg -is official client → nginx → checked-out external dataplane → fixture proxy -→ official server, with the newest available image built from the control plane's `main` -branch registering and publishing the fixture through Redis. The backend-only -proxy rewrites `Host` to -`localhost:3000`, which the official fixture's DNS-rebinding protection -requires, while leaving external-dataplane header protections unchanged. The -control plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives -in `tests/conformance/`. - -The scoped client leg then treats the external dataplane as an MCP client: the -official runner starts a scenario backend, the adapter publishes an isolated -route to Redis, and a downstream `tools/call` makes the external dataplane -connect to that backend. It covers tool calls, per-request client metadata and -protocol-version retry, standard MCP headers, and custom parameter headers. The -control plane is stopped first so its periodic publisher cannot replace the -scenario route or probe the observation backend. OAuth client scenarios remain -a control-plane responsibility. Server and client results are written below -`server/` and `client/`, with separate `expected-failures.yml` and -`client-expected-failures.yml` baselines. - -The official fixture keeps some diagnostic tools out of `tools/list`. Because -the external dataplane fails closed unless the control plane publishes a tool -schema, checks that require those hidden tools remain explicit server-leg -baseline entries rather than bypassing schema validation in the harness. - -`make conformance` runs both legs locally, while `make conformance-bless` runs -both and refreshes both expected-failure baselines from that run. - -Because this conformance CLI cannot set a bearer header, nginx adds an -ephemeral control-plane token when one is absent; there is no auth proxy or -repository-owned JavaScript. A route probe prevents built-in-dataplane fallback. -Counts and the official fixture log appear directly in the Actions log. The -job does not retain a separate conformance artifact. -`upstream-fixture-failures.yml` records the pinned fixture's seven scored -failures and one warning; its other 47 failures are extension or pending -scenarios and are already unscored. CI prints the exact server -actual-versus-baseline diff, adds annotations for unexpected and stale entries, -and writes the same comparison to the job summary. - -## Full-Stack Integration Harness - -[`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools) -wires the ContextForge control plane, built-in dataplane, and this ContextForge -external dataplane together. The stock Python Compose stack contains both the -control plane and built-in dataplane. The harness adds two intentional -differences: nginx routes the selected `/servers/{virtual_host_id}/mcp` path to -the external dataplane as `/contextforge-rs/servers/{virtual_host_id}/mcp`, and -the control plane runs with `DATAPLANE_PUBLISHER=true` so virtual-server config -reaches the external dataplane through Redis. - -### Quick Start +## MCP Conformance -```bash -scripts/cf-integration.sh up -``` - -This checks out the Python control-plane/built-in-dataplane repository under -`.integration/mcp-context-forge`, pulls the published external-dataplane image, -and starts the combined stack plus a local MCP counter backend. The admin UI is -at `http://localhost:8080/admin` (`admin@example.com` / `changeme`). A Fast Time -backend is auto-registered as a fixed virtual server, so the commands below -work with no manual UI step. +[`cf-integration`](https://crates.io/crates/cf-integration) +owns the official fixture, control-plane registration, Compose topology, server +and client runners, result rendering, and transactional baseline handling. This +repository keeps only the CI invocation, Make targets, and expected findings. -### Route Probe +Comment exactly `/conformance` on a pull request to run the **Conformance** +Actions workflow. Only repository owners, members, and collaborators can start +it. The workflow acknowledges the command, tests the pull request merge commit, +and reports the final result back to the pull request. It runs the modern client +and modern server eras through the external dataplane. Selecting that lane also +runs the fixture-direct server leg and the scoped external-dataplane client leg: ```bash -scripts/cf-integration.sh probe +cargo binstall cf-integration@0.1.0 --no-confirm +make conformance ``` -Verifies the public nginx-to-external-dataplane route end to end: a 401 negative check, `initialize`, session reuse, `tools/list`, and `tools/call`. - -### Full Test Runs - -| Command | What it runs | -| --- | --- | -| `scripts/cf-integration.sh test-all` | Every live lane against the running stack, with per-test result rows and full output in a timestamped log under `.integration/test-logs/`. | -| `CF_TEST_ALL_LOCUST=true scripts/cf-integration.sh test-all` | Same, plus the full Locust load run as a final lane. | -| `scripts/cf-integration.sh test-all-up` | Start or update the stack, then `test-all` without the load lane. | -| `scripts/cf-integration.sh test-all-up-load` | Start or update the stack, then `test-all` with the load lane. | - -Individual lanes: `live-mcp`, `live-rbac`, `live-protocol`, and `live-all`. -`live-mcp` is the green lane: the full MCP protocol end-to-end suite passes -against this harness. Remaining failures in other lanes measure known -external-dataplane feature gaps; the harness `reports/` directory keeps the -current classification. - -### Built-In-Dataplane Baseline - -To separate external-dataplane regressions from Python behavior, the harness -can run the stock `IBM/mcp-context-forge` stack. MCP traffic then uses the -ContextForge built-in dataplane; the external dataplane, nginx split, and -publisher are absent: +The Make target tests the committed data-plane `HEAD`. It rejects tracked +uncommitted changes because the CLI clones the selected repository and commit +into `.integration/`. To use another local CLI binary: ```bash -scripts/cf-integration.sh down # frees the shared host ports -scripts/cf-integration.sh controlplane-test-all # up + live core + locust +CF_INTEGRATION=/path/to/cf-integration \ + make conformance ``` -Individual steps: `controlplane-up`, `controlplane-live-core`, `controlplane-live-all`, `controlplane-locust`, and `controlplane-down`. The baseline load run is covered in [Performance](performance.md). +Update every selected baseline atomically only after all operational work and +baseline evaluation succeeds: -### Key Settings +```bash +make conformance-bless +``` -| Variable | Purpose | -| --- | --- | -| `CF_DATAPLANE_IMAGE` / `CF_DATAPLANE_VERSION` | Which published external-dataplane image the stack runs. | -| `CF_CONTROLPLANE_IMAGE` / `CF_CONTROLPLANE_REF` | Which `IBM/mcp-context-forge` Python image and git ref to use for the control plane and built-in dataplane. | -| `NGINX_PORT` | Public front-door port (default `8080`). | -| `CF_TEST_LOG_DIR` | Where `test-all` writes timestamped logs. | +Baselines are partitioned beneath +`tests/conformance/baselines///`. Server findings +use `fixture-direct.yml` and `external-data-plane.yml`; scoped client findings +use `client/external-data-plane.yml`. Operational failures are always failures +and cannot be blessed. Runtime checkouts, logs, results, and reports remain +beneath `.integration/`. diff --git a/docker/conformance.Dockerfile b/docker/conformance.Dockerfile deleted file mode 100644 index 494cd43d..00000000 --- a/docker/conformance.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM rust:1.96.1 AS builder -WORKDIR /app - -RUN < "${baseline_file}" <<'EOF' -client: [] -EOF -for scenario in tools_call request-metadata http-standard-headers; do - cat > "${results_dir}/${scenario}-2026-08-20T00-00-00-000Z/checks.json" <<'EOF' -[{"id":"passes","status":"SUCCESS"}] -EOF -done -cat > "${results_dir}/http-custom-headers-2026-08-20T00-00-00-000Z/checks.json" <<'EOF' -[ - {"id":"mirrors","status":"SUCCESS"}, - {"id":"mirrors","status":"FAILURE"}, - {"id":"encodes","status":"WARNING"}, - {"id":"omits-null","status":"SUCCESS"} -] -EOF - -"${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}" - -cat > "${state_dir}/expected.yml" <<'EOF' -# Generated by `make conformance-bless` from scoped client findings. -# OAuth client scenarios are control-plane responsibilities and are not run here. -client: - - http-custom-headers:encodes - - http-custom-headers:mirrors -EOF -diff -u "${state_dir}/expected.yml" "${baseline_file}" - -second_output="$("${script_dir}/bless-client-baseline.sh" "${results_dir}" "${baseline_file}")" -grep --fixed-strings --quiet -- 'was already current' <<< "${second_output}" - -echo 'client conformance baseline tests passed' diff --git a/tests/conformance/bless-client-baseline.sh b/tests/conformance/bless-client-baseline.sh deleted file mode 100755 index d27af6e4..00000000 --- a/tests/conformance/bless-client-baseline.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ "$#" -gt 2 ]; then - echo "Usage: bless-client-baseline.sh [results-dir [baseline-file]]" >&2 - exit 2 -fi - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -results_dir="${1:-${repo_root}/conformance-results/client}" -baseline_file="${2:-${script_dir}/client-expected-failures.yml}" -scenarios=(tools_call request-metadata http-standard-headers http-custom-headers) - -for command in awk cmp cp jq mktemp sed sort; do - if ! command -v "${command}" > /dev/null 2>&1; then - echo "Required command not found: ${command}" >&2 - exit 2 - fi -done -if [ ! -d "${results_dir}" ]; then - echo "No client conformance results directory: ${results_dir}" >&2 - exit 2 -fi -if [ ! -f "${baseline_file}" ]; then - echo "Client conformance baseline not found: ${baseline_file}" >&2 - exit 2 -fi - -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-baseline.XXXXXX")" -findings="${state_dir}/findings.txt" -baseline_entries="${state_dir}/baseline-entries.txt" -candidate="${state_dir}/client-expected-failures.yml" - -cleanup() { - rm -f -- "${findings}" "${baseline_entries}" "${candidate}" - rmdir -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -: > "${findings}" -for scenario in "${scenarios[@]}"; do - matches=("${results_dir}/${scenario}-"*/checks.json) - if [ ! -e "${matches[0]}" ]; then - echo "No client conformance results found for scenario: ${scenario}" >&2 - exit 2 - fi - checks_file="${matches[$((${#matches[@]} - 1))]}" - jq --raw-output --arg scenario "${scenario}" ' - def severity($status): - if $status == "FAILURE" then 3 - elif $status == "WARNING" then 2 - elif $status == "SUCCESS" then 1 - else 0 - end; - - reduce (.[] | select(.status != "INFO")) as $check - ({}; - ($check.id) as $id | - if .[$id] == null or severity($check.status) >= severity(.[$id].status) - then .[$id] = $check - else . - end) | - to_entries[] | - select(.value.status == "FAILURE" or .value.status == "WARNING") | - ($scenario + ":" + .key) - ' "${checks_file}" >> "${findings}" -done -LC_ALL=C sort -u -o "${findings}" "${findings}" - -awk ' - /^client:$/ { in_client = 1; next } - in_client && /^[^[:space:]]/ { exit } - in_client && /^[[:space:]]*-[[:space:]]+/ { - line = $0 - sub(/^[[:space:]]*-[[:space:]]+/, "", line) - sub(/[[:space:]]+#.*$/, "", line) - print line - } -' "${baseline_file}" | LC_ALL=C sort -u > "${baseline_entries}" - -{ - # shellcheck disable=SC2016 # Backticks are literal Markdown. - echo '# Generated by `make conformance-bless` from scoped client findings.' - echo '# OAuth client scenarios are control-plane responsibilities and are not run here.' - if [ -s "${findings}" ]; then - echo 'client:' - sed 's/^/ - /' "${findings}" - else - echo 'client: []' - fi -} > "${candidate}" - -if cmp --silent "${findings}" "${baseline_entries}"; then - echo "Client conformance baseline was already current: ${baseline_file}" -else - cp "${candidate}" "${baseline_file}" - echo "Client conformance baseline updated: ${baseline_file}" -fi diff --git a/tests/conformance/client-expected-failures.yml b/tests/conformance/client-expected-failures.yml deleted file mode 100644 index 5f19d937..00000000 --- a/tests/conformance/client-expected-failures.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Dataplane-owned upstream MCP client findings for the scoped client lane. -# OAuth scenarios are control-plane responsibilities and are not run here. -client: - # The shell adapter drives the dataplane's outbound client path but is not a - # full MCP client: it does not discover x-mcp-header annotations or generate - # Mcp-Param-* headers. Header forwarding is covered by gateway integration tests. - - http-custom-headers:sep-2243-client-supports-custom-headers - - http-custom-headers:sep-2243-client-mirrors-designated-params - - http-custom-headers:sep-2243-client-encode-values - - http-custom-headers:sep-2243-client-base64-unsafe diff --git a/tests/conformance/client-under-test-test.sh b/tests/conformance/client-under-test-test.sh deleted file mode 100755 index 3387f24a..00000000 --- a/tests/conformance/client-under-test-test.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-client-adapter-test.XXXXXX")" -fake_bin="${state_dir}/bin" -docker_args="${state_dir}/docker-args" -curl_bodies="${state_dir}/curl-bodies" - -cleanup() { - rm -rf -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -mkdir -p "${fake_bin}" -cat > "${fake_bin}/docker" <<'EOF' -#!/usr/bin/env bash -printf '%s\n' "$*" > "${FAKE_DOCKER_ARGS}" -EOF -cat > "${fake_bin}/curl" <<'EOF' -#!/usr/bin/env bash -while [ "$#" -gt 0 ]; do - if [ "$1" = "--data" ]; then - shift - printf '%s\n' "$1" >> "${FAKE_CURL_BODIES}" - fi - shift -done -printf '%s\n' 'data: {"jsonrpc":"2.0","id":1,"result":{"content":[]}}' -EOF -chmod +x "${fake_bin}/docker" "${fake_bin}/curl" - -export PATH="${fake_bin}:${PATH}" -export FAKE_DOCKER_ARGS="${docker_args}" -export FAKE_CURL_BODIES="${curl_bodies}" -export MCP_CONFORMANCE_PROTOCOL_VERSION=2026-07-28 -export MCP_CONFORMANCE_SUBJECT=test-subject -export MCP_CONFORMANCE_CLIENT_SERVER_ID=test-client-server -export MCP_CONFORMANCE_PORT=18080 -export MCP_CONFORMANCE_SCENARIO=http-custom-headers -export MCP_CONFORMANCE_CONTEXT='{ - "name": "http-custom-headers", - "toolCalls": [ - {"name": "first", "arguments": {"region": "west"}}, - {"name": "second", "arguments": {"verbose": null}} - ] -}' - -"${script_dir}/client-under-test.sh" "http://localhost:43123/mcp" - -grep --fixed-strings --quiet -- 'http://host.docker.internal:43123/mcp' "${docker_args}" -grep --fixed-strings --quiet -- '["first","second"]' "${docker_args}" -test "$(wc -l < "${curl_bodies}" | tr -d '[:space:]')" -eq 2 -jq --exit-status --slurp ' - length == 2 and - .[0].method == "tools/call" and - .[0].params.name == "first" and - .[0].params.arguments.region == "west" and - .[0].params._meta["io.modelcontextprotocol/protocolVersion"] == "2026-07-28" and - .[1].params.name == "second" and - .[1].params.arguments.verbose == null -' "${curl_bodies}" > /dev/null - -echo 'client conformance adapter tests passed' diff --git a/tests/conformance/client-under-test.sh b/tests/conformance/client-under-test.sh deleted file mode 100755 index 909ee2f1..00000000 --- a/tests/conformance/client-under-test.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${MCP_CONFORMANCE_SCENARIO:?MCP_CONFORMANCE_SCENARIO must be set by the conformance runner}" -: "${MCP_CONFORMANCE_PROTOCOL_VERSION:?MCP_CONFORMANCE_PROTOCOL_VERSION must be set by the conformance runner}" -: "${MCP_CONFORMANCE_SUBJECT:?MCP_CONFORMANCE_SUBJECT must be set}" - -if [ "$#" -ne 1 ]; then - echo "Usage: client-under-test.sh " >&2 - exit 2 -fi -if [ "${MCP_CONFORMANCE_PROTOCOL_VERSION}" != "2026-07-28" ]; then - echo "Only MCP 2026-07-28 client conformance is supported" >&2 - exit 2 -fi - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -compose_file="${script_dir}/docker-compose.yml" -virtual_host_id="${MCP_CONFORMANCE_CLIENT_SERVER_ID:-dataplane-client-conformance}" -conformance_port="${MCP_CONFORMANCE_PORT:-8080}" -scenario_server_url="$1" -backend_url="${scenario_server_url/\/\/localhost:/\/\/host.docker.internal:}" -backend_url="${backend_url/\/\/127.0.0.1:/\/\/host.docker.internal:}" - -case "${MCP_CONFORMANCE_SCENARIO}" in - tools_call) - tool_calls='[{"name":"add_numbers","arguments":{"a":2,"b":3}}]' - ;; - request-metadata) - tool_calls='[{"name":"metadata_probe","arguments":{}}]' - ;; - http-standard-headers) - tool_calls='[{"name":"test_headers","arguments":{}}]' - ;; - http-custom-headers) - : "${MCP_CONFORMANCE_CONTEXT:?MCP_CONFORMANCE_CONTEXT is required for http-custom-headers}" - tool_calls="$(jq --exit-status --compact-output '.toolCalls' <<< "${MCP_CONFORMANCE_CONTEXT}")" - ;; - *) - echo "Unsupported dataplane client conformance scenario: ${MCP_CONFORMANCE_SCENARIO}" >&2 - exit 2 - ;; -esac - -tool_names="$(jq --exit-status --compact-output '[.[].name] | unique' <<< "${tool_calls}")" -docker compose -f "${compose_file}" run --rm --no-deps \ - --entrypoint python3 control-plane \ - /opt/contextforge-conformance/write_client_config.py \ - "${MCP_CONFORMANCE_SUBJECT}" \ - "${virtual_host_id}" \ - "${backend_url}" \ - "${tool_names}" \ - > /dev/null - -endpoint="http://127.0.0.1:${conformance_port}/servers/${virtual_host_id}/mcp" -while IFS= read -r tool_call; do - tool_name="$(jq --exit-status --raw-output '.name' <<< "${tool_call}")" - arguments="$(jq --exit-status --compact-output '.arguments' <<< "${tool_call}")" - request="$(jq --null-input --compact-output \ - --arg name "${tool_name}" \ - --argjson arguments "${arguments}" \ - --arg version "${MCP_CONFORMANCE_PROTOCOL_VERSION}" \ - '{ - jsonrpc: "2.0", - id: 1, - method: "tools/call", - params: { - name: $name, - arguments: $arguments, - _meta: { - "io.modelcontextprotocol/protocolVersion": $version, - "io.modelcontextprotocol/clientInfo": { - name: "dataplane-client-conformance-driver", - version: "1.0.0" - }, - "io.modelcontextprotocol/clientCapabilities": {} - } - } - }')" - - response="$(curl --silent --show-error --fail-with-body \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json, text/event-stream' \ - --header "MCP-Protocol-Version: ${MCP_CONFORMANCE_PROTOCOL_VERSION}" \ - --header 'MCP-Method: tools/call' \ - --header "MCP-Name: ${tool_name}" \ - --data "${request}" \ - "${endpoint}")" - - response_json="$(sed -n 's/^data: //p' <<< "${response}" | head -n 1)" - if [ -z "${response_json}" ]; then - response_json="${response}" - fi - if ! jq --exit-status '.result != null and .error == null' <<< "${response_json}" > /dev/null; then - echo "Dataplane rejected client conformance tool call ${tool_name}:" >&2 - echo "${response}" >&2 - exit 1 - fi -done < <(jq --compact-output '.[]' <<< "${tool_calls}") diff --git a/tests/conformance/disable-flaky-progress.patch b/tests/conformance/disable-flaky-progress.patch deleted file mode 100644 index 41b4fc11..00000000 --- a/tests/conformance/disable-flaky-progress.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/requirements/2026-07-28.yaml b/requirements/2026-07-28.yaml ---- a/requirements/2026-07-28.yaml -+++ b/requirements/2026-07-28.yaml -@@ -45,7 +45,6 @@ server: - - tools-call-embedded-resource - - tools-call-mixed-content - - tools-call-error -- - tools-call-with-progress - - server-sse-multiple-streams - - resources-list - - resources-read-text diff --git a/tests/conformance/docker-compose.yml b/tests/conformance/docker-compose.yml deleted file mode 100644 index d16e8fcb..00000000 --- a/tests/conformance/docker-compose.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: contextforge-conformance - -services: - redis: - image: redis:8.8.1-alpine3.23 - command: [redis-server, --save, "", --appendonly, "no"] - networks: [contextforge] - healthcheck: - test: [CMD, redis-cli, ping] - interval: 2s - timeout: 2s - retries: 30 - - fixture-proxy: - image: nginx:1.30.4-alpine3.24 - networks: [contextforge] - extra_hosts: - - host.docker.internal:host-gateway - volumes: - - ./fixture-proxy.conf:/etc/nginx/conf.d/default.conf:ro - healthcheck: - test: - - CMD-SHELL - - >- - curl --fail --silent --show-error --output /dev/null - --request POST - --header 'Content-Type: application/json' - --header 'Accept: application/json, text/event-stream' - --header 'MCP-Protocol-Version: 2026-07-28' - --header 'MCP-Method: server/discover' - --data '{"jsonrpc":"2.0","id":"health","method":"server/discover","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{}}}}' - http://127.0.0.1/mcp - interval: 1s - timeout: 5s - retries: 60 - - control-plane: - image: ${CF_CONTROLPLANE_IMAGE:-ghcr.io/ibm/mcp-context-forge:latest} - ports: - - "127.0.0.1:4444:4444" - networks: [contextforge] - environment: - HOST: 0.0.0.0 - PORT: "4444" - DATABASE_URL: sqlite:////tmp/contextforge-conformance.db - CACHE_TYPE: redis - REDIS_URL: redis://redis:6379/0 - JWT_SECRET_KEY: contextforge-conformance-jwt-secret-at-least-32-bytes - JWT_ALGORITHM: RS256 - JWT_PUBLIC_KEY_PATH: /keys/jwt.key.pub - JWT_PRIVATE_KEY_PATH: /keys/jwt.key - AUTH_ENCRYPTION_SECRET: contextforge-conformance-encryption-secret-at-least-32-bytes # pragma: allowlist secret - AUTH_REQUIRED: "true" - BASIC_AUTH_USER: admin - BASIC_AUTH_PASSWORD: conformance-admin-password # pragma: allowlist secret - PLATFORM_ADMIN_EMAIL: admin@example.com - PLATFORM_ADMIN_PASSWORD: conformance-admin-password # pragma: allowlist secret - MCPGATEWAY_SKIP_MIGRATIONS: "false" - MCPGATEWAY_UI_ENABLED: "false" - MCPGATEWAY_ADMIN_API_ENABLED: "true" - SECURE_COOKIES: "false" - SSRF_ALLOW_PRIVATE_NETWORKS: "true" - DATAPLANE_PUBLISHER: "true" - DATAPLANE_PUBLISHER_INTERVAL_SECONDS: "2" - PASSWORD_CHANGE_ENFORCEMENT_ENABLED: "false" - ADMIN_REQUIRE_PASSWORD_CHANGE_ON_BOOTSTRAP: "false" - REQUIRE_PASSWORD_CHANGE_FOR_DEFAULT_PASSWORD: "false" - GATEWAY_TOOL_NAME_SEPARATOR: _ - GUNICORN_WORKERS: "1" - LOG_LEVEL: INFO - volumes: - - ../../assets:/keys:ro - - ./write_client_config.py:/opt/contextforge-conformance/write_client_config.py:ro - depends_on: - redis: - condition: service_healthy - healthcheck: - test: [CMD, curl, --fail, http://127.0.0.1:4444/health] - interval: 5s - timeout: 5s - retries: 60 - start_period: 10s - - data-plane: - image: ${CF_DATAPLANE_IMAGE:-ghcr.io/contextforge-org/contextforge-data-plane:latest} - networks: [contextforge] - extra_hosts: - - host.docker.internal:host-gateway - environment: - CONTEXTFORGE_DATA_PLANE_ADDRESS: 0.0.0.0:4445 - CONTEXTFORGE_DATA_PLANE_REDIS_HOSTNAME: redis - CONTEXTFORGE_DATA_PLANE_REDIS_PORT: "6379" - CONTEXTFORGE_DATA_PLANE_REDIS_CONNECTION_MODE: plain-text - CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE: plain-text-or-tls - CONTEXTFORGE_DATA_PLANE_TOKEN_VERIFICATION_PUBLIC_KEY: /keys/jwt.key.pub - CONTEXTFORGE_DATA_PLANE_TOKEN_VERIFICATION_PRIVATE_KEY: /keys/jwt.key # pragma: allowlist secret - CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS: "0" - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS: >- - 127.0.0.1:${MCP_CONFORMANCE_PORT:-8080},localhost:${MCP_CONFORMANCE_PORT:-8080} - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_ORIGINS: >- - http://127.0.0.1:${MCP_CONFORMANCE_PORT:-8080},http://localhost:${MCP_CONFORMANCE_PORT:-8080} - RUST_LOG: info - volumes: - - ../../assets:/keys:ro - depends_on: - redis: - condition: service_healthy - - nginx: - image: nginx:1.30.4-alpine3.24 - ports: - - "127.0.0.1:${MCP_CONFORMANCE_PORT:-8080}:80" - networks: [contextforge] - environment: - MCP_CONFORMANCE_TOKEN: ${MCP_CONFORMANCE_TOKEN:?} - NGINX_ENVSUBST_TEMPLATE_DIR: /templates - NGINX_ENVSUBST_TEMPLATE_SUFFIX: .template - NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx - volumes: - - ./nginx.conf.template:/templates/nginx.conf.template:ro - depends_on: - control-plane: - condition: service_healthy - # Full readiness, routing, and auth are verified by start-dataplane-and-nginx.sh. - data-plane: - condition: service_started - healthcheck: - test: [CMD, curl, --fail, http://127.0.0.1/health] - interval: 2s - timeout: 2s - retries: 30 - -networks: - contextforge: - driver: bridge diff --git a/tests/conformance/expected-failures.yml b/tests/conformance/expected-failures.yml deleted file mode 100644 index c38bf4cd..00000000 --- a/tests/conformance/expected-failures.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by `make conformance-bless` from scored dataplane findings. -# Pinned fixture findings are excluded; see upstream-fixture-failures.yml. -server: - - caching:sep-2549-cache-scope-valid - - caching:sep-2549-prompts-list-caching-hints - - caching:sep-2549-resources-list-caching-hints - - caching:sep-2549-resources-templates-list-caching-hints - - caching:sep-2549-tools-list-caching-hints - - caching:sep-2549-ttl-non-negative - - completion-complete:completion-complete - - input-required-result-basic-elicitation:sep-2322-elicitation-incomplete - - input-required-result-basic-list-roots:sep-2322-list-roots-incomplete - - input-required-result-basic-sampling:sep-2322-sampling-incomplete - - input-required-result-capability-check:sep-2322-respect-client-capabilities - - input-required-result-ignore-extra-params:sep-2322-ignore-unexpected-params - - input-required-result-missing-input-response:sep-2322-missing-response-rerequests - - input-required-result-multi-round:sep-2322-multi-round-r1 - - input-required-result-multiple-input-requests:sep-2322-multiple-inputs-incomplete - - input-required-result-non-tool-request:sep-2322-non-tool-incomplete - - input-required-result-request-state:sep-2322-request-state-incomplete - - input-required-result-result-type:sep-2322-result-type-included - - input-required-result-tampered-state:sep-2322-reject-tampered-state - - prompts-list:prompts-list - - resources-list:resources-list - - server-stateless:sep-2575-discover-capabilities-match-handlers - - server-stateless:sep-2575-http-server-no-independent-requests-on-stream - - server-stateless:sep-2575-missing-capability-http-400 - - server-stateless:sep-2575-server-declares-prompts-in-discover - - server-stateless:sep-2575-server-no-log-without-loglevel - - server-stateless:sep-2575-server-rejects-undeclared-capability - - tools-call-with-progress:tools-call-with-progress - - tools-list:tools-list diff --git a/tests/conformance/fixture-proxy.conf b/tests/conformance/fixture-proxy.conf deleted file mode 100644 index 7fac91be..00000000 --- a/tests/conformance/fixture-proxy.conf +++ /dev/null @@ -1,18 +0,0 @@ -server { - listen 80 default_server; - server_name _; - - location / { - proxy_pass http://host.docker.internal:3000; - proxy_http_version 1.1; - proxy_set_header Host localhost:3000; - proxy_set_header Connection ""; - proxy_request_buffering on; - proxy_buffering off; - proxy_cache off; - proxy_connect_timeout 30s; - proxy_send_timeout 1h; - proxy_read_timeout 1h; - add_header X-Accel-Buffering "no" always; - } -} diff --git a/tests/conformance/nginx.conf.template b/tests/conformance/nginx.conf.template deleted file mode 100644 index 404e8d17..00000000 --- a/tests/conformance/nginx.conf.template +++ /dev/null @@ -1,65 +0,0 @@ -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - access_log /var/log/nginx/access.log; - resolver 127.0.0.11 valid=5s ipv6=off; - - map $http_authorization $conformance_authorization { - default $http_authorization; - "" "Bearer ${MCP_CONFORMANCE_TOKEN}"; - } - - server { - listen 80 default_server; - server_name _; - - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header Authorization $conformance_authorization; - proxy_set_header Mcp-Session-Id $http_mcp_session_id; - proxy_set_header Mcp-Protocol-Version $http_mcp_protocol_version; - proxy_set_header Origin $http_origin; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header Connection ""; - - location ~ ^/servers/([^/]+)/mcp/?$ { - set $dataplane http://data-plane:4445; - proxy_pass $dataplane/contextforge-rs/servers/$1/mcp$is_args$args; - proxy_request_buffering on; - proxy_buffering off; - proxy_cache off; - proxy_connect_timeout 30s; - proxy_send_timeout 1h; - proxy_read_timeout 1h; - add_header X-Accel-Buffering "no" always; - add_header X-CF-Conformance-Backend dataplane always; - } - - location = /health { - set $control_plane http://control-plane:4444; - proxy_pass $control_plane/health; - proxy_connect_timeout 5s; - proxy_read_timeout 5s; - } - - location / { - set $control_plane http://control-plane:4444; - proxy_pass $control_plane; - proxy_buffering off; - proxy_connect_timeout 30s; - proxy_send_timeout 120s; - proxy_read_timeout 120s; - } - } -} diff --git a/tests/conformance/register-fixture.sh b/tests/conformance/register-fixture.sh deleted file mode 100755 index dca91dcc..00000000 --- a/tests/conformance/register-fixture.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${GITHUB_ENV:?GITHUB_ENV must be set}" -: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -compose_file="${script_dir}/docker-compose.yml" - -bootstrap_token="$({ - docker compose -f "${compose_file}" exec -T control-plane \ - python3 -m mcpgateway.utils.create_jwt_token \ - --username admin@example.com --admin --exp 120 -} 2>/dev/null | tail -n 1)" -test -n "${bootstrap_token}" -if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - echo "::add-mask::${bootstrap_token}" -fi - -api_request() { - local method="$1" - local path="$2" - local body="${3-}" - local args=( - --silent --show-error --fail-with-body - --request "${method}" - --header "Authorization: Bearer ${bootstrap_token}" - --header "Content-Type: application/json" - "http://127.0.0.1:4444${path}" - ) - if [ -n "${body}" ]; then - args+=(--data "${body}") - fi - curl "${args[@]}" -} - -gateway="$(api_request POST /gateways '{ - "name": "_", - "url": "http://fixture-proxy/mcp", - "transport": "STREAMABLEHTTP", - "authType": "authheaders", - "authHeaders": [{"key": "Host", "value": "localhost:3000"}], - "description": "Official MCP alpha.11 conformance fixture through the test-only Host proxy" -}')" -gateway_id="$(jq --exit-status --raw-output '.id' <<< "${gateway}")" - -api_request POST \ - "/gateways/${gateway_id}/tools/refresh?include_resources=true&include_prompts=true" \ - '{}' > /dev/null - -tool_ids='[]' -resource_ids='[]' -prompt_ids='[]' -has_tool=0 -has_resource=0 -has_prompt=0 -for _ in $(seq 1 120); do - tools="$(api_request GET /tools)" - resources="$(api_request GET /resources)" - prompts="$(api_request GET /prompts)" - - tool_ids="$(jq --compact-output --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${tools}")" - resource_ids="$(jq --compact-output --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${resources}")" - prompt_ids="$(jq --compact-output --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${prompts}")" - - has_tool="$(jq --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id and .name == "test_simple_text")] | length' \ - <<< "${tools}")" - has_resource="$(jq --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id and .uri == "test://static-text")] | length' \ - <<< "${resources}")" - has_prompt="$(jq --arg id "${gateway_id}" \ - '[.[] | select((.gateway_id // .gatewayId) == $id and .name == "test_simple_prompt")] | length' \ - <<< "${prompts}")" - if [ "${has_tool}" -gt 0 ] && [ "${has_resource}" -gt 0 ] && [ "${has_prompt}" -gt 0 ]; then - break - fi - sleep 0.5 -done -test "${has_tool}" -gt 0 -test "${has_resource}" -gt 0 -test "${has_prompt}" -gt 0 - -server_payload="$(jq --null-input --compact-output \ - --arg id "${MCP_CONFORMANCE_SERVER_ID}" \ - --argjson tools "${tool_ids}" \ - --argjson resources "${resource_ids}" \ - --argjson prompts "${prompt_ids}" \ - '{server: { - id: $id, - name: "Official MCP Conformance Server", - description: "Virtual server for alpha.11 conformance", - associated_tools: $tools, - associated_resources: $resources, - associated_prompts: $prompts - }}')" -api_request POST /servers "${server_payload}" > /dev/null - -token_response="$(api_request POST /v1/tokens '{ - "name": "MCP conformance CI", - "description": "Ephemeral dataplane token", - "expires_in_days": 1, - "user_email": "admin@example.com" -}')" -conformance_token="$(jq --exit-status --raw-output '.access_token' <<< "${token_response}")" -if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - echo "::add-mask::${conformance_token}" -fi - -jwt_payload="$(cut -d. -f2 <<< "${conformance_token}")" -jwt_payload="${jwt_payload//-/+}" -jwt_payload="${jwt_payload//_/\/}" -case $((${#jwt_payload} % 4)) in - 2) jwt_payload="${jwt_payload}==" ;; - 3) jwt_payload="${jwt_payload}=" ;; -esac -conformance_subject="$(jq --raw-input --exit-status --raw-output '@base64d | fromjson | .sub' <<< "${jwt_payload}")" - -echo "MCP_CONFORMANCE_TOKEN=${conformance_token}" >> "${GITHUB_ENV}" -echo "MCP_CONFORMANCE_SUBJECT=${conformance_subject}" >> "${GITHUB_ENV}" diff --git a/tests/conformance/report-baseline-diff-test.sh b/tests/conformance/report-baseline-diff-test.sh deleted file mode 100755 index 525f357b..00000000 --- a/tests/conformance/report-baseline-diff-test.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -reporter="${script_dir}/report-baseline-diff.sh" -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-test.XXXXXX")" -suite_dir="${state_dir}/suite" -results_dir="${state_dir}/results" -baseline_file="${state_dir}/expected-failures.yml" -upstream_file="${state_dir}/upstream-fixture-failures.yml" -summary_file="${state_dir}/summary.md" - -cleanup() { - rm -rf -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -mkdir -p "${suite_dir}/requirements" "${results_dir}" - -cat > "${suite_dir}/requirements/2026-07-28.yaml" <<'EOF' -server: - - expected-check - - expected-whole - - regression - - xpass-check - - xpass-whole - - absent-check - - upstream - - duplicate - - normal-pass -EOF - -cat > "${baseline_file}" <<'EOF' -server: - - expected-check:known - - expected-whole - - xpass-check:fixed - - xpass-whole - - absent-check:not-emitted -EOF - -cat > "${upstream_file}" <<'EOF' -server: - - upstream:fixture-defect -EOF - -write_checks() { - scenario="$1" - checks="$2" - result_dir="${results_dir}/server-${scenario}-2026-08-18T12-00-00-000Z" - mkdir -p "${result_dir}" - printf '%s\n' "${checks}" > "${result_dir}/checks.json" -} - -write_checks expected-check '[{"id":"known","status":"FAILURE"}]' -write_checks expected-whole '[{"id":"any-failure","status":"WARNING"}]' -write_checks regression '[{"id":"new-failure","status":"FAILURE"}]' -write_checks xpass-check '[{"id":"fixed","status":"SUCCESS"}]' -write_checks xpass-whole '[{"id":"all-good","status":"SUCCESS"}]' -write_checks absent-check '[{"id":"other","status":"SUCCESS"}]' -write_checks upstream '[{"id":"fixture-defect","status":"FAILURE","errorMessage":"must not be reported as a dataplane failure"}]' -write_checks duplicate '[{"id":"repeated","status":"FAILURE"},{"id":"repeated","status":"SUCCESS"}]' -write_checks normal-pass '[{"id":"good","status":"SUCCESS"},{"id":"not-applicable","status":"SKIPPED"}]' - -assert_contains() { - haystack="$1" - needle="$2" - if [[ "${haystack}" != *"${needle}"* ]]; then - echo "Expected output to contain: ${needle}" >&2 - echo "${haystack}" >&2 - exit 1 - fi -} - -assert_not_contains() { - haystack="$1" - needle="$2" - if [[ "${haystack}" == *"${needle}"* ]]; then - echo "Expected output not to contain: ${needle}" >&2 - echo "${haystack}" >&2 - exit 1 - fi -} - -set +e -output="$( - GITHUB_ACTIONS=true \ - GITHUB_STEP_SUMMARY="${summary_file}" \ - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" 2>&1 -)" -status="$?" -set -e - -if [ "${status}" -ne 1 ]; then - echo "Expected mismatch status 1, got ${status}" >&2 - echo "${output}" >&2 - exit 1 -fi - -assert_contains "${output}" 'XFAIL expected-check:known' -assert_contains "${output}" 'XFAIL expected-whole' -assert_contains "${output}" 'UPSTREAM upstream:fixture-defect' -assert_contains "${output}" 'FAIL duplicate:repeated (expected PASS, got FAILURE)' -assert_contains "${output}" 'FAIL regression:new-failure (expected PASS, got FAILURE)' -assert_contains "${output}" 'XPASS xpass-check:fixed (expected FAILURE, got PASS)' -assert_contains "${output}" 'XPASS xpass-whole (expected FAILURE, got PASS)' -assert_not_contains "${output}" 'XPASS absent-check:not-emitted' -assert_not_contains "${output}" '::error title=Expected conformance pass failed::upstream:fixture-defect' - -summary="$(cat "${summary_file}")" -assert_contains "${summary}" '| Pinned fixture findings ignored | 1 |' -assert_not_contains "${summary}" 'upstream:fixture-defect' - -cat > "${state_dir}/unmatched-upstream.yml" <<'EOF' -server: - - never-seen:fixture-defect -EOF -set +e -unmatched_upstream_output="$( - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" \ - "${results_dir}" \ - "${baseline_file}" \ - "${state_dir}/unmatched-upstream.yml" 2>&1 -)" -unmatched_upstream_status="$?" -set -e -if [ "${unmatched_upstream_status}" -ne 1 ]; then - echo "Expected unmatched-upstream status 1, got ${unmatched_upstream_status}" >&2 - exit 1 -fi -assert_contains "${unmatched_upstream_output}" 'FAIL upstream:fixture-defect' - -echo 'server: []' > "${state_dir}/empty-baseline.yml" -set +e -empty_baseline_output="$( - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" \ - "${results_dir}" \ - "${state_dir}/empty-baseline.yml" \ - "${upstream_file}" 2>&1 -)" -empty_baseline_status="$?" -set -e -if [ "${empty_baseline_status}" -ne 1 ]; then - echo "Expected empty-baseline status 1, got ${empty_baseline_status}" >&2 - exit 1 -fi -assert_contains "${empty_baseline_output}" 'FAIL regression:new-failure' - -bless_output="$( - MCP_CONFORMANCE_COLOR=never \ - MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" --bless "${results_dir}" "${baseline_file}" "${upstream_file}" -)" -assert_contains "${bless_output}" "BLESS updated ${baseline_file}" - -cat > "${state_dir}/expected-after-bless.yml" <<'EOF' -# Generated by `make conformance-bless` from scored dataplane findings. -# Pinned fixture findings are excluded; see upstream-fixture-failures.yml. -server: - - duplicate:repeated - - expected-check:known - - expected-whole:any-failure - - regression:new-failure -EOF -diff -u "${state_dir}/expected-after-bless.yml" "${baseline_file}" - -MCP_CONFORMANCE_COLOR=never \ -MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" > /dev/null - -baseline_before_missing="$(cat "${baseline_file}")" -set +e -MCP_CONFORMANCE_COLOR=never \ -MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ - "${reporter}" --bless "${state_dir}/missing" "${baseline_file}" "${upstream_file}" > /dev/null 2>&1 -missing_status="$?" -set -e - -if [ "${missing_status}" -ne 2 ]; then - echo "Expected missing-results status 2, got ${missing_status}" >&2 - exit 1 -fi -if [ "$(cat "${baseline_file}")" != "${baseline_before_missing}" ]; then - echo 'Bless changed the baseline without results' >&2 - exit 1 -fi - -echo 'conformance reporter tests passed' diff --git a/tests/conformance/report-baseline-diff.sh b/tests/conformance/report-baseline-diff.sh deleted file mode 100755 index 6f7eeb92..00000000 --- a/tests/conformance/report-baseline-diff.sh +++ /dev/null @@ -1,433 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - cat <<'EOF' -Usage: report-baseline-diff.sh [--bless] [results-dir [baseline-file [upstream-file]]] - -Compare scored MCP conformance checks with the expected-failure baseline. -With --bless, replace the baseline with the current dataplane-owned findings. -EOF -} - -bless=false -case "${1:-}" in - --bless) - bless=true - shift - ;; - --help|-h) - usage - exit 0 - ;; -esac - -if [ "$#" -gt 3 ]; then - usage >&2 - exit 2 -fi - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -results_dir="${1:-${repo_root}/conformance-results}" -baseline_file="${2:-${script_dir}/expected-failures.yml}" -upstream_file="${3:-${script_dir}/upstream-fixture-failures.yml}" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -spec_version="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}" -requirements_file="${suite_dir}/requirements/${spec_version}.yaml" - -for command in awk cmp cp cut find grep jq sed sort wc; do - if ! command -v "${command}" > /dev/null 2>&1; then - echo "Required command not found: ${command}" >&2 - exit 2 - fi -done - -for required_file in "${baseline_file}" "${upstream_file}" "${requirements_file}"; do - if [ ! -f "${required_file}" ]; then - echo "Required conformance file not found: ${required_file}" >&2 - exit 2 - fi -done - -color_mode="${MCP_CONFORMANCE_COLOR:-${CARGO_TERM_COLOR:-auto}}" -case "${color_mode}" in - always) - use_color=true - ;; - never) - use_color=false - ;; - auto) - if [ -t 1 ] && [ "${TERM:-}" != "dumb" ] && [ -z "${NO_COLOR:-}" ]; then - use_color=true - else - use_color=false - fi - ;; - *) - echo "MCP_CONFORMANCE_COLOR must be auto, always, or never; got: ${color_mode}" >&2 - exit 2 - ;; -esac - -if [ -n "${NO_COLOR:-}" ]; then - use_color=false -fi - -if ${use_color}; then - bold=$'\033[1m' - dim=$'\033[2m' - red=$'\033[31m' - green=$'\033[32m' - yellow=$'\033[33m' - cyan=$'\033[36m' - reset=$'\033[0m' -else - bold="" - dim="" - red="" - green="" - yellow="" - cyan="" - reset="" -fi - -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-diff.XXXXXX")" -actual_checks="${state_dir}/actual-checks.tsv" -actual_findings="${state_dir}/actual-findings.tsv" -actual_keys="${state_dir}/actual-keys.txt" -baseline_entries="${state_dir}/baseline-entries.txt" -upstream_entries="${state_dir}/upstream-entries.txt" -scored_scenarios="${state_dir}/scored-scenarios.txt" -executed_scenarios="${state_dir}/executed-scenarios.txt" -owned_findings="${state_dir}/owned-findings.txt" -expected_entries="${state_dir}/expected-entries.txt" -unexpected_entries="${state_dir}/unexpected-entries.txt" -stale_entries="${state_dir}/stale-entries.txt" -upstream_matches="${state_dir}/upstream-matches.txt" -passing_checks="${state_dir}/passing-checks.txt" -skipped_checks="${state_dir}/skipped-checks.txt" -baseline_candidate="${state_dir}/expected-failures.yml" - -cleanup() { - rm -f -- \ - "${actual_checks}" \ - "${actual_findings}" \ - "${actual_keys}" \ - "${baseline_entries}" \ - "${upstream_entries}" \ - "${scored_scenarios}" \ - "${executed_scenarios}" \ - "${owned_findings}" \ - "${expected_entries}" \ - "${unexpected_entries}" \ - "${stale_entries}" \ - "${upstream_matches}" \ - "${passing_checks}" \ - "${skipped_checks}" \ - "${baseline_candidate}" - rmdir -- "${state_dir}" -} -trap cleanup EXIT INT TERM - -read_baseline() { - awk ' - /^[[:space:]]*-[[:space:]]+/ { - line = $0 - sub(/^[[:space:]]*-[[:space:]]+/, "", line) - sub(/[[:space:]]+#.*$/, "", line) - print line - } - ' "$1" | LC_ALL=C sort -u -} - -line_count() { - wc -l < "$1" | tr -d '[:space:]' -} - -print_row() { - local color="$1" - local label="$2" - local message="$3" - printf ' %b%10s%b %s\n' "${color}${bold}" "${label}" "${reset}" "${message}" -} - -emit_error() { - local title="$1" - local message="$2" - if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - echo "::error title=${title}::${message}" - fi -} - -write_missing_results_summary() { - local message="$1" - print_row "${red}" "ERROR" "${message}" - emit_error "Conformance results missing" "${message}" - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - printf '## MCP %s conformance\n\n❌ %s\n' "${spec_version}" "${message}" \ - >> "${GITHUB_STEP_SUMMARY}" - fi -} - -awk ' - /^server:$/ { in_server = 1; next } - in_server && /^[^[:space:]]/ { exit } - in_server && /^[[:space:]]*-[[:space:]]+/ { - line = $0 - sub(/^[[:space:]]*-[[:space:]]+/, "", line) - print line - } -' "${requirements_file}" | LC_ALL=C sort -u > "${scored_scenarios}" - -read_baseline "${baseline_file}" > "${baseline_entries}" -read_baseline "${upstream_file}" > "${upstream_entries}" - -printf '\n%bMCP conformance%b %b(%s)%b\n' "${bold}" "${reset}" "${dim}" "${spec_version}" "${reset}" - -if [ ! -d "${results_dir}" ]; then - write_missing_results_summary "No results directory: ${results_dir}" - exit 2 -fi - -: > "${actual_checks}" -: > "${executed_scenarios}" -while IFS= read -r -d '' checks_file; do - result_name="$(basename -- "$(dirname -- "${checks_file}")")" - if [[ ! "${result_name}" =~ ^server-(.*)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{3}Z$ ]]; then - echo "Skipping unrecognized result directory: ${result_name}" >&2 - continue - fi - scenario="${BASH_REMATCH[1]}" - - if ! grep --fixed-strings --line-regexp --quiet -- "${scenario}" "${scored_scenarios}"; then - continue - fi - - echo "${scenario}" >> "${executed_scenarios}" - jq --raw-output --arg scenario "${scenario}" ' - def severity($status): - if $status == "FAILURE" then 3 - elif $status == "WARNING" then 2 - elif $status == "SUCCESS" then 1 - else 0 - end; - - reduce (.[] | select(.status != "INFO")) as $check - ({}; - ($check.id) as $id | - if .[$id] == null or severity($check.status) >= severity(.[$id].status) - then .[$id] = $check - else . - end) | - to_entries[] | - [($scenario + ":" + .key), .value.status, (.value.errorMessage // "")] | - @tsv - ' "${checks_file}" >> "${actual_checks}" -done < <(find "${results_dir}" -type f -name checks.json -print0) - -LC_ALL=C sort -u -o "${executed_scenarios}" "${executed_scenarios}" -if [ ! -s "${executed_scenarios}" ]; then - write_missing_results_summary "No scored conformance results were found in ${results_dir}" - exit 2 -fi - -LC_ALL=C sort -u -o "${actual_checks}" "${actual_checks}" -awk -F '\t' '$2 == "FAILURE" || $2 == "WARNING"' "${actual_checks}" > "${actual_findings}" -cut -f 1 "${actual_findings}" | LC_ALL=C sort -u > "${actual_keys}" -awk -F '\t' '$2 == "SUCCESS" { print $1 }' "${actual_checks}" | LC_ALL=C sort -u > "${passing_checks}" -awk -F '\t' '$2 == "SKIPPED" { print $1 }' "${actual_checks}" | LC_ALL=C sort -u > "${skipped_checks}" - -# Pinned fixture findings are informational: they neither satisfy the dataplane -# baseline nor count as unexpected dataplane failures. -awk -v upstream_file="${upstream_entries}" ' - FILENAME == upstream_file { - upstream[$1] = 1 - next - } - { - scenario = $1 - sub(/:.*/, "", scenario) - if (($1 in upstream) || (scenario in upstream)) print $1 - } -' "${upstream_entries}" "${actual_keys}" | LC_ALL=C sort -u > "${upstream_matches}" - -awk -v upstream_file="${upstream_matches}" ' - FILENAME == upstream_file { upstream[$1] = 1; next } - !($1 in upstream) { print $1 } -' "${upstream_matches}" "${actual_keys}" > "${owned_findings}" - -bless_changed=false -if ${bless}; then - { - # shellcheck disable=SC2016 # Backticks are literal Markdown. - echo '# Generated by `make conformance-bless` from scored dataplane findings.' - echo '# Pinned fixture findings are excluded; see upstream-fixture-failures.yml.' - if [ -s "${owned_findings}" ]; then - echo 'server:' - sed 's/^/ - /' "${owned_findings}" - else - echo 'server: []' - fi - } > "${baseline_candidate}" - - if ! cmp --silent "${owned_findings}" "${baseline_entries}"; then - cp "${baseline_candidate}" "${baseline_file}" - bless_changed=true - fi - read_baseline "${baseline_file}" > "${baseline_entries}" -fi - -# Match actual owned findings against exact or whole-scenario baseline entries. -awk -v baseline_file="${baseline_entries}" ' - FILENAME == baseline_file { - baseline[$1] = 1 - if (index($1, ":") == 0) whole[$1] = 1 - next - } - { - scenario = $1 - sub(/:.*/, "", scenario) - if (scenario in whole) matched[scenario] = 1 - else if ($1 in baseline) matched[$1] = 1 - } - END { - for (entry in matched) print entry - } -' "${baseline_entries}" "${owned_findings}" | LC_ALL=C sort -u > "${expected_entries}" - -awk -v baseline_file="${baseline_entries}" ' - FILENAME == baseline_file { - baseline[$1] = 1 - if (index($1, ":") == 0) whole[$1] = 1 - next - } - { - scenario = $1 - sub(/:.*/, "", scenario) - if (!(($1 in baseline) || (scenario in whole))) print $1 - } -' "${baseline_entries}" "${owned_findings}" > "${unexpected_entries}" - -# An exact baseline entry is stale only after a demonstrated SUCCESS. Missing -# and SKIPPED checks carry no pass signal. A whole-scenario entry is stale when -# the scenario ran without any dataplane-owned finding. -awk -F '\t' \ - -v checks_file="${actual_checks}" \ - -v scenarios_file="${executed_scenarios}" \ - -v findings_file="${owned_findings}" ' - FILENAME == checks_file { status[$1] = $2; next } - FILENAME == scenarios_file { executed[$1] = 1; next } - FILENAME == findings_file { - scenario = $1 - sub(/:.*/, "", scenario) - failed[scenario] = 1 - next - } - index($1, ":") == 0 { - if (($1 in executed) && !($1 in failed)) print $1 - next - } - status[$1] == "SUCCESS" { print $1 } - ' "${actual_checks}" "${executed_scenarios}" "${owned_findings}" "${baseline_entries}" \ - | LC_ALL=C sort -u > "${stale_entries}" - -pass_count="$(line_count "${passing_checks}")" -skip_count="$(line_count "${skipped_checks}")" -expected_count="$(line_count "${expected_entries}")" -unexpected_count="$(line_count "${unexpected_entries}")" -stale_count="$(line_count "${stale_entries}")" -upstream_count="$(line_count "${upstream_matches}")" - -print_row "${green}" "PASS" "${pass_count} scored checks passed" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - print_row "${yellow}" "XFAIL" "${key} ${dim}(expected failure reproduced)${reset}" -done < "${expected_entries}" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - print_row "${cyan}" "UPSTREAM" "${key} ${dim}(ignored pinned-fixture finding)${reset}" -done < "${upstream_matches}" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - status="$(awk -F '\t' -v key="${key}" '$1 == key { print $2; exit }' "${actual_findings}")" - print_row "${red}" "FAIL" "${key} ${dim}(expected PASS, got ${status})${reset}" - emit_error "Expected conformance pass failed" "${key}" -done < "${unexpected_entries}" - -while IFS= read -r key; do - [ -n "${key}" ] || continue - print_row "${red}" "XPASS" "${key} ${dim}(expected FAILURE, got PASS)${reset}" - emit_error "Expected conformance failure passed" "${key}" -done < "${stale_entries}" - -if [ "${skip_count}" -gt 0 ]; then - print_row "${dim}" "SKIP" "${skip_count} scored checks skipped" -fi - -if ${bless}; then - if ${bless_changed}; then - print_row "${green}" "BLESS" "updated ${baseline_file}" - else - print_row "${green}" "BLESS" "${baseline_file} was already current" - fi -fi - -printf '\n%bSummary%b: %s passed, %s expected failures, %s upstream findings ignored, %s failed, %s unexpected passes\n' \ - "${bold}" "${reset}" \ - "${pass_count}" "${expected_count}" "${upstream_count}" "${unexpected_count}" "${stale_count}" - -if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - { - echo "## MCP ${spec_version} conformance" - echo - echo '| Outcome | Count |' - echo '| --- | ---: |' - echo "| Scored checks passed | ${pass_count} |" - echo "| Expected failures reproduced | ${expected_count} |" - echo "| Pinned fixture findings ignored | ${upstream_count} |" - echo "| Expected pass, got failure | ${unexpected_count} |" - echo "| Expected failure, got pass | ${stale_count} |" - echo "| Skipped checks | ${skip_count} |" - - if [ "${unexpected_count}" -gt 0 ]; then - echo - echo '### Expected pass, got failure' - while IFS= read -r key; do - [ -n "${key}" ] || continue - status="$(awk -F '\t' -v key="${key}" '$1 == key { print $2; exit }' "${actual_findings}")" - echo "- \`${key}\` — ${status}" - done < "${unexpected_entries}" - fi - - if [ "${stale_count}" -gt 0 ]; then - echo - echo '### Expected failure, got pass' - while IFS= read -r key; do - [ -n "${key}" ] || continue - echo "- \`${key}\`" - done < "${stale_entries}" - fi - - echo - if ${bless}; then - echo "✅ Expected-failure baseline updated with ${expected_count} dataplane findings." - elif [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then - echo '✅ Actual dataplane findings match the expected-failure baseline.' - else - echo '❌ Actual dataplane findings do not match the expected-failure baseline.' - fi - } >> "${GITHUB_STEP_SUMMARY}" -fi - -if ${bless}; then - exit 0 -fi - -if [ "${unexpected_count}" -gt 0 ] || [ "${stale_count}" -gt 0 ]; then - exit 1 -fi diff --git a/tests/conformance/resolve-control-plane-image.sh b/tests/conformance/resolve-control-plane-image.sh deleted file mode 100755 index c1b87cd7..00000000 --- a/tests/conformance/resolve-control-plane-image.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repository="IBM/mcp-context-forge" -image_repository="ghcr.io/ibm/mcp-context-forge" -api_url="https://api.github.com/repos/${repository}/commits?sha=main&per_page=100" -curl_args=( - --fail - --silent - --show-error - --retry 3 - --retry-all-errors - --header "Accept: application/vnd.github+json" - --header "X-GitHub-Api-Version: 2022-11-28" -) -if [ -n "${GITHUB_TOKEN:-}" ]; then - curl_args+=(--header "Authorization: Bearer ${GITHUB_TOKEN}") -fi - -commit_shas="$(curl "${curl_args[@]}" "${api_url}" | jq --exit-status --raw-output \ - '.[] | .sha | select(test("^[0-9a-f]{40}$"))')" - -while IFS= read -r commit_sha; do - image="${image_repository}:${commit_sha}" - if docker manifest inspect "${image}" > /dev/null 2>&1; then - echo "Resolved latest control-plane main image: ${image}" >&2 - echo "${image}" - exit 0 - fi -done <<< "${commit_shas}" - -echo "No published control-plane image found in the latest 100 main commits" >&2 -exit 1 diff --git a/tests/conformance/run-client-conformance.sh b/tests/conformance/run-client-conformance.sh deleted file mode 100755 index 289c2d41..00000000 --- a/tests/conformance/run-client-conformance.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${MCP_CONFORMANCE_SPEC_VERSION:?MCP_CONFORMANCE_SPEC_VERSION must be set}" -: "${MCP_CONFORMANCE_SUBJECT:?MCP_CONFORMANCE_SUBJECT must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -results_root="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" -results_dir="${MCP_CONFORMANCE_CLIENT_RESULTS_DIR:-${results_root}/client}" -compose_file="${script_dir}/docker-compose.yml" -baseline_file="${script_dir}/client-expected-failures.yml" -client_command="${script_dir}/client-under-test.sh" -scenarios=(tools_call request-metadata http-standard-headers http-custom-headers) - -mkdir -p "${results_dir}" - -# The control-plane publisher would overwrite the scenario-specific Redis -# config, and its own backend probes would contaminate the client observations. -docker compose -f "${compose_file}" stop control-plane > /dev/null - -status=0 -for scenario in "${scenarios[@]}"; do - set +e - ( - cd "${suite_dir}" - npm start -- \ - client \ - --command "${client_command}" \ - --scenario "${scenario}" \ - --spec-version "${MCP_CONFORMANCE_SPEC_VERSION}" \ - --expected-failures "${baseline_file}" \ - --timeout 60000 \ - --output-dir "${results_dir}" - ) - scenario_status="$?" - set -e - if [ "${scenario_status}" -ne 0 ]; then - status=1 - fi -done - -exit "${status}" diff --git a/tests/conformance/run-conformance.sh b/tests/conformance/run-conformance.sh deleted file mode 100755 index 93a8f740..00000000 --- a/tests/conformance/run-conformance.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}" -: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" -: "${MCP_CONFORMANCE_SPEC_VERSION:?MCP_CONFORMANCE_SPEC_VERSION must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -conformance_port="${MCP_CONFORMANCE_PORT:-8080}" -results_root="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" -results_dir="${MCP_CONFORMANCE_SERVER_RESULTS_DIR:-${results_root}/server}" - -mkdir -p "${results_dir}" - -set +e -( - cd "${suite_dir}" - npm start -- \ - server \ - --url "http://127.0.0.1:${conformance_port}/servers/${MCP_CONFORMANCE_SERVER_ID}/mcp" \ - --requirements "${MCP_CONFORMANCE_SPEC_VERSION}" \ - --expected-failures "${script_dir}/expected-failures.yml" \ - --output-dir "${results_dir}" -) -runner_status="$?" -set -e - -echo "status=${runner_status}" >> "${GITHUB_OUTPUT}" diff --git a/tests/conformance/run-local.sh b/tests/conformance/run-local.sh deleted file mode 100755 index ea26ab0e..00000000 --- a/tests/conformance/run-local.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -compose_file="${script_dir}/docker-compose.yml" - -export MCP_CONFORMANCE_VERSION="${MCP_CONFORMANCE_VERSION:-0.2.0-alpha.11}" -export MCP_CONFORMANCE_SOURCE_SHA="${MCP_CONFORMANCE_SOURCE_SHA:-c321dd32035556e6769d3724a8ee97d87c3faaac}" -export MCP_CONFORMANCE_SPEC_VERSION="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}" -export MCP_CONFORMANCE_SERVER_ID="${MCP_CONFORMANCE_SERVER_ID:-3f33286667d34b65a31c3bafd30e4c21}" -export MCP_CONFORMANCE_SUITE_DIR="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -export CF_DATAPLANE_IMAGE="${CF_DATAPLANE_IMAGE:-contextforge-data-plane:conformance}" -export MCP_CONFORMANCE_COLOR="${MCP_CONFORMANCE_COLOR:-auto}" - -for command in curl docker git jq node npm; do - if ! command -v "${command}" > /dev/null 2>&1; then - echo "Required command not found: ${command}" >&2 - exit 1 - fi -done -docker compose version > /dev/null - -if [ -z "${CF_CONTROLPLANE_IMAGE:-}" ]; then - CF_CONTROLPLANE_IMAGE="$("${script_dir}/resolve-control-plane-image.sh")" - export CF_CONTROLPLANE_IMAGE -fi - -if [ -e "${MCP_CONFORMANCE_SUITE_DIR}" ] && [ ! -d "${MCP_CONFORMANCE_SUITE_DIR}/.git" ]; then - echo "MCP_CONFORMANCE_SUITE_DIR is not a git checkout: ${MCP_CONFORMANCE_SUITE_DIR}" >&2 - exit 1 -fi - -if [ ! -d "${MCP_CONFORMANCE_SUITE_DIR}/.git" ]; then - echo "Checking out the official conformance suite." - git clone --filter=blob:none \ - https://github.com/modelcontextprotocol/conformance.git \ - "${MCP_CONFORMANCE_SUITE_DIR}" - git -C "${MCP_CONFORMANCE_SUITE_DIR}" checkout --detach "${MCP_CONFORMANCE_SOURCE_SHA}" -fi - -suite_sha="$(git -C "${MCP_CONFORMANCE_SUITE_DIR}" rev-parse HEAD)" -if [ "${suite_sha}" != "${MCP_CONFORMANCE_SOURCE_SHA}" ]; then - echo "Conformance checkout is at ${suite_sha}; expected ${MCP_CONFORMANCE_SOURCE_SHA}." >&2 - echo "Use a checkout at the pinned commit or set MCP_CONFORMANCE_SUITE_DIR." >&2 - exit 1 -fi - -( - echo "Installing official conformance dependencies." - cd "${MCP_CONFORMANCE_SUITE_DIR}" - test "$(node -p "require('./package.json').version")" = "${MCP_CONFORMANCE_VERSION}" - npm ci --ignore-scripts -) - -state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-conformance.XXXXXX")" -export GITHUB_ENV="${state_dir}/github-env" -export GITHUB_OUTPUT="${state_dir}/github-output" -touch "${GITHUB_ENV}" "${GITHUB_OUTPUT}" -mkdir -p "${repo_root}/conformance-results" -export MCP_CONFORMANCE_RESULTS_DIR -MCP_CONFORMANCE_RESULTS_DIR="$(mktemp -d "${repo_root}/conformance-results/run.XXXXXX")" - -# shellcheck disable=SC2329 # Invoked by the trap below. -cleanup() { - local status="$?" - trap - EXIT INT TERM - if [ "${status}" -ne 0 ]; then - echo "Conformance run failed; printing live stack logs." >&2 - MCP_CONFORMANCE_TOKEN=diagnostics-only \ - docker compose -f "${compose_file}" logs --no-color || true - if [ -f "${repo_root}/conformance-logs/reference-server.log" ]; then - echo "Official fixture log:" >&2 - sed -n '1,240p' "${repo_root}/conformance-logs/reference-server.log" >&2 - fi - fi - MCP_CONFORMANCE_TOKEN="${MCP_CONFORMANCE_TOKEN:-cleanup-only}" \ - "${script_dir}/stop-live-stack.sh" || true - rm -f -- "${GITHUB_ENV}" "${GITHUB_OUTPUT}" - rmdir -- "${state_dir}" - exit "${status}" -} -trap cleanup EXIT INT TERM - -MCP_CONFORMANCE_TOKEN=pull-only \ - docker compose -f "${compose_file}" pull redis fixture-proxy control-plane nginx -echo "Starting the fixture and control plane." -MCP_CONFORMANCE_TOKEN=bootstrap-only \ - "${script_dir}/start-fixture-and-control-plane.sh" -echo "Registering the fixture through the control plane." -MCP_CONFORMANCE_TOKEN=bootstrap-only \ - "${script_dir}/register-fixture.sh" - -set -a -# shellcheck disable=SC1090 -source "${GITHUB_ENV}" -set +a - -echo "Starting the dataplane and nginx." -"${script_dir}/start-dataplane-and-nginx.sh" -echo "Running MCP ${MCP_CONFORMANCE_SPEC_VERSION} conformance." -"${script_dir}/run-conformance.sh" - -set +e -echo "Running scoped MCP ${MCP_CONFORMANCE_SPEC_VERSION} client conformance." -"${script_dir}/run-client-conformance.sh" -client_status="$?" -set -e - -runner_status="$(sed -n 's/^status=//p' "${GITHUB_OUTPUT}" | tail -n 1)" -if [ -z "${runner_status}" ]; then - echo "Conformance runner did not report a status." >&2 - exit 1 -fi - -set +e -if [ "${MCP_CONFORMANCE_BLESS:-false}" = "true" ]; then - "${script_dir}/report-baseline-diff.sh" --bless "${MCP_CONFORMANCE_RESULTS_DIR}/server" - report_status="$?" - "${script_dir}/bless-client-baseline.sh" \ - "${MCP_CONFORMANCE_RESULTS_DIR}/client" \ - "${script_dir}/client-expected-failures.yml" - client_baseline_status="$?" - if [ "${client_baseline_status}" -eq 0 ]; then - client_status=0 - fi -else - "${script_dir}/report-baseline-diff.sh" "${MCP_CONFORMANCE_RESULTS_DIR}/server" - report_status="$?" - client_baseline_status=0 -fi -set -e - -if [ "${runner_status}" -ne 0 ] && [ "${report_status}" -eq 0 ]; then - echo "Official runner status ${runner_status} contained no dataplane baseline mismatch." -fi -if [ "${client_baseline_status}" -ne 0 ]; then - exit "${client_baseline_status}" -fi -if [ "${client_status}" -ne 0 ]; then - exit "${client_status}" -fi -exit "${report_status}" diff --git a/tests/conformance/start-dataplane-and-nginx.sh b/tests/conformance/start-dataplane-and-nginx.sh deleted file mode 100755 index c2a793e2..00000000 --- a/tests/conformance/start-dataplane-and-nginx.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -log_dir="${repo_root}/conformance-logs" -conformance_port="${MCP_CONFORMANCE_PORT:-8080}" - -docker compose -f "${script_dir}/docker-compose.yml" \ - up -d --wait data-plane nginx - -endpoint="http://127.0.0.1:${conformance_port}/servers/${MCP_CONFORMANCE_SERVER_ID}/mcp" -request='{ - "jsonrpc": "2.0", - "id": 1, - "method": "server/discover", - "params": { - "_meta": { - "io.modelcontextprotocol/protocolVersion": "2026-07-28", - "io.modelcontextprotocol/clientInfo": { - "name": "ci-route-probe", - "version": "1.0.0" - }, - "io.modelcontextprotocol/clientCapabilities": {} - } - } -}' - -for _ in $(seq 1 120); do - curl --silent --show-error \ - --dump-header "${log_dir}/route-probe-headers.txt" \ - --output "${log_dir}/route-probe-body.txt" \ - --request POST \ - --header 'Content-Type: application/json' \ - --header 'Accept: application/json, text/event-stream' \ - --header 'MCP-Protocol-Version: 2026-07-28' \ - --header 'MCP-Method: server/discover' \ - --data "${request}" \ - "${endpoint}" || true - if grep --ignore-case --quiet '^X-CF-Conformance-Backend: dataplane' \ - "${log_dir}/route-probe-headers.txt" \ - && sed -n 's/^data: //p' "${log_dir}/route-probe-body.txt" \ - | jq --exit-status \ - '.result.supportedVersions | index("2026-07-28") != null' \ - > /dev/null 2>&1; then - exit 0 - fi - sleep 0.5 -done - -echo "Modern MCP route did not reach the dataplane through nginx" >&2 -cat "${log_dir}/route-probe-headers.txt" >&2 -cat "${log_dir}/route-probe-body.txt" >&2 -exit 1 diff --git a/tests/conformance/start-fixture-and-control-plane.sh b/tests/conformance/start-fixture-and-control-plane.sh deleted file mode 100755 index 667d8f75..00000000 --- a/tests/conformance/start-fixture-and-control-plane.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" -log_dir="${repo_root}/conformance-logs" - -mkdir -p "${log_dir}" "${repo_root}/conformance-results" - -( - cd "${suite_dir}" - PORT=3000 npm exec -- tsx examples/servers/typescript/everything-server.ts -) > "${log_dir}/reference-server.log" 2>&1 & -echo "$!" > "${log_dir}/reference-server.pid" - -docker compose -f "${script_dir}/docker-compose.yml" \ - up -d --wait redis fixture-proxy control-plane - -for _ in $(seq 1 120); do - if curl --silent --output /dev/null http://127.0.0.1:3000/mcp; then - exit 0 - fi - sleep 0.25 -done - -echo "Timed out waiting for the official conformance fixture" >&2 -exit 1 diff --git a/tests/conformance/stop-live-stack.sh b/tests/conformance/stop-live-stack.sh deleted file mode 100755 index e88cf942..00000000 --- a/tests/conformance/stop-live-stack.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "${script_dir}/../.." && pwd)" -pid_file="${repo_root}/conformance-logs/reference-server.pid" - -if [ -f "${pid_file}" ]; then - kill "$(cat "${pid_file}")" 2>/dev/null || true -fi - -docker compose -f "${script_dir}/docker-compose.yml" \ - down --volumes --remove-orphans diff --git a/tests/conformance/upstream-fixture-failures.yml b/tests/conformance/upstream-fixture-failures.yml deleted file mode 100644 index 08239bda..00000000 --- a/tests/conformance/upstream-fixture-failures.yml +++ /dev/null @@ -1,14 +0,0 @@ -# The pinned alpha.11 JavaScript fixture reports 129 passed and 54 failed when -# the official runner targets it directly at MCP 2026-07-28, plus one scored -# warning. Of those failures, 47 belong to extension or pending scenarios and -# are already unscored by the requirements file. These are its seven scored -# failures and one warning. Update this file with MCP_CONFORMANCE_SOURCE_SHA. -server: - - tools-call-simple-text:wire-schema-valid - - tools-call-image:wire-schema-valid - - tools-call-audio:wire-schema-valid - - tools-call-embedded-resource:wire-schema-valid - - tools-call-mixed-content:wire-schema-valid - - tools-call-error:wire-schema-valid - - tools-call-with-progress:wire-schema-valid - - input-required-result-validate-input:sep-2322-validate-input-responses diff --git a/tests/conformance/write_client_config.py b/tests/conformance/write_client_config.py deleted file mode 100755 index b31e8b11..00000000 --- a/tests/conformance/write_client_config.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env python3 -"""Publish an isolated upstream-client conformance route for the dataplane.""" - -from __future__ import annotations - -import argparse -import json -import os -from urllib.parse import urlparse - -import msgpack -import redis - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("subject") - parser.add_argument("virtual_host_id") - parser.add_argument("backend_url") - parser.add_argument("tool_names_json") - return parser.parse_args() - - -def main() -> None: - args = parse_args() - redis_url = os.environ.get("REDIS_URL") - if not redis_url: - raise SystemExit("REDIS_URL is required") - - parsed_url = urlparse(args.backend_url) - if parsed_url.scheme not in {"http", "https"} or not parsed_url.hostname: - raise SystemExit("backend_url must be an absolute HTTP(S) URL") - - tool_names = json.loads(args.tool_names_json) - if ( - not isinstance(tool_names, list) - or not tool_names - or not all(isinstance(name, str) and name for name in tool_names) - ): - raise SystemExit("tool_names_json must be a non-empty JSON string array") - - backend_name = "conformance-backend" - config = { - "virtual_hosts": { - args.virtual_host_id: { - "backends": { - backend_name: { - "name": backend_name, - "url": args.backend_url, - "passthrough_headers": [], - "add_headers": {}, - "remove_headers": [], - "allowed_tool_names": tool_names, - "tool_schemas": {name: {} for name in tool_names}, - "tool_name_aliases": {}, - "allowed_resource_names": [], - "allowed_prompt_names": [], - } - } - } - } - } - - key = msgpack.dumps(("UserConfig", args.subject), use_bin_type=True) - value = msgpack.dumps(config, use_bin_type=True) - client = redis.Redis.from_url(redis_url, decode_responses=False) - client.set(key, value, ex=600) - - -if __name__ == "__main__": - main()