diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml index 5efd79c1d..dbaf2144e 100644 --- a/.github/.licenserc.yaml +++ b/.github/.licenserc.yaml @@ -51,6 +51,10 @@ header: - 'AGENTS.md' - 'dokimi/data/agent_profiles/**' - 'dokimi/data/skills/**' + # Reach conformance fixtures: the CLAUDE.md and AGENTS.override.md files a harness loads verbatim (AGENTS.md is + # ignored above, at any depth). + - 'dokimi/data/reach-conformance/**/CLAUDE.md' + - 'dokimi/data/reach-conformance/**/AGENTS.override.md' # Eval oracles: patches applied to the workspace and reference answers # written verbatim as the agent's answer (.agent-eval/run.py). - '.agent-eval/tasks/**/*.patch' diff --git a/.github/workflows/reach-matrix.yml b/.github/workflows/reach-matrix.yml new file mode 100644 index 000000000..38a7dd9e1 --- /dev/null +++ b/.github/workflows/reach-matrix.yml @@ -0,0 +1,215 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: reach-matrix + +# Paid model calls, so never a PR check; a harness without a model, credential or CLI is skipped, not failed. +# Design and secrets: dokimi/docs/reach-matrix.md. +on: + schedule: + - cron: '37 4 * * *' + workflow_dispatch: + +permissions: {} + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + claude-code: + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' + - name: Check the credential + id: creds + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + run: | + if [ -n "$ANTHROPIC_API_KEY" ]; then backend=anthropic + elif [ -n "$AWS_BEARER_TOKEN_BEDROCK" ]; then backend=bedrock + else backend=none; fi + echo "backend=$backend" >> "$GITHUB_OUTPUT" + if [ "$backend" = none ]; then echo "present=false" >> "$GITHUB_OUTPUT"; else echo "present=true" >> "$GITHUB_OUTPUT"; fi + - name: Install the CLI + if: steps.creds.outputs.present == 'true' && vars.REACH_MODEL_CLAUDE_CODE != '' + run: npm install -g @anthropic-ai/claude-code@2.1.316 + - name: Run reach over the fixtures + working-directory: dokimi + env: + PYTHONPATH: . + REACH_MODEL: ${{ vars.REACH_MODEL_CLAUDE_CODE }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + CLAUDE_CODE_USE_BEDROCK: ${{ steps.creds.outputs.backend == 'bedrock' && '1' || '' }} + AWS_REGION: ${{ vars.AWS_REGION }} + run: python scripts/reach_conformance.py run --harness "claude-code=$REACH_MODEL" --out reach-out + - name: Upload the matrix + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix-claude-code + path: dokimi/reach-out/claude-code.matrix.json + if-no-files-found: warn + retention-days: 30 + + codex: + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' + - name: Check the credential + id: creds + env: + CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} + run: | + if [ -n "$CODEX_API_KEY" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi + - name: Install the CLI + if: steps.creds.outputs.present == 'true' && vars.REACH_MODEL_CODEX != '' + run: npm install -g @openai/codex@0.166.0 + - name: Run reach over the fixtures + working-directory: dokimi + env: + PYTHONPATH: . + REACH_MODEL: ${{ vars.REACH_MODEL_CODEX }} + CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} + run: python scripts/reach_conformance.py run --harness "codex=$REACH_MODEL" --out reach-out + - name: Upload the matrix + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix-codex + path: dokimi/reach-out/codex.matrix.json + if-no-files-found: warn + retention-days: 30 + + copilot: + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 120 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '22' + - name: Check the credential + id: creds + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + run: | + if [ -n "$COPILOT_GITHUB_TOKEN" ]; then echo "present=true" >> "$GITHUB_OUTPUT"; else echo "present=false" >> "$GITHUB_OUTPUT"; fi + - name: Install the CLI + if: steps.creds.outputs.present == 'true' && vars.REACH_MODEL_COPILOT != '' + run: npm install -g @github/copilot@1.0.105 + - name: Run reach over the fixtures + working-directory: dokimi + env: + PYTHONPATH: . + REACH_MODEL: ${{ vars.REACH_MODEL_COPILOT }} + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + run: python scripts/reach_conformance.py run --harness "copilot=$REACH_MODEL" --out reach-out + - name: Upload the matrix + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix-copilot + path: dokimi/reach-out/copilot.matrix.json + if-no-files-found: warn + retention-days: 30 + + matrix: + needs: [claude-code, codex, copilot] + if: always() + permissions: + contents: read + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: dokimi/requirements.lock.txt + - name: Install dokimi + working-directory: dokimi + run: pip install --require-hashes -r requirements.lock.txt + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: reach-matrix-* + merge-multiple: true + path: dokimi/reach-in + - name: Merge the matrices + working-directory: dokimi + env: + PYTHONPATH: . + run: | + missing=0 + for harness in claude-code codex copilot; do + if [ ! -f "reach-in/$harness.matrix.json" ]; then + echo "::error::no $harness matrix: its job failed before writing one" + missing=1 + fi + done + [ "$missing" -eq 0 ] || exit 1 + python scripts/reach_conformance.py merge reach-in/*.matrix.json \ + --out reach-out/reach-matrix.json --md reach-out/reach-matrix.md + cat reach-out/reach-matrix.md >> "$GITHUB_STEP_SUMMARY" + - name: Upload the matrix page + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: reach-matrix + path: | + dokimi/reach-out/reach-matrix.json + dokimi/reach-out/reach-matrix.md + if-no-files-found: error + retention-days: 90 diff --git a/dokimi/benchmarks/spec.v1.yaml b/dokimi/benchmarks/spec.v1.yaml index b36efa758..753c06a0d 100644 --- a/dokimi/benchmarks/spec.v1.yaml +++ b/dokimi/benchmarks/spec.v1.yaml @@ -76,7 +76,7 @@ future: # that matched almost nothing, max_docs one that escaped; raise it with each new doc. golden_manifest: min_docs: 10 - max_docs: 37 + max_docs: 38 include: - docs/**/*.md exclude: diff --git a/dokimi/changelog.d/20261031_160000_napetrov_reach-matrix.md b/dokimi/changelog.d/20261031_160000_napetrov_reach-matrix.md new file mode 100644 index 000000000..14892b22e --- /dev/null +++ b/dokimi/changelog.d/20261031_160000_napetrov_reach-matrix.md @@ -0,0 +1,8 @@ + + +### Added + +- **A reach conformance suite and a nightly agent × version matrix** — `data/reach-conformance/` holds eight fixture repositories (root, nested, `CLAUDE.md` with `AGENTS.md`, a Claude Code `@` import, an `AGENTS.override.md`, a subdirectory shadow, a file over the size cap, a symbolic link), and `expectations.yaml` (`reach_conformance.v1`) gives what each harness's vendor documentation says each file loads in each scenario, or `unknown`. `scripts/reach_conformance.py` checks the expectations against the fixtures, runs `repo reach --format receipt` over them for each harness with a model, credential and CLI present (the rest are recorded as skipped), and merges the per-harness `reach_matrix.v1` JSON into one matrix and one markdown page. `.github/workflows/reach-matrix.yml` runs it nightly and on dispatch, not as a pull-request check, and uploads the matrix as an artifact. The design, the agreement rules and the limits are in `docs/reach-matrix.md`. diff --git a/dokimi/data/reach-conformance/claude-and-agents/AGENTS.md b/dokimi/data/reach-conformance/claude-and-agents/AGENTS.md new file mode 100644 index 000000000..ea44db060 --- /dev/null +++ b/dokimi/data/reach-conformance/claude-and-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +These instructions are for the other coding agents. diff --git a/dokimi/data/reach-conformance/claude-and-agents/CLAUDE.md b/dokimi/data/reach-conformance/claude-and-agents/CLAUDE.md new file mode 100644 index 000000000..62571eb4a --- /dev/null +++ b/dokimi/data/reach-conformance/claude-and-agents/CLAUDE.md @@ -0,0 +1,3 @@ +# Claude instructions + +These instructions are for Claude Code. diff --git a/dokimi/data/reach-conformance/claude-imports-agents/AGENTS.md b/dokimi/data/reach-conformance/claude-imports-agents/AGENTS.md new file mode 100644 index 000000000..9806e4047 --- /dev/null +++ b/dokimi/data/reach-conformance/claude-imports-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +Shared rules for every coding agent. diff --git a/dokimi/data/reach-conformance/claude-imports-agents/CLAUDE.md b/dokimi/data/reach-conformance/claude-imports-agents/CLAUDE.md new file mode 100644 index 000000000..8dc3d3719 --- /dev/null +++ b/dokimi/data/reach-conformance/claude-imports-agents/CLAUDE.md @@ -0,0 +1,5 @@ +# Claude instructions + +Shared rules live in AGENTS.md. + +@AGENTS.md diff --git a/dokimi/data/reach-conformance/expectations.yaml b/dokimi/data/reach-conformance/expectations.yaml new file mode 100644 index 000000000..bd9e11cac --- /dev/null +++ b/dokimi/data/reach-conformance/expectations.yaml @@ -0,0 +1,117 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Vendor-documented state per file and reach scenario; unlisted means unknown (no claim). +# Sources and scenario meanings: docs/reach-matrix.md. +schema: reach_conformance.v1 +cases: + root-agents: + summary: AGENTS.md at the repository root, and nothing else. + claude-code: + why: With no CLAUDE.md in the working directory or above, Claude Code reads AGENTS.md. + files: + AGENTS.md: {"cwd=.": loaded} + codex: + why: Codex checks AGENTS.override.md then AGENTS.md in each directory from the git root to cwd. + files: + AGENTS.md: {"cwd=.": loaded} + + nested-agents: + summary: A root AGENTS.md and a packages/app/AGENTS.md below it. + claude-code: + why: >- + Every AGENTS.md in the working directory and above loads at start-up. A subdirectory's + AGENTS.md loads when Claude reads a file there with the Read tool. + files: + AGENTS.md: {"cwd=.": loaded, "cwd=packages/app": loaded, "read=packages/app": loaded} + packages/app/AGENTS.md: {"cwd=.": not_loaded, "cwd=packages/app": loaded, "read=packages/app": loaded} + codex: + why: >- + Codex walks from the git root down to cwd, one file per directory. A directory off that path + is not read; the docs do not describe a read-triggered load, so read= is unknown. + files: + AGENTS.md: {"cwd=.": loaded, "cwd=packages/app": loaded} + packages/app/AGENTS.md: {"cwd=.": not_loaded, "cwd=packages/app": loaded} + + claude-and-agents: + summary: A root CLAUDE.md beside a root AGENTS.md. + claude-code: + why: >- + AGENTS.md is read only when there is no CLAUDE.md in the working directory or above; the + default setting (claude-md-or-agents-md) reads the CLAUDE.md files alone. + files: + CLAUDE.md: {"cwd=.": loaded} + AGENTS.md: {"cwd=.": not_loaded} + codex: + why: >- + Codex checks AGENTS.override.md, AGENTS.md and the project_doc_fallback_filenames list only, + and CLAUDE.md is not on that list by default. + files: + AGENTS.md: {"cwd=.": loaded} + CLAUDE.md: {"cwd=.": not_loaded} + + claude-imports-agents: + summary: A root CLAUDE.md that imports AGENTS.md with @AGENTS.md. + claude-code: + why: CLAUDE.md @path imports are expanded and loaded with the file that references them. + files: + CLAUDE.md: {"cwd=.": loaded} + AGENTS.md: {"cwd=.": loaded} + codex: + why: Codex does not read CLAUDE.md by default, so the import is never followed. + files: + AGENTS.md: {"cwd=.": loaded} + CLAUDE.md: {"cwd=.": not_loaded} + + override-file: + summary: A root AGENTS.md beside an AGENTS.override.md. + claude-code: + why: The Claude Code AGENTS.md rules list AGENTS.override.md among the files it does not read. + files: + AGENTS.md: {"cwd=.": loaded} + AGENTS.override.md: {"cwd=.": not_loaded} + codex: + why: >- + In each directory AGENTS.override.md is used in place of AGENTS.md, and at most one of the + two is read. + files: + AGENTS.override.md: {"cwd=.": loaded} + AGENTS.md: {"cwd=.": not_loaded} + + subtree-claude-shadow: + summary: sub/AGENTS.md and sub/CLAUDE.md in the same subdirectory. + claude-code: + why: >- + A CLAUDE.md in the working directory loads at start-up. AGENTS.md is not read in a directory + that has a CLAUDE.md of its own. + files: + sub/CLAUDE.md: {"cwd=.": not_loaded, "cwd=sub": loaded, "read=sub": loaded} + sub/AGENTS.md: {"cwd=.": not_loaded, "cwd=sub": not_loaded, "read=sub": not_loaded} + codex: + why: Codex reads sub/AGENTS.md when cwd is sub, and never reads CLAUDE.md by default. + files: + sub/AGENTS.md: {"cwd=.": not_loaded, "cwd=sub": loaded} + sub/CLAUDE.md: {"cwd=.": not_loaded, "cwd=sub": not_loaded} + + large-file: + summary: A root AGENTS.md of about 40 KB, above Codex's 32 KiB default. + claude-code: + why: >- + No size limit is documented. The memory page advises under 200 lines per file, which is a + target and not a cut-off. So the state is unknown. + files: {} + codex: + why: >- + project_doc_max_bytes (default 32 KiB) caps the combined size of the chain. The docs do not + say whether a file that crosses the cap is cut, so the state is unknown. + files: {} + + symlinked-file: + summary: CLAUDE.md is a symbolic link to AGENTS.md. + claude-code: + why: The docs do not say how a symbolic link named CLAUDE.md is read, so the state is unknown. + files: {} + codex: + why: Codex reads AGENTS.md itself, and the link's name is not one it checks. + files: + AGENTS.md: {"cwd=.": loaded} diff --git a/dokimi/data/reach-conformance/guidance.yaml b/dokimi/data/reach-conformance/guidance.yaml new file mode 100644 index 000000000..f53cef4ff --- /dev/null +++ b/dokimi/data/reach-conformance/guidance.yaml @@ -0,0 +1,13 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# The guidance-only manifest every conformance case is probed with (`repo reach --manifest-file`). +# AGENTS.override.md is named here so every harness probes it, not only the ones whose profile names it. +schema: repo_manifest.v1 +id: reach-conformance +guidance: + globs: + - "**/AGENTS.md" + - "**/AGENTS.override.md" + - "**/CLAUDE.md" + - "**/CLAUDE.local.md" diff --git a/dokimi/data/reach-conformance/large-file/AGENTS.md b/dokimi/data/reach-conformance/large-file/AGENTS.md new file mode 100644 index 000000000..56b874120 --- /dev/null +++ b/dokimi/data/reach-conformance/large-file/AGENTS.md @@ -0,0 +1,460 @@ +Filler line 1: padding for the size case, not an instruction, nothing to act on here. +Filler line 2: padding for the size case, not an instruction, nothing to act on here. +Filler line 3: padding for the size case, not an instruction, nothing to act on here. +Filler line 4: padding for the size case, not an instruction, nothing to act on here. +Filler line 5: padding for the size case, not an instruction, nothing to act on here. +Filler line 6: padding for the size case, not an instruction, nothing to act on here. +Filler line 7: padding for the size case, not an instruction, nothing to act on here. +Filler line 8: padding for the size case, not an instruction, nothing to act on here. +Filler line 9: padding for the size case, not an instruction, nothing to act on here. +Filler line 10: padding for the size case, not an instruction, nothing to act on here. +Filler line 11: padding for the size case, not an instruction, nothing to act on here. +Filler line 12: padding for the size case, not an instruction, nothing to act on here. +Filler line 13: padding for the size case, not an instruction, nothing to act on here. +Filler line 14: padding for the size case, not an instruction, nothing to act on here. +Filler line 15: padding for the size case, not an instruction, nothing to act on here. +Filler line 16: padding for the size case, not an instruction, nothing to act on here. +Filler line 17: padding for the size case, not an instruction, nothing to act on here. +Filler line 18: padding for the size case, not an instruction, nothing to act on here. +Filler line 19: padding for the size case, not an instruction, nothing to act on here. +Filler line 20: padding for the size case, not an instruction, nothing to act on here. +Filler line 21: padding for the size case, not an instruction, nothing to act on here. +Filler line 22: padding for the size case, not an instruction, nothing to act on here. +Filler line 23: padding for the size case, not an instruction, nothing to act on here. +Filler line 24: padding for the size case, not an instruction, nothing to act on here. +Filler line 25: padding for the size case, not an instruction, nothing to act on here. +Filler line 26: padding for the size case, not an instruction, nothing to act on here. +Filler line 27: padding for the size case, not an instruction, nothing to act on here. +Filler line 28: padding for the size case, not an instruction, nothing to act on here. +Filler line 29: padding for the size case, not an instruction, nothing to act on here. +Filler line 30: padding for the size case, not an instruction, nothing to act on here. +Filler line 31: padding for the size case, not an instruction, nothing to act on here. +Filler line 32: padding for the size case, not an instruction, nothing to act on here. +Filler line 33: padding for the size case, not an instruction, nothing to act on here. +Filler line 34: padding for the size case, not an instruction, nothing to act on here. +Filler line 35: padding for the size case, not an instruction, nothing to act on here. +Filler line 36: padding for the size case, not an instruction, nothing to act on here. +Filler line 37: padding for the size case, not an instruction, nothing to act on here. +Filler line 38: padding for the size case, not an instruction, nothing to act on here. +Filler line 39: padding for the size case, not an instruction, nothing to act on here. +Filler line 40: padding for the size case, not an instruction, nothing to act on here. +Filler line 41: padding for the size case, not an instruction, nothing to act on here. +Filler line 42: padding for the size case, not an instruction, nothing to act on here. +Filler line 43: padding for the size case, not an instruction, nothing to act on here. +Filler line 44: padding for the size case, not an instruction, nothing to act on here. +Filler line 45: padding for the size case, not an instruction, nothing to act on here. +Filler line 46: padding for the size case, not an instruction, nothing to act on here. +Filler line 47: padding for the size case, not an instruction, nothing to act on here. +Filler line 48: padding for the size case, not an instruction, nothing to act on here. +Filler line 49: padding for the size case, not an instruction, nothing to act on here. +Filler line 50: padding for the size case, not an instruction, nothing to act on here. +Filler line 51: padding for the size case, not an instruction, nothing to act on here. +Filler line 52: padding for the size case, not an instruction, nothing to act on here. +Filler line 53: padding for the size case, not an instruction, nothing to act on here. +Filler line 54: padding for the size case, not an instruction, nothing to act on here. +Filler line 55: padding for the size case, not an instruction, nothing to act on here. +Filler line 56: padding for the size case, not an instruction, nothing to act on here. +Filler line 57: padding for the size case, not an instruction, nothing to act on here. +Filler line 58: padding for the size case, not an instruction, nothing to act on here. +Filler line 59: padding for the size case, not an instruction, nothing to act on here. +Filler line 60: padding for the size case, not an instruction, nothing to act on here. +Filler line 61: padding for the size case, not an instruction, nothing to act on here. +Filler line 62: padding for the size case, not an instruction, nothing to act on here. +Filler line 63: padding for the size case, not an instruction, nothing to act on here. +Filler line 64: padding for the size case, not an instruction, nothing to act on here. +Filler line 65: padding for the size case, not an instruction, nothing to act on here. +Filler line 66: padding for the size case, not an instruction, nothing to act on here. +Filler line 67: padding for the size case, not an instruction, nothing to act on here. +Filler line 68: padding for the size case, not an instruction, nothing to act on here. +Filler line 69: padding for the size case, not an instruction, nothing to act on here. +Filler line 70: padding for the size case, not an instruction, nothing to act on here. +Filler line 71: padding for the size case, not an instruction, nothing to act on here. +Filler line 72: padding for the size case, not an instruction, nothing to act on here. +Filler line 73: padding for the size case, not an instruction, nothing to act on here. +Filler line 74: padding for the size case, not an instruction, nothing to act on here. +Filler line 75: padding for the size case, not an instruction, nothing to act on here. +Filler line 76: padding for the size case, not an instruction, nothing to act on here. +Filler line 77: padding for the size case, not an instruction, nothing to act on here. +Filler line 78: padding for the size case, not an instruction, nothing to act on here. +Filler line 79: padding for the size case, not an instruction, nothing to act on here. +Filler line 80: padding for the size case, not an instruction, nothing to act on here. +Filler line 81: padding for the size case, not an instruction, nothing to act on here. +Filler line 82: padding for the size case, not an instruction, nothing to act on here. +Filler line 83: padding for the size case, not an instruction, nothing to act on here. +Filler line 84: padding for the size case, not an instruction, nothing to act on here. +Filler line 85: padding for the size case, not an instruction, nothing to act on here. +Filler line 86: padding for the size case, not an instruction, nothing to act on here. +Filler line 87: padding for the size case, not an instruction, nothing to act on here. +Filler line 88: padding for the size case, not an instruction, nothing to act on here. +Filler line 89: padding for the size case, not an instruction, nothing to act on here. +Filler line 90: padding for the size case, not an instruction, nothing to act on here. +Filler line 91: padding for the size case, not an instruction, nothing to act on here. +Filler line 92: padding for the size case, not an instruction, nothing to act on here. +Filler line 93: padding for the size case, not an instruction, nothing to act on here. +Filler line 94: padding for the size case, not an instruction, nothing to act on here. +Filler line 95: padding for the size case, not an instruction, nothing to act on here. +Filler line 96: padding for the size case, not an instruction, nothing to act on here. +Filler line 97: padding for the size case, not an instruction, nothing to act on here. +Filler line 98: padding for the size case, not an instruction, nothing to act on here. +Filler line 99: padding for the size case, not an instruction, nothing to act on here. +Filler line 100: padding for the size case, not an instruction, nothing to act on here. +Filler line 101: padding for the size case, not an instruction, nothing to act on here. +Filler line 102: padding for the size case, not an instruction, nothing to act on here. +Filler line 103: padding for the size case, not an instruction, nothing to act on here. +Filler line 104: padding for the size case, not an instruction, nothing to act on here. +Filler line 105: padding for the size case, not an instruction, nothing to act on here. +Filler line 106: padding for the size case, not an instruction, nothing to act on here. +Filler line 107: padding for the size case, not an instruction, nothing to act on here. +Filler line 108: padding for the size case, not an instruction, nothing to act on here. +Filler line 109: padding for the size case, not an instruction, nothing to act on here. +Filler line 110: padding for the size case, not an instruction, nothing to act on here. +Filler line 111: padding for the size case, not an instruction, nothing to act on here. +Filler line 112: padding for the size case, not an instruction, nothing to act on here. +Filler line 113: padding for the size case, not an instruction, nothing to act on here. +Filler line 114: padding for the size case, not an instruction, nothing to act on here. +Filler line 115: padding for the size case, not an instruction, nothing to act on here. +Filler line 116: padding for the size case, not an instruction, nothing to act on here. +Filler line 117: padding for the size case, not an instruction, nothing to act on here. +Filler line 118: padding for the size case, not an instruction, nothing to act on here. +Filler line 119: padding for the size case, not an instruction, nothing to act on here. +Filler line 120: padding for the size case, not an instruction, nothing to act on here. +Filler line 121: padding for the size case, not an instruction, nothing to act on here. +Filler line 122: padding for the size case, not an instruction, nothing to act on here. +Filler line 123: padding for the size case, not an instruction, nothing to act on here. +Filler line 124: padding for the size case, not an instruction, nothing to act on here. +Filler line 125: padding for the size case, not an instruction, nothing to act on here. +Filler line 126: padding for the size case, not an instruction, nothing to act on here. +Filler line 127: padding for the size case, not an instruction, nothing to act on here. +Filler line 128: padding for the size case, not an instruction, nothing to act on here. +Filler line 129: padding for the size case, not an instruction, nothing to act on here. +Filler line 130: padding for the size case, not an instruction, nothing to act on here. +Filler line 131: padding for the size case, not an instruction, nothing to act on here. +Filler line 132: padding for the size case, not an instruction, nothing to act on here. +Filler line 133: padding for the size case, not an instruction, nothing to act on here. +Filler line 134: padding for the size case, not an instruction, nothing to act on here. +Filler line 135: padding for the size case, not an instruction, nothing to act on here. +Filler line 136: padding for the size case, not an instruction, nothing to act on here. +Filler line 137: padding for the size case, not an instruction, nothing to act on here. +Filler line 138: padding for the size case, not an instruction, nothing to act on here. +Filler line 139: padding for the size case, not an instruction, nothing to act on here. +Filler line 140: padding for the size case, not an instruction, nothing to act on here. +Filler line 141: padding for the size case, not an instruction, nothing to act on here. +Filler line 142: padding for the size case, not an instruction, nothing to act on here. +Filler line 143: padding for the size case, not an instruction, nothing to act on here. +Filler line 144: padding for the size case, not an instruction, nothing to act on here. +Filler line 145: padding for the size case, not an instruction, nothing to act on here. +Filler line 146: padding for the size case, not an instruction, nothing to act on here. +Filler line 147: padding for the size case, not an instruction, nothing to act on here. +Filler line 148: padding for the size case, not an instruction, nothing to act on here. +Filler line 149: padding for the size case, not an instruction, nothing to act on here. +Filler line 150: padding for the size case, not an instruction, nothing to act on here. +Filler line 151: padding for the size case, not an instruction, nothing to act on here. +Filler line 152: padding for the size case, not an instruction, nothing to act on here. +Filler line 153: padding for the size case, not an instruction, nothing to act on here. +Filler line 154: padding for the size case, not an instruction, nothing to act on here. +Filler line 155: padding for the size case, not an instruction, nothing to act on here. +Filler line 156: padding for the size case, not an instruction, nothing to act on here. +Filler line 157: padding for the size case, not an instruction, nothing to act on here. +Filler line 158: padding for the size case, not an instruction, nothing to act on here. +Filler line 159: padding for the size case, not an instruction, nothing to act on here. +Filler line 160: padding for the size case, not an instruction, nothing to act on here. +Filler line 161: padding for the size case, not an instruction, nothing to act on here. +Filler line 162: padding for the size case, not an instruction, nothing to act on here. +Filler line 163: padding for the size case, not an instruction, nothing to act on here. +Filler line 164: padding for the size case, not an instruction, nothing to act on here. +Filler line 165: padding for the size case, not an instruction, nothing to act on here. +Filler line 166: padding for the size case, not an instruction, nothing to act on here. +Filler line 167: padding for the size case, not an instruction, nothing to act on here. +Filler line 168: padding for the size case, not an instruction, nothing to act on here. +Filler line 169: padding for the size case, not an instruction, nothing to act on here. +Filler line 170: padding for the size case, not an instruction, nothing to act on here. +Filler line 171: padding for the size case, not an instruction, nothing to act on here. +Filler line 172: padding for the size case, not an instruction, nothing to act on here. +Filler line 173: padding for the size case, not an instruction, nothing to act on here. +Filler line 174: padding for the size case, not an instruction, nothing to act on here. +Filler line 175: padding for the size case, not an instruction, nothing to act on here. +Filler line 176: padding for the size case, not an instruction, nothing to act on here. +Filler line 177: padding for the size case, not an instruction, nothing to act on here. +Filler line 178: padding for the size case, not an instruction, nothing to act on here. +Filler line 179: padding for the size case, not an instruction, nothing to act on here. +Filler line 180: padding for the size case, not an instruction, nothing to act on here. +Filler line 181: padding for the size case, not an instruction, nothing to act on here. +Filler line 182: padding for the size case, not an instruction, nothing to act on here. +Filler line 183: padding for the size case, not an instruction, nothing to act on here. +Filler line 184: padding for the size case, not an instruction, nothing to act on here. +Filler line 185: padding for the size case, not an instruction, nothing to act on here. +Filler line 186: padding for the size case, not an instruction, nothing to act on here. +Filler line 187: padding for the size case, not an instruction, nothing to act on here. +Filler line 188: padding for the size case, not an instruction, nothing to act on here. +Filler line 189: padding for the size case, not an instruction, nothing to act on here. +Filler line 190: padding for the size case, not an instruction, nothing to act on here. +Filler line 191: padding for the size case, not an instruction, nothing to act on here. +Filler line 192: padding for the size case, not an instruction, nothing to act on here. +Filler line 193: padding for the size case, not an instruction, nothing to act on here. +Filler line 194: padding for the size case, not an instruction, nothing to act on here. +Filler line 195: padding for the size case, not an instruction, nothing to act on here. +Filler line 196: padding for the size case, not an instruction, nothing to act on here. +Filler line 197: padding for the size case, not an instruction, nothing to act on here. +Filler line 198: padding for the size case, not an instruction, nothing to act on here. +Filler line 199: padding for the size case, not an instruction, nothing to act on here. +Filler line 200: padding for the size case, not an instruction, nothing to act on here. +Filler line 201: padding for the size case, not an instruction, nothing to act on here. +Filler line 202: padding for the size case, not an instruction, nothing to act on here. +Filler line 203: padding for the size case, not an instruction, nothing to act on here. +Filler line 204: padding for the size case, not an instruction, nothing to act on here. +Filler line 205: padding for the size case, not an instruction, nothing to act on here. +Filler line 206: padding for the size case, not an instruction, nothing to act on here. +Filler line 207: padding for the size case, not an instruction, nothing to act on here. +Filler line 208: padding for the size case, not an instruction, nothing to act on here. +Filler line 209: padding for the size case, not an instruction, nothing to act on here. +Filler line 210: padding for the size case, not an instruction, nothing to act on here. +Filler line 211: padding for the size case, not an instruction, nothing to act on here. +Filler line 212: padding for the size case, not an instruction, nothing to act on here. +Filler line 213: padding for the size case, not an instruction, nothing to act on here. +Filler line 214: padding for the size case, not an instruction, nothing to act on here. +Filler line 215: padding for the size case, not an instruction, nothing to act on here. +Filler line 216: padding for the size case, not an instruction, nothing to act on here. +Filler line 217: padding for the size case, not an instruction, nothing to act on here. +Filler line 218: padding for the size case, not an instruction, nothing to act on here. +Filler line 219: padding for the size case, not an instruction, nothing to act on here. +Filler line 220: padding for the size case, not an instruction, nothing to act on here. +Filler line 221: padding for the size case, not an instruction, nothing to act on here. +Filler line 222: padding for the size case, not an instruction, nothing to act on here. +Filler line 223: padding for the size case, not an instruction, nothing to act on here. +Filler line 224: padding for the size case, not an instruction, nothing to act on here. +Filler line 225: padding for the size case, not an instruction, nothing to act on here. +Filler line 226: padding for the size case, not an instruction, nothing to act on here. +Filler line 227: padding for the size case, not an instruction, nothing to act on here. +Filler line 228: padding for the size case, not an instruction, nothing to act on here. +Filler line 229: padding for the size case, not an instruction, nothing to act on here. +Filler line 230: padding for the size case, not an instruction, nothing to act on here. +Filler line 231: padding for the size case, not an instruction, nothing to act on here. +Filler line 232: padding for the size case, not an instruction, nothing to act on here. +Filler line 233: padding for the size case, not an instruction, nothing to act on here. +Filler line 234: padding for the size case, not an instruction, nothing to act on here. +Filler line 235: padding for the size case, not an instruction, nothing to act on here. +Filler line 236: padding for the size case, not an instruction, nothing to act on here. +Filler line 237: padding for the size case, not an instruction, nothing to act on here. +Filler line 238: padding for the size case, not an instruction, nothing to act on here. +Filler line 239: padding for the size case, not an instruction, nothing to act on here. +Filler line 240: padding for the size case, not an instruction, nothing to act on here. +Filler line 241: padding for the size case, not an instruction, nothing to act on here. +Filler line 242: padding for the size case, not an instruction, nothing to act on here. +Filler line 243: padding for the size case, not an instruction, nothing to act on here. +Filler line 244: padding for the size case, not an instruction, nothing to act on here. +Filler line 245: padding for the size case, not an instruction, nothing to act on here. +Filler line 246: padding for the size case, not an instruction, nothing to act on here. +Filler line 247: padding for the size case, not an instruction, nothing to act on here. +Filler line 248: padding for the size case, not an instruction, nothing to act on here. +Filler line 249: padding for the size case, not an instruction, nothing to act on here. +Filler line 250: padding for the size case, not an instruction, nothing to act on here. +Filler line 251: padding for the size case, not an instruction, nothing to act on here. +Filler line 252: padding for the size case, not an instruction, nothing to act on here. +Filler line 253: padding for the size case, not an instruction, nothing to act on here. +Filler line 254: padding for the size case, not an instruction, nothing to act on here. +Filler line 255: padding for the size case, not an instruction, nothing to act on here. +Filler line 256: padding for the size case, not an instruction, nothing to act on here. +Filler line 257: padding for the size case, not an instruction, nothing to act on here. +Filler line 258: padding for the size case, not an instruction, nothing to act on here. +Filler line 259: padding for the size case, not an instruction, nothing to act on here. +Filler line 260: padding for the size case, not an instruction, nothing to act on here. +Filler line 261: padding for the size case, not an instruction, nothing to act on here. +Filler line 262: padding for the size case, not an instruction, nothing to act on here. +Filler line 263: padding for the size case, not an instruction, nothing to act on here. +Filler line 264: padding for the size case, not an instruction, nothing to act on here. +Filler line 265: padding for the size case, not an instruction, nothing to act on here. +Filler line 266: padding for the size case, not an instruction, nothing to act on here. +Filler line 267: padding for the size case, not an instruction, nothing to act on here. +Filler line 268: padding for the size case, not an instruction, nothing to act on here. +Filler line 269: padding for the size case, not an instruction, nothing to act on here. +Filler line 270: padding for the size case, not an instruction, nothing to act on here. +Filler line 271: padding for the size case, not an instruction, nothing to act on here. +Filler line 272: padding for the size case, not an instruction, nothing to act on here. +Filler line 273: padding for the size case, not an instruction, nothing to act on here. +Filler line 274: padding for the size case, not an instruction, nothing to act on here. +Filler line 275: padding for the size case, not an instruction, nothing to act on here. +Filler line 276: padding for the size case, not an instruction, nothing to act on here. +Filler line 277: padding for the size case, not an instruction, nothing to act on here. +Filler line 278: padding for the size case, not an instruction, nothing to act on here. +Filler line 279: padding for the size case, not an instruction, nothing to act on here. +Filler line 280: padding for the size case, not an instruction, nothing to act on here. +Filler line 281: padding for the size case, not an instruction, nothing to act on here. +Filler line 282: padding for the size case, not an instruction, nothing to act on here. +Filler line 283: padding for the size case, not an instruction, nothing to act on here. +Filler line 284: padding for the size case, not an instruction, nothing to act on here. +Filler line 285: padding for the size case, not an instruction, nothing to act on here. +Filler line 286: padding for the size case, not an instruction, nothing to act on here. +Filler line 287: padding for the size case, not an instruction, nothing to act on here. +Filler line 288: padding for the size case, not an instruction, nothing to act on here. +Filler line 289: padding for the size case, not an instruction, nothing to act on here. +Filler line 290: padding for the size case, not an instruction, nothing to act on here. +Filler line 291: padding for the size case, not an instruction, nothing to act on here. +Filler line 292: padding for the size case, not an instruction, nothing to act on here. +Filler line 293: padding for the size case, not an instruction, nothing to act on here. +Filler line 294: padding for the size case, not an instruction, nothing to act on here. +Filler line 295: padding for the size case, not an instruction, nothing to act on here. +Filler line 296: padding for the size case, not an instruction, nothing to act on here. +Filler line 297: padding for the size case, not an instruction, nothing to act on here. +Filler line 298: padding for the size case, not an instruction, nothing to act on here. +Filler line 299: padding for the size case, not an instruction, nothing to act on here. +Filler line 300: padding for the size case, not an instruction, nothing to act on here. +Filler line 301: padding for the size case, not an instruction, nothing to act on here. +Filler line 302: padding for the size case, not an instruction, nothing to act on here. +Filler line 303: padding for the size case, not an instruction, nothing to act on here. +Filler line 304: padding for the size case, not an instruction, nothing to act on here. +Filler line 305: padding for the size case, not an instruction, nothing to act on here. +Filler line 306: padding for the size case, not an instruction, nothing to act on here. +Filler line 307: padding for the size case, not an instruction, nothing to act on here. +Filler line 308: padding for the size case, not an instruction, nothing to act on here. +Filler line 309: padding for the size case, not an instruction, nothing to act on here. +Filler line 310: padding for the size case, not an instruction, nothing to act on here. +Filler line 311: padding for the size case, not an instruction, nothing to act on here. +Filler line 312: padding for the size case, not an instruction, nothing to act on here. +Filler line 313: padding for the size case, not an instruction, nothing to act on here. +Filler line 314: padding for the size case, not an instruction, nothing to act on here. +Filler line 315: padding for the size case, not an instruction, nothing to act on here. +Filler line 316: padding for the size case, not an instruction, nothing to act on here. +Filler line 317: padding for the size case, not an instruction, nothing to act on here. +Filler line 318: padding for the size case, not an instruction, nothing to act on here. +Filler line 319: padding for the size case, not an instruction, nothing to act on here. +Filler line 320: padding for the size case, not an instruction, nothing to act on here. +Filler line 321: padding for the size case, not an instruction, nothing to act on here. +Filler line 322: padding for the size case, not an instruction, nothing to act on here. +Filler line 323: padding for the size case, not an instruction, nothing to act on here. +Filler line 324: padding for the size case, not an instruction, nothing to act on here. +Filler line 325: padding for the size case, not an instruction, nothing to act on here. +Filler line 326: padding for the size case, not an instruction, nothing to act on here. +Filler line 327: padding for the size case, not an instruction, nothing to act on here. +Filler line 328: padding for the size case, not an instruction, nothing to act on here. +Filler line 329: padding for the size case, not an instruction, nothing to act on here. +Filler line 330: padding for the size case, not an instruction, nothing to act on here. +Filler line 331: padding for the size case, not an instruction, nothing to act on here. +Filler line 332: padding for the size case, not an instruction, nothing to act on here. +Filler line 333: padding for the size case, not an instruction, nothing to act on here. +Filler line 334: padding for the size case, not an instruction, nothing to act on here. +Filler line 335: padding for the size case, not an instruction, nothing to act on here. +Filler line 336: padding for the size case, not an instruction, nothing to act on here. +Filler line 337: padding for the size case, not an instruction, nothing to act on here. +Filler line 338: padding for the size case, not an instruction, nothing to act on here. +Filler line 339: padding for the size case, not an instruction, nothing to act on here. +Filler line 340: padding for the size case, not an instruction, nothing to act on here. +Filler line 341: padding for the size case, not an instruction, nothing to act on here. +Filler line 342: padding for the size case, not an instruction, nothing to act on here. +Filler line 343: padding for the size case, not an instruction, nothing to act on here. +Filler line 344: padding for the size case, not an instruction, nothing to act on here. +Filler line 345: padding for the size case, not an instruction, nothing to act on here. +Filler line 346: padding for the size case, not an instruction, nothing to act on here. +Filler line 347: padding for the size case, not an instruction, nothing to act on here. +Filler line 348: padding for the size case, not an instruction, nothing to act on here. +Filler line 349: padding for the size case, not an instruction, nothing to act on here. +Filler line 350: padding for the size case, not an instruction, nothing to act on here. +Filler line 351: padding for the size case, not an instruction, nothing to act on here. +Filler line 352: padding for the size case, not an instruction, nothing to act on here. +Filler line 353: padding for the size case, not an instruction, nothing to act on here. +Filler line 354: padding for the size case, not an instruction, nothing to act on here. +Filler line 355: padding for the size case, not an instruction, nothing to act on here. +Filler line 356: padding for the size case, not an instruction, nothing to act on here. +Filler line 357: padding for the size case, not an instruction, nothing to act on here. +Filler line 358: padding for the size case, not an instruction, nothing to act on here. +Filler line 359: padding for the size case, not an instruction, nothing to act on here. +Filler line 360: padding for the size case, not an instruction, nothing to act on here. +Filler line 361: padding for the size case, not an instruction, nothing to act on here. +Filler line 362: padding for the size case, not an instruction, nothing to act on here. +Filler line 363: padding for the size case, not an instruction, nothing to act on here. +Filler line 364: padding for the size case, not an instruction, nothing to act on here. +Filler line 365: padding for the size case, not an instruction, nothing to act on here. +Filler line 366: padding for the size case, not an instruction, nothing to act on here. +Filler line 367: padding for the size case, not an instruction, nothing to act on here. +Filler line 368: padding for the size case, not an instruction, nothing to act on here. +Filler line 369: padding for the size case, not an instruction, nothing to act on here. +Filler line 370: padding for the size case, not an instruction, nothing to act on here. +Filler line 371: padding for the size case, not an instruction, nothing to act on here. +Filler line 372: padding for the size case, not an instruction, nothing to act on here. +Filler line 373: padding for the size case, not an instruction, nothing to act on here. +Filler line 374: padding for the size case, not an instruction, nothing to act on here. +Filler line 375: padding for the size case, not an instruction, nothing to act on here. +Filler line 376: padding for the size case, not an instruction, nothing to act on here. +Filler line 377: padding for the size case, not an instruction, nothing to act on here. +Filler line 378: padding for the size case, not an instruction, nothing to act on here. +Filler line 379: padding for the size case, not an instruction, nothing to act on here. +Filler line 380: padding for the size case, not an instruction, nothing to act on here. +Filler line 381: padding for the size case, not an instruction, nothing to act on here. +Filler line 382: padding for the size case, not an instruction, nothing to act on here. +Filler line 383: padding for the size case, not an instruction, nothing to act on here. +Filler line 384: padding for the size case, not an instruction, nothing to act on here. +Filler line 385: padding for the size case, not an instruction, nothing to act on here. +Filler line 386: padding for the size case, not an instruction, nothing to act on here. +Filler line 387: padding for the size case, not an instruction, nothing to act on here. +Filler line 388: padding for the size case, not an instruction, nothing to act on here. +Filler line 389: padding for the size case, not an instruction, nothing to act on here. +Filler line 390: padding for the size case, not an instruction, nothing to act on here. +Filler line 391: padding for the size case, not an instruction, nothing to act on here. +Filler line 392: padding for the size case, not an instruction, nothing to act on here. +Filler line 393: padding for the size case, not an instruction, nothing to act on here. +Filler line 394: padding for the size case, not an instruction, nothing to act on here. +Filler line 395: padding for the size case, not an instruction, nothing to act on here. +Filler line 396: padding for the size case, not an instruction, nothing to act on here. +Filler line 397: padding for the size case, not an instruction, nothing to act on here. +Filler line 398: padding for the size case, not an instruction, nothing to act on here. +Filler line 399: padding for the size case, not an instruction, nothing to act on here. +Filler line 400: padding for the size case, not an instruction, nothing to act on here. +Filler line 401: padding for the size case, not an instruction, nothing to act on here. +Filler line 402: padding for the size case, not an instruction, nothing to act on here. +Filler line 403: padding for the size case, not an instruction, nothing to act on here. +Filler line 404: padding for the size case, not an instruction, nothing to act on here. +Filler line 405: padding for the size case, not an instruction, nothing to act on here. +Filler line 406: padding for the size case, not an instruction, nothing to act on here. +Filler line 407: padding for the size case, not an instruction, nothing to act on here. +Filler line 408: padding for the size case, not an instruction, nothing to act on here. +Filler line 409: padding for the size case, not an instruction, nothing to act on here. +Filler line 410: padding for the size case, not an instruction, nothing to act on here. +Filler line 411: padding for the size case, not an instruction, nothing to act on here. +Filler line 412: padding for the size case, not an instruction, nothing to act on here. +Filler line 413: padding for the size case, not an instruction, nothing to act on here. +Filler line 414: padding for the size case, not an instruction, nothing to act on here. +Filler line 415: padding for the size case, not an instruction, nothing to act on here. +Filler line 416: padding for the size case, not an instruction, nothing to act on here. +Filler line 417: padding for the size case, not an instruction, nothing to act on here. +Filler line 418: padding for the size case, not an instruction, nothing to act on here. +Filler line 419: padding for the size case, not an instruction, nothing to act on here. +Filler line 420: padding for the size case, not an instruction, nothing to act on here. +Filler line 421: padding for the size case, not an instruction, nothing to act on here. +Filler line 422: padding for the size case, not an instruction, nothing to act on here. +Filler line 423: padding for the size case, not an instruction, nothing to act on here. +Filler line 424: padding for the size case, not an instruction, nothing to act on here. +Filler line 425: padding for the size case, not an instruction, nothing to act on here. +Filler line 426: padding for the size case, not an instruction, nothing to act on here. +Filler line 427: padding for the size case, not an instruction, nothing to act on here. +Filler line 428: padding for the size case, not an instruction, nothing to act on here. +Filler line 429: padding for the size case, not an instruction, nothing to act on here. +Filler line 430: padding for the size case, not an instruction, nothing to act on here. +Filler line 431: padding for the size case, not an instruction, nothing to act on here. +Filler line 432: padding for the size case, not an instruction, nothing to act on here. +Filler line 433: padding for the size case, not an instruction, nothing to act on here. +Filler line 434: padding for the size case, not an instruction, nothing to act on here. +Filler line 435: padding for the size case, not an instruction, nothing to act on here. +Filler line 436: padding for the size case, not an instruction, nothing to act on here. +Filler line 437: padding for the size case, not an instruction, nothing to act on here. +Filler line 438: padding for the size case, not an instruction, nothing to act on here. +Filler line 439: padding for the size case, not an instruction, nothing to act on here. +Filler line 440: padding for the size case, not an instruction, nothing to act on here. +Filler line 441: padding for the size case, not an instruction, nothing to act on here. +Filler line 442: padding for the size case, not an instruction, nothing to act on here. +Filler line 443: padding for the size case, not an instruction, nothing to act on here. +Filler line 444: padding for the size case, not an instruction, nothing to act on here. +Filler line 445: padding for the size case, not an instruction, nothing to act on here. +Filler line 446: padding for the size case, not an instruction, nothing to act on here. +Filler line 447: padding for the size case, not an instruction, nothing to act on here. +Filler line 448: padding for the size case, not an instruction, nothing to act on here. +Filler line 449: padding for the size case, not an instruction, nothing to act on here. +Filler line 450: padding for the size case, not an instruction, nothing to act on here. +Filler line 451: padding for the size case, not an instruction, nothing to act on here. +Filler line 452: padding for the size case, not an instruction, nothing to act on here. +Filler line 453: padding for the size case, not an instruction, nothing to act on here. +Filler line 454: padding for the size case, not an instruction, nothing to act on here. +Filler line 455: padding for the size case, not an instruction, nothing to act on here. +Filler line 456: padding for the size case, not an instruction, nothing to act on here. +Filler line 457: padding for the size case, not an instruction, nothing to act on here. +Filler line 458: padding for the size case, not an instruction, nothing to act on here. +Filler line 459: padding for the size case, not an instruction, nothing to act on here. +Filler line 460: padding for the size case, not an instruction, nothing to act on here. diff --git a/dokimi/data/reach-conformance/nested-agents/AGENTS.md b/dokimi/data/reach-conformance/nested-agents/AGENTS.md new file mode 100644 index 000000000..9f772df7b --- /dev/null +++ b/dokimi/data/reach-conformance/nested-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Repository guide + +Code under packages/ has its own guide next to it. diff --git a/dokimi/data/reach-conformance/nested-agents/packages/app/AGENTS.md b/dokimi/data/reach-conformance/nested-agents/packages/app/AGENTS.md new file mode 100644 index 000000000..09d2d628e --- /dev/null +++ b/dokimi/data/reach-conformance/nested-agents/packages/app/AGENTS.md @@ -0,0 +1,3 @@ +# App guide + +The app package uses pytest. Keep tests next to the code. diff --git a/dokimi/data/reach-conformance/override-file/AGENTS.md b/dokimi/data/reach-conformance/override-file/AGENTS.md new file mode 100644 index 000000000..6ee948ce1 --- /dev/null +++ b/dokimi/data/reach-conformance/override-file/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +The base rules, used where no override file sits. diff --git a/dokimi/data/reach-conformance/override-file/AGENTS.override.md b/dokimi/data/reach-conformance/override-file/AGENTS.override.md new file mode 100644 index 000000000..4bff2cc4a --- /dev/null +++ b/dokimi/data/reach-conformance/override-file/AGENTS.override.md @@ -0,0 +1,3 @@ +# Override guide + +The rules that replace the base file in this directory. diff --git a/dokimi/data/reach-conformance/root-agents/AGENTS.md b/dokimi/data/reach-conformance/root-agents/AGENTS.md new file mode 100644 index 000000000..76e6e8dd5 --- /dev/null +++ b/dokimi/data/reach-conformance/root-agents/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +Run `make test` from the repository root. Nothing else applies. diff --git a/dokimi/data/reach-conformance/subtree-claude-shadow/sub/AGENTS.md b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/AGENTS.md new file mode 100644 index 000000000..01e8f4ba6 --- /dev/null +++ b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/AGENTS.md @@ -0,0 +1,3 @@ +# Subtree agent guide + +Used only where the directory has no CLAUDE.md. diff --git a/dokimi/data/reach-conformance/subtree-claude-shadow/sub/CLAUDE.md b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/CLAUDE.md new file mode 100644 index 000000000..73e174a85 --- /dev/null +++ b/dokimi/data/reach-conformance/subtree-claude-shadow/sub/CLAUDE.md @@ -0,0 +1,3 @@ +# Subtree Claude guide + +This directory has its own CLAUDE.md. diff --git a/dokimi/data/reach-conformance/symlinked-file/AGENTS.md b/dokimi/data/reach-conformance/symlinked-file/AGENTS.md new file mode 100644 index 000000000..483863b75 --- /dev/null +++ b/dokimi/data/reach-conformance/symlinked-file/AGENTS.md @@ -0,0 +1,3 @@ +# Agent guide + +The file CLAUDE.md links to this one. diff --git a/dokimi/data/reach-conformance/symlinked-file/CLAUDE.md b/dokimi/data/reach-conformance/symlinked-file/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/dokimi/data/reach-conformance/symlinked-file/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/dokimi/docs/README.md b/dokimi/docs/README.md index 4133acaa6..0125d2d4e 100644 --- a/dokimi/docs/README.md +++ b/dokimi/docs/README.md @@ -353,6 +353,10 @@ code. harness profiles, and the guidance reach probe; and how to run its task self-check, agent runs, report and arm comparison (`repo tasks check`, `repo run`, `repo report`, `repo compare`). Tasks and graders stay in the scanned repository. +- [reach-matrix.md](reach-matrix.md) — the reach conformance suite: fixture + repositories, what each harness's vendor documentation says they load, the + `scripts/reach_conformance.py` run and merge that produce the agent × version + matrix, and the nightly workflow that publishes it. - [repo-evaluation-mergeability.md](repo-evaluation-mergeability.md) — *(proposed)* grading whether a maintainer would merge a test-passing change: `rubric.json` (blockers, non-blockers, scope), deterministic scope and diff --git a/dokimi/docs/reach-matrix.md b/dokimi/docs/reach-matrix.md new file mode 100644 index 000000000..3e2fc8c05 --- /dev/null +++ b/dokimi/docs/reach-matrix.md @@ -0,0 +1,169 @@ + + +# Reach conformance matrix + +The reach conformance suite asks, for each agent harness and version, which guidance files it loads in a set of +small fixture repositories, and compares what it observed with what the harness's vendor documentation says. A +nightly workflow runs it and publishes the result as the **reach matrix**: harness, version, fixture, file, +scenario, observed state, documented state. + +It answers the question behind agents.md issues #211, #213 and #135: which file a given harness actually reads, +and where its documentation and its behaviour differ. + +## What a run is + +A run is one harness, at one version and one model, over one fixture. It is `repo reach` (see +[repo-evaluation-contract.md](repo-evaluation-contract.md) §7) with `--format receipt`: the effective-instructions +receipt of §7.1 is written to the run's output directory, and each source's observed state is compared with the +documented one. A harness that cannot run here gets a `skipped` run and no cells. + +The matrix has two kinds of record: + +- `runs`: one per harness and fixture, with a status. `ran` means every scenario answered; `errors` means a + receipt came back with a scenario that gave no usable answer; `failed` means no usable receipt (a timeout, a + crash before it was written, an invalid or unreadable receipt, or one in which no scenario answered, as when the + credential is rejected; the reason is the first scenario's error); `skipped` means the harness could not run + here, with the reason. +- `cells`: one per source file per scenario, with the observed state, the documented state and an agreement: + `agrees`, `disagrees`, `inconclusive` (observed unknown, which includes a `not_loaded` from a scenario that answered in prose and so may under-report; or only `fetched`: reach saw the file only in a `read=` scenario, after the agent read it with a tool, so its loading rule is not shown), `no_claim` (the docs say nothing), + or `alias` (a link, whose state is its target's and is not compared). + +The schema is `reach_matrix.v1` (`dokimi/schemas/reach_matrix.v1.json`). + +## The fixtures + +Each directory under `data/reach-conformance/` is one case. Each holds the guidance files of one loading rule, +at the root or in subdirectories. + +| case | what it holds | the rule it tests | +|---|---|---| +| `root-agents` | `AGENTS.md` at the root | the baseline: a root file is read at the root | +| `nested-agents` | a root `AGENTS.md` and `packages/app/AGENTS.md` | a subdirectory file loads when cwd is inside it, or (Claude Code) when a file there is read | +| `claude-and-agents` | `CLAUDE.md` beside `AGENTS.md` | Claude Code reads `AGENTS.md` only when no `CLAUDE.md` is in scope | +| `claude-imports-agents` | `CLAUDE.md` containing `@AGENTS.md` | a Claude Code `@` import is followed; Codex does not read `CLAUDE.md` | +| `override-file` | `AGENTS.md` beside `AGENTS.override.md` | Codex uses the override in place of `AGENTS.md` | +| `subtree-claude-shadow` | `sub/AGENTS.md` beside `sub/CLAUDE.md` | a directory's `CLAUDE.md` shadows its `AGENTS.md` in Claude Code | +| `large-file` | a root `AGENTS.md` of about 40 KB | the size cap; the docs give no cut-off for Claude Code, and 32 KiB for Codex. A file cut at the cap reads as `unknown` | +| `symlinked-file` | `CLAUDE.md` as a symbolic link to `AGENTS.md` | how a link is named and read; the docs do not say | + +The fixtures are plain directories. `guidance.yaml` next to them is a `repo_manifest.v1` that names every +guidance file glob (`AGENTS.override.md` included), so every harness probes the same set of files whatever its +own profile names. Each fixture holds only guidance files: no tasks and no code. + +## Where the documented states come from + +`data/reach-conformance/expectations.yaml` (`reach_conformance.v1`) holds, for each case and harness, the +documented state of each file in each scenario, and a `why` in words. The sources are the vendor pages: + +- Claude Code: `code.claude.com/docs/en/memory`, the "AGENTS.md" and "How CLAUDE.md files load" sections. +- Codex: `learn.chatgpt.com/docs/agent-configuration/agents-md`, and the `project_doc_max_bytes` and + `project_doc_fallback_filenames` settings. + +A file or scenario the expectations do not list is `unknown`. Then the cell's agreement is `no_claim`: the +observation is reported, and nothing is claimed about it. Copilot has no documented start-up behaviour for its +CLI, so every Copilot cell is `no_claim`; a Copilot run still records what it observed. + +Scenario ids are `cwd=.` (start in the root), `cwd=` (start in a subdirectory) and `read=` (start in +the root and have the model read a file in ``). `cwd=` and `read=` exist only for a directory +below the root that holds a guidance file; `check` rejects any other id. + +`python scripts/reach_conformance.py check` verifies the expectations against the fixture tree: every fixture +has an entry, every named file exists in its fixture, and every named scenario's directory exists. It exits 1 +on any drift, and `run` refuses to start while there is drift. + +## Running it + +From the dokimi project directory, with the dependencies installed (`make dev`), and a harness CLI on `PATH`. Set +`PYTHONPATH=.`: `make dev` installs the dependencies, not the package, and a script's own directory is what Python +puts on the import path. + +```sh +PYTHONPATH=. python scripts/reach_conformance.py check +PYTHONPATH=. python scripts/reach_conformance.py run --harness codex= --out results/reach-out +PYTHONPATH=. python scripts/reach_conformance.py merge results/reach-out/*.matrix.json --out reports/reach-matrix.json --md reports/reach-matrix.md +``` + +`--harness NAME[=MODEL]` is repeatable: `claude-code`, `codex` and `copilot` are known. A harness without a model +is skipped, as is one without its credential or its CLI (`unavailable` in `dokimi/repos/conformance.py` +names which). `run` writes `.matrix.json` to `--out`, and the per-fixture receipts under +`-runs/`. Each scenario is capped by `--timeout` (seconds per scenario, default 300). `--budget` (USD, +per scenario, default 0.10) is passed to Claude Code as `--max-budget-usd` only: Codex and Copilot have no budget +flag, so for them the timeout is the only cap. `run` exits 1 on a preflight or input error (expectations drift, a +harness given twice, an input that cannot be loaded, before any fixture runs) or when a run is `failed`; errors-only +and skipped runs exit 0. `run` gives each +`repo reach` process exactly the environment it was started with, and each harness gets only what its profile +forwards from that. + +Credentials are the names the harness profile forwards (`dokimi/repos/profiles/*.yaml`, `env_allow`): +`ANTHROPIC_API_KEY` or `AWS_BEARER_TOKEN_BEDROCK` for Claude Code, `CODEX_API_KEY` for Codex (`OPENAI_API_KEY` is +not forwarded), `COPILOT_GITHUB_TOKEN` for Copilot. Codex runs with `dokimi/repos/conformance_profiles/codex.yaml`, +which defines a provider that reads `CODEX_API_KEY` (`codex exec` sends no bearer without one), so the key is what +authenticates the run. For Bedrock, also set `CLAUDE_CODE_USE_BEDROCK=1` and +`AWS_REGION`: Claude Code does not select Bedrock from the token alone. + +`merge` joins matrices from several harnesses or runs. It refuses a second run of the same harness version on the +same fixture, so a merge cannot double-count (failed and skipped runs hold no cells and are kept as they are). Each +matrix carries `expectations_digest`, the sha256 of the expectations it was scored against; `merge` refuses matrices +with different digests, and the `merge` command refuses ones whose digest is not that of its `--expectations` file. Each also carries +`fixtures_digest`, the sha256 of everything under the fixtures root its runs observed, and `merge` refuses matrices +with different ones: an edited fixture would otherwise read as a change between harness versions. The root also +holds `guidance.yaml` and, by default, `expectations.yaml`, so an edit to either, even to a comment, changes the digest. + +The offline tests (`tests/test_reach_conformance.py`) cover the agreement rules, scenario ids, alias handling, +skipped and failed runs, merge, rendering, the drift checks, and the CLI, with the harness runs faked. No test +calls a model. + +## Limits + +- **Marker self-report, not behaviour.** `observed` is what the model reported seeing in its prompt, through marker + tokens (see §7). A model can under-report, and a `loaded` cell does not show the instruction was followed. +- **Links.** A symbolic link is reported as an alias of its target. The receipt cannot show a harness ignoring + the link's name, so an `alias` cell is not evidence that a harness reads or skips it. +- **Unknown is not false.** `unknown` in the documented column means the docs are silent, not that the file is + not loaded. +- **One version per run.** The version is the one the harness reports; a matrix shows the versions that ran. +- **Cost.** Each scenario makes at least one model call. Each Claude Code scenario is capped at `--budget`, not + the whole fixture or suite (a fixture with three scenarios can spend three times the cap); Codex and Copilot + scenarios are bounded only by `--timeout`. The nightly run covers every scenario of every fixture for each + harness that has a model, a credential and a CLI. + +## Nightly workflow + +`.github/workflows/reach-matrix.yml` runs on a schedule (daily, 04:37 UTC) and on `workflow_dispatch`. It is not a +pull-request check, and it has no `pull_request` trigger. + +- One job per harness. Each checks its credential, installs its CLI only when the credential and the model + variable are both present (pinned versions: `@anthropic-ai/claude-code`, `@openai/codex`, `@github/copilot`), and + runs `run` with the model from a repository variable (`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, + `REACH_MODEL_COPILOT`). With no variable, the harness is recorded as skipped. +- Each job gets only its own secret in its environment. The Claude Code job uses the Anthropic key when it is set; + otherwise the Bedrock token, with `CLAUDE_CODE_USE_BEDROCK=1` and `AWS_REGION` from the variable of that name. +- The `matrix` job checks that all three matrices arrived and fails, publishing nothing, when one is missing (its + harness job died before writing it). It merges the three artifacts, appends the page to the run summary, and + uploads `reach-matrix.json` and `reach-matrix.md` as the `reach-matrix` artifact (90 days). Each harness's matrix + is a 30-day artifact of its own. +- `permissions: {}` at the top; each job has `contents: read`. + +To enable a harness, set its model variable and its secret. Nothing is committed: no key, token or model string +is in the repository. + +Secrets: `ANTHROPIC_API_KEY` (or `AWS_BEARER_TOKEN_BEDROCK`), `CODEX_API_KEY`, `COPILOT_GITHUB_TOKEN`. Variables: +`REACH_MODEL_CLAUDE_CODE`, `REACH_MODEL_CODEX`, `REACH_MODEL_COPILOT`, and `AWS_REGION` for the Bedrock path. +Runners: `intel-ubuntu-latest` on the Intel organisation, `ubuntu-latest` elsewhere; each job sets up Node.js for +its CLI install. + +## Adding a case or a harness + +A case is a directory under `data/reach-conformance/` with the guidance files it exercises, plus an entry in +`expectations.yaml` for every harness whose documentation says something about it. `check` must pass. + +A harness is a `Harness` in `HARNESSES` (`dokimi/repos/conformance.py`), with its CLI and the credential +names its profile forwards, a profile under `dokimi/repos/profiles/`, its npm pin in the workflow, and a +job in the workflow. Its documented states go in `expectations.yaml` with the vendor page as `why`. + +Both schemas are versioned (`reach_conformance.v1`, `reach_matrix.v1`). Compatible widening may edit the existing +version; breaking changes require a new version, updating `SCHEMA_VERSIONS` and retaining the old version in +`SUPERSEDED_VERSIONS` (`dokimi/artifacts.py`). diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index 92f8caa80..625b59709 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -13154,6 +13154,7 @@ "'plan'", "'plugin_delta'", "'questions'", + "'reach_matrix'", "'redteam'", "'repo_budget'", "'repo_environment'", diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 6377283ab..d7db1840d 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -1175,6 +1175,10 @@ reported, prose above the limit, unseeded tokens). The receipt-level `unknowns` repeat the limits above and carry the notice that start-up imports and skill listings are not probed. +The same receipt, run over the fixtures in `data/reach-conformance/` for several +harnesses and versions, is the reach conformance matrix; see +[reach-matrix.md](reach-matrix.md). + ## 8. L0 scan (`repo_readiness.v1`) `repo scan ` checks every mechanically checkable claim in the guidance diff --git a/dokimi/docs/runs.md b/dokimi/docs/runs.md index 140f941dd..5244bdaf4 100644 --- a/dokimi/docs/runs.md +++ b/dokimi/docs/runs.md @@ -44,8 +44,8 @@ skipped: 1 JSON with no leading schema_version, 1 JSON of an unknown schema_vers fields depends on the kind: `product` of a questions set, `library_name` of an arms run, `artifact.ref` of a novelty run, `repo.id` of a reach run, `judge_model` and `evaluated_at` of an eval. An artifact that records no - subject shows `-` (an eval, a task run, a questions set from `questions - generate`, which records the model but not the product); nothing is guessed + subject shows `-` (an eval, a task run, a reach matrix, a questions set from + `questions generate`, which records the model but not the product); nothing is guessed from the path. Rows are newest first. When an artifact records no time, its file's mtime is used for the order and shown with `*`. - **Nothing is dropped silently.** The summary counts the files skipped, by diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index fe128e8fe..0d897ca68 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -14,8 +14,8 @@ ``result`` (one status per planned cell), ``repo_budget`` (what a repository puts into an agent's context), ``effective_instructions`` (the effective-instructions/v0 receipt of a reach probe), ``need_graph`` and ``eval_coverage`` (a frozen need graph and an -eval set's coverage of it) and ``task_inspect`` (which eval-task format each -path under a tree is). This module stamps each with a +eval set's coverage of it), ``task_inspect`` (which eval-task format each +path under a tree is) and ``reach_matrix`` (the reach conformance matrix). This module stamps each with a ``schema_version`` and validates it against the JSON Schema in ``dokimi/schemas/`` on save and load, so fixtures and baselines stay forward-compatible and self-describing. @@ -65,6 +65,7 @@ "cli_routes": "cli_routes.v1", "need_graph": "need_graph.v1", "eval_coverage": "eval_coverage.v1", + "reach_matrix": "reach_matrix.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py new file mode 100644 index 000000000..474456acc --- /dev/null +++ b/dokimi/dokimi/repos/conformance.py @@ -0,0 +1,413 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The reach conformance suite (docs/reach-matrix.md): fixture repositories, what each harness's vendor +documentation says they load, and the matrix of what each harness run observed. + +A run is one harness at one version and model over one fixture: ``repo reach --format receipt`` writes its +effective-instructions receipt, and each source's observed state is compared with the documented one. A +harness that cannot run here (no model, credential or CLI) gets a ``skipped`` run and no cells. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sys +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Collection, Mapping, Sequence + +import yaml + +from dokimi import process +from dokimi.artifacts import load_artifact, save_artifact, schema_errors +from dokimi.repos.reach import PROSE_LIMIT + +SCHEMA_EXPECTATIONS = "reach_conformance.v1" +RECEIPT_NAME = "effective_instructions.json" +UNKNOWN = "unknown" +RUN_STATUSES = ("ran", "errors", "failed", "skipped") +AGREEMENTS = ("agrees", "disagrees", "inconclusive", "no_claim", "alias") +DEFAULT_TIMEOUT_S = 300 +DEFAULT_BUDGET_USD = 0.10 +# The guidance names data/reach-conformance/guidance.yaml globs, so every harness probes them in each fixture. +MANIFEST_NAMES = ("AGENTS.md", "AGENTS.override.md", "CLAUDE.md", "CLAUDE.local.md") + + +class ConformanceError(RuntimeError): + """An expectations file, a matrix or a merge that cannot be used.""" + + +@dataclass(frozen=True) +class Harness: + """A harness the suite knows: its CLI, the variables one of which must be set for a run, and the profile + ``repo reach`` runs it with (a built-in name, or a profile YAML path; the harness name when ``profile`` is empty).""" + + name: str + bin: str + credentials: tuple[str, ...] + profile: str = "" + + @property + def profile_ref(self) -> str: + return self.profile or self.name + + +# The codex profile with a provider that reads CODEX_API_KEY: the built-in profile sends no bearer (see the file). +# Package data, so an installed wheel finds it next to this module. +CODEX_PROFILE = Path(__file__).resolve().parent / "conformance_profiles" / "codex.yaml" + +# Credentials are the names the profile forwards (repos/profiles/*.yaml env_allow), so a key set here is one the run gets. +HARNESSES: dict[str, Harness] = {h.name: h for h in ( + Harness("claude-code", "claude", ("ANTHROPIC_API_KEY", "AWS_BEARER_TOKEN_BEDROCK")), + Harness("codex", "codex", ("CODEX_API_KEY",), profile=str(CODEX_PROFILE)), + Harness("copilot", "copilot", ("COPILOT_GITHUB_TOKEN",)), +)} + + +def _which_on(cmd: str, search: str) -> str | None: + """``cmd`` on ``search``, a PATH value: the search a run's child process makes, not this process's.""" + return shutil.which(cmd, path=search) + + +def unavailable(harness: Harness, model: str, env: Mapping[str, str], + which: Callable[[str, str], str | None] = _which_on) -> str | None: + """Why ``harness`` cannot run here, or None when it can. The CLI is looked up on the PATH of ``env``.""" + if not model: + return "no model configured for this harness" + if not any(env.get(name) for name in harness.credentials): + return f"none of {', '.join(harness.credentials)} is set" + if which(harness.bin, env.get("PATH", os.defpath)) is None: + return f"{harness.bin} is not on PATH" + return None + + +def cases(fixtures: Path) -> list[str]: + """The fixture repositories: the directories under ``fixtures``, by name.""" + return sorted(p.name for p in fixtures.iterdir() if p.is_dir()) + + +def load_expectations(path: Path) -> dict[str, Any]: + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, ValueError, yaml.YAMLError) as exc: + raise ConformanceError(f"{path}: {exc}") from exc + errors = schema_errors(SCHEMA_EXPECTATIONS, data) + if errors: + raise ConformanceError(f"{path}: " + "; ".join(errors)) + return data + + +def expectations_digest(expect: Mapping[str, Any]) -> str: + """The sha256 of the expectations as canonical JSON: matrices built from different documented states differ in it.""" + text = json.dumps(expect, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def fixtures_digest(fixtures: Path) -> str: + """The sha256 of everything under the fixtures root (each file's path and bytes, a link's target), including its guidance file and, by default, its expectations file.""" + h = hashlib.sha256() + for p in sorted(fixtures.rglob("*"), key=lambda p: p.relative_to(fixtures).as_posix()): + if not (p.is_symlink() or p.is_file()): + continue + data = b"L" + os.readlink(p).encode() if p.is_symlink() else b"F" + p.read_bytes() + for part in (p.relative_to(fixtures).as_posix().encode(), data): + h.update(len(part).to_bytes(8, "big") + part) + return h.hexdigest() + + +def check_expectations(expect: Mapping[str, Any], fixtures: Path) -> list[str]: + """What makes the expectations disagree with the fixture tree; empty when they agree.""" + declared, found = set(expect["cases"]), set(cases(fixtures)) + problems = [f"no expectations for fixture {c}" for c in sorted(found - declared)] + problems += [f"expectations name no fixture {c}" for c in sorted(declared - found)] + for case in sorted(declared & found): + problems += _case_problems(case, expect["cases"][case], fixtures / case) + return problems + + +def scenarios_of(root: Path) -> set[str]: + """The scenario ids reach runs over one fixture: ``cwd=.``, and ``cwd=`` and ``read=`` for each + directory below the root that holds a guidance file the conformance manifest globs.""" + dirs = {p.relative_to(root).parent.as_posix() for p in root.rglob("*") if p.name in MANIFEST_NAMES} + dirs.discard(".") + return {"cwd=."} | {f"cwd={d}" for d in dirs} | {f"read={d}" for d in dirs} + + +def _case_problems(case: str, block: Mapping[str, Any], root: Path) -> list[str]: + out = [] + produced = scenarios_of(root) + for harness, body in block.items(): + if harness == "summary": + continue + if harness not in HARNESSES: + out.append(f"{case}: {harness} is not a harness") + for path, states in body["files"].items(): + if not (root / path).is_file() and not (root / path).is_symlink(): + out.append(f"{case}: {harness} names {path}, which the fixture does not hold") + for scenario in states: + if scenario not in produced: + out.append(f"{case}: {harness} names scenario {scenario}, which the fixture does not have") + return out + + +def documented(expect: Mapping[str, Any], case: str, harness: str, path: str, scenario: str) -> str: + """The state the harness's vendor documentation gives this file in this scenario; unknown when it says nothing.""" + block = expect["cases"].get(case, {}).get(harness) + if not block: + return UNKNOWN + return block["files"].get(path, {}).get(scenario, UNKNOWN) + + +def scenario_id(run: Mapping[str, Any]) -> str: + """The reach scenario id of one receipt run: ``read=`` for a run told to read a file, else ``cwd=``.""" + if run.get("target"): + return "read=" + str(PurePosixPath(run["target"]).parent) + return "cwd=" + (run.get("cwd") or ".") + + +def agreement(observed: str, documented_state: str, alias: bool) -> str: + if alias: + return "alias" + if documented_state == UNKNOWN: + return "no_claim" + if observed in (UNKNOWN, "fetched"): + return "inconclusive" + return "agrees" if observed == documented_state else "disagrees" + + +def cells_of(receipt: Mapping[str, Any], case: str, expect: Mapping[str, Any], + prose: Collection[str] = ()) -> list[dict[str, Any]]: + """One cell per source per scenario of one receipt (an effective-instructions receipt of one fixture). + A ``not_loaded`` in a scenario of ``prose`` (it answered in prose, so it may under-report) is observed unknown.""" + harness, version = receipt["runtime"]["name"], receipt["runtime"]["version"] + out = [] + for run in receipt["receipts"]: + scenario = scenario_id(run) + for src in run["sources"]: + alias = src.get("alias_of") + state = documented(expect, case, harness, src["path"], scenario) + observed = UNKNOWN if scenario in prose and src["observed"] == "not_loaded" else src["observed"] + cell = {"harness": harness, "version": version, "case": case, "scenario": scenario, + "path": src["path"], "observed": observed, "documented": state, + "agreement": agreement(observed, state, alias is not None)} + if alias: + cell["alias_of"] = alias + out.append(cell) + return out + + +def run_record(harness: str, case: str, status: str, reason: str | None, version: str, model: str, + scenarios: int) -> dict[str, Any]: + return {"harness": harness, "version": version, "model": model, "case": case, "status": status, + "reason": reason, "scenarios": scenarios} + + +def summary(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + counts = Counter(c["agreement"] for c in cells) + status = Counter(r["status"] for r in runs) + return {**{a: counts.get(a, 0) for a in AGREEMENTS}, "runs": {s: status.get(s, 0) for s in RUN_STATUSES}} + + +def build_matrix(runs: Sequence[Mapping[str, Any]], cells: Sequence[Mapping[str, Any]], digest: str, + fixtures: str, generated_at: str | None = None) -> dict[str, Any]: + """The ``reach_matrix.v1`` payload: runs and cells in a stable order, with the counts.""" + return { + "schema_version": "reach_matrix.v1", + "generated_at": generated_at or datetime.now(timezone.utc).isoformat(), + "expectations": SCHEMA_EXPECTATIONS, + "expectations_digest": digest, + "fixtures_digest": fixtures, + "runs": sorted((dict(r) for r in runs), key=lambda r: (r["case"], r["harness"], r["version"])), + "cells": sorted((dict(c) for c in cells), + key=lambda c: (c["case"], c["harness"], c["version"], c["path"], c["scenario"])), + "summary": summary(runs, cells), + } + + +def _reason(proc: process.Completed) -> str: + if proc.timed_out: + return "timed out" + tail = proc.stderr_bytes.decode("utf-8", "replace").strip()[-300:] + return f"exit {proc.returncode}" + (f": {tail}" if tail else "") + + +def _scenario_rows(out: Path) -> list[dict[str, Any]]: + """The scenario rows of the run's ``reach.json``; none when it is missing or unreadable.""" + try: + rows = json.loads((out / "reach.json").read_text(encoding="utf-8"))["scenarios"] + except (OSError, ValueError, KeyError, TypeError): + return [] + return [r for r in rows if isinstance(r, dict)] if isinstance(rows, list) else [] + + +def run_case(harness: Harness, model: str, fixture: Path, guidance: Path, out: Path, + expect: Mapping[str, Any], *, timeout: int, budget: float, + env: Mapping[str, str]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Run ``repo reach`` over one fixture with exactly ``env`` as its environment; its run record and cells. + The receipt is read from ``out``.""" + case = fixture.name + try: + shutil.rmtree(out) # a receipt left by an earlier run is not this run's + except FileNotFoundError: + pass + except OSError as exc: # a receipt that stays would be read as this run's: fail the run instead + return run_record(harness.name, case, "failed", f"cannot clear {out}: {exc}", UNKNOWN, model, 0), [] + argv = [sys.executable, "-m", "dokimi", "repo", "reach", str(fixture), "--manifest-file", str(guidance), + "--harness", harness.profile_ref, "--model", model, "--out", str(out), "--format", "receipt", + "--timeout", str(timeout), "--budget", f"{budget:g}"] + proc = process.run(argv, timeout=timeout * 16 + 300, env=process.Env.only((), env)) + receipt_path = out / RECEIPT_NAME + if proc.timed_out or not receipt_path.is_file(): + return run_record(harness.name, case, "failed", _reason(proc), UNKNOWN, model, 0), [] + try: + receipt = load_artifact("effective_instructions", receipt_path) + except (OSError, ValueError) as exc: # one bad receipt fails its run, not the whole harness + return run_record(harness.name, case, "failed", f"invalid receipt: {exc}", UNKNOWN, model, 0), [] + rows = _scenario_rows(out) + errors = [str(r["error"]) for r in rows if r.get("error")] + if rows and len(errors) == len(rows): # no scenario answered (a revoked key, a hung CLI): nothing was observed + return run_record(harness.name, case, "failed", errors[0], receipt["runtime"]["version"], model, 0), [] + status = "ran" if proc.returncode == 0 else "errors" + reason = None if status == "ran" else (errors[0] if errors else _reason(proc)) + record = run_record(harness.name, case, status, reason, receipt["runtime"]["version"], model, + len(receipt["receipts"])) + prose = {str(r.get("id")) for r in rows if (r.get("prose") or 0) > PROSE_LIMIT} + return record, cells_of(receipt, case, expect, prose) + + +def check_out_dir(out: Path, fixtures: Path) -> None: + """Refuse an output directory that is, holds or sits in the fixtures: a run clears ``/`` first.""" + a, b = out.resolve(), fixtures.resolve() + if a == b or a.is_relative_to(b) or b.is_relative_to(a): + raise ConformanceError(f"{out} and the fixtures {fixtures} overlap: a run would clear a fixture's directory") + + +def run_harness(harness: Harness, model: str, fixtures: Path, expect: Mapping[str, Any], out_dir: Path, *, + timeout: int = DEFAULT_TIMEOUT_S, budget: float = DEFAULT_BUDGET_USD, + env: Mapping[str, str], which: Callable[[str, str], str | None] = _which_on) -> dict[str, Any]: + """One harness over every fixture, as a matrix. A harness that cannot run here gets a skipped run per fixture. + + ``env`` decides availability and is the whole environment each run gets (nothing else is inherited).""" + check_out_dir(out_dir, fixtures) + reason = unavailable(harness, model, env, which) + guidance = fixtures / "guidance.yaml" + runs: list[dict[str, Any]] = [] + cells: list[dict[str, Any]] = [] + for case in cases(fixtures): + if reason: + runs.append(run_record(harness.name, case, "skipped", reason, UNKNOWN, model, 0)) + continue + record, found = run_case(harness, model, fixtures / case, guidance, out_dir / case, expect, + timeout=timeout, budget=budget, env=env) + runs.append(record) + cells += found + return build_matrix(runs, cells, expectations_digest(expect), fixtures_digest(fixtures)) + + +def merge(matrices: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """One matrix from several: the runs and cells of each, and no run of the same known harness version twice. + + The matrices must come from one revision of the expectations and of the fixtures, else their states differ.""" + if not matrices: + raise ConformanceError("nothing to merge") + digests = sorted({m["expectations_digest"] for m in matrices}) + if len(digests) > 1: + raise ConformanceError(f"the matrices come from {len(digests)} revisions of the expectations: " + + ", ".join(d[:12] for d in digests)) + trees = sorted({m["fixtures_digest"] for m in matrices}) + if len(trees) > 1: + raise ConformanceError(f"the matrices come from {len(trees)} fixture trees: " + ", ".join(d[:12] for d in trees)) + runs: list[dict[str, Any]] = [] + cells: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + for matrix in matrices: + for run in matrix["runs"]: + if run["status"] in ("failed", "skipped"): # no receipt, so no cells to count twice and no version to key on + continue + key = (run["harness"], run["version"], run["case"]) + if key in seen: + raise ConformanceError(f"two matrices hold {run['harness']} {run['version']} on {run['case']}") + seen.add(key) + runs += matrix["runs"] + cells += matrix["cells"] + latest = max((m["generated_at"] for m in matrices), key=_instant) + return build_matrix(runs, cells, digests[0], trees[0], generated_at=latest) + + +def _instant(text: str) -> datetime: + """The moment a ``generated_at`` names: offsets differ between matrices, so text does not order them.""" + try: + when = datetime.fromisoformat(text) + except ValueError as exc: + raise ConformanceError(f"generated_at {text!r} is not an ISO 8601 timestamp") from exc + if when.tzinfo is None: + raise ConformanceError(f"generated_at {text!r} has no time zone") + return when + + +def write_matrix(matrix: Mapping[str, Any], path: Path) -> None: + save_artifact("reach_matrix", dict(matrix), path) + + +def _table_text(text: str) -> str: + """One table cell: a stderr reason can run over several lines, and a pipe would end the cell early.""" + return " ".join(text.split()).replace("|", "\\|") + + +def _cell_text(cell: Mapping[str, Any] | None) -> str: + if cell is None: + return "-" + text = cell["observed"] + if cell["agreement"] == "alias": + text = f"link, {text}" + elif cell["agreement"] == "disagrees": + text += f" (docs: {cell['documented']})" + return _table_text(text) + + +def render_markdown(matrix: Mapping[str, Any], summaries: Mapping[str, str] | None = None) -> str: + """The matrix as a markdown page: the runs, then one table per fixture with a column per harness version.""" + summaries = summaries or {} + counts = matrix["summary"] + columns = sorted({(r["harness"], r["version"]) for r in matrix["runs"] if r["status"] not in ("skipped", "failed")}) + # every dynamic table field goes through _table_text: a pipe or newline in any of them breaks the row + lines = [ + "# Reach matrix", "", + f"Generated {matrix['generated_at']}. Observed states are what each run's model reported seeing " + "(`repo reach`); documented states are the vendor documentation's, in `data/reach-conformance/expectations.yaml`.", + "", + f"Cells: {counts['agrees']} agree with the documentation, {counts['disagrees']} disagree, " + f"{counts['inconclusive']} inconclusive, {counts['no_claim']} with no documented claim, " + f"{counts['alias']} links. Runs: {counts['runs']['ran']} ran, {counts['runs']['errors']} with errors, " + f"{counts['runs']['failed']} failed, {counts['runs']['skipped']} skipped.", "", + "## Runs", "", "| fixture | harness | version | model | status | reason |", "|---|---|---|---|---|---|", + ] + lines += [f"| {_table_text(r['case'])} | {_table_text(r['harness'])} | {_table_text(r['version'])} | " + f"{_table_text(r['model'] or '-')} | {_table_text(r['status'])} | {_table_text(r['reason'] or '-')} |" + for r in matrix["runs"]] + lines += ["", "Cell legend: `loaded` or `not_loaded` as observed; `fetched` is read with a tool first; " + "`unknown` is no usable answer. `(docs: X)` marks a disagreement with the documentation; `link, X` " + "marks a symlink; `inconclusive` is an observed `unknown` or `fetched` where the documentation names a " + "state; a cell with none of these agrees, or the documentation makes no claim.", ""] + for case in sorted({c["case"] for c in matrix["cells"]}): + lines += _case_section(matrix, case, columns, summaries.get(case, "")) + return "\n".join(lines).rstrip() + "\n" + + +def _case_section(matrix: Mapping[str, Any], case: str, columns: list[tuple[str, str]], summary_text: str) -> list[str]: + cells = [c for c in matrix["cells"] if c["case"] == case] + index = {(c["harness"], c["version"], c["path"], c["scenario"]): c for c in cells} + rows = sorted({(c["path"], c["scenario"]) for c in cells}) + header = ["file", "scenario"] + [_table_text(f"{h} {v}") for h, v in columns] + out = [f"## {case}", ""] + ([summary_text, ""] if summary_text else []) + out += ["| " + " | ".join(header) + " |", "|" + "---|" * len(header)] + for path, scenario in rows: + texts = [_cell_text(index.get((h, v, path, scenario))) for h, v in columns] + out.append("| " + " | ".join([_table_text(path), _table_text(scenario), *texts]) + " |") + return out + [""] diff --git a/dokimi/dokimi/repos/conformance_profiles/codex.yaml b/dokimi/dokimi/repos/conformance_profiles/codex.yaml new file mode 100644 index 000000000..f4d3d4f68 --- /dev/null +++ b/dokimi/dokimi/repos/conformance_profiles/codex.yaml @@ -0,0 +1,74 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# The codex profile of the reach suite: profiles/codex.yaml plus a provider that reads CODEX_API_KEY, because the +# built-in one sends no bearer (401 Missing bearer). Keep the two in step. +schema: harness_profile.v1 +name: codex +bin: codex # npm's codex is a node launcher; mount the native one (.../vendor//bin/codex) with --bin +version_cmd: ["{bin}", "--version"] +# Codex 0.160 runs every tool call through codex-code-mode-host, which it starts from +# beside its own binary; without it in the container no command can run. +bin_companions: [codex-code-mode-host] +# `codex exec` is non-interactive; the prompt follows `--`, so a leading "-" is not a flag. No turn or spend cap: +# {max_turns}/{budget} are not passed and the timeout bounds the run. CODEX_HOME is fresh per run (no --ignore-user-config). +invoke: + - "{bin}" + - "exec" + - "--model" + - "{model}" + - "--ephemeral" + - "--skip-git-repo-check" + - "--json" + - "--color" + - "never" + - "{tools}" + - "--" + - "{prompt}" +# Codex has no tool-off switch: `none` and `read` both get its bwrap sandbox, which needs user namespaces. +# Where the host denies them every command fails; `all` relies on the framework's container for isolation. +tools: + none: ["--sandbox", "read-only"] + read: ["--sandbox", "read-only"] + all: ["--dangerously-bypass-approvals-and-sandbox"] +config_dir_env: CODEX_HOME +env_allow: + - PATH + - HOME + - LANG + - SYSTEMROOT + - CODEX_API_KEY +# OPENAI_API_KEY is not forwarded: `all` mode's shell reads every forwarded variable, and `codex exec` reads neither key. + - OPENAI_BASE_URL + - http_proxy + - https_proxy + - no_proxy + - HTTP_PROXY + - HTTPS_PROXY + - NO_PROXY +# The provider `model_provider` selects; its env_key names the variable that holds the bearer. +config_files: + config.toml: | + model_provider = "conformance" + + [model_providers.conformance] + name = "OpenAI" + base_url = "https://api.openai.com/v1" + wire_api = "responses" + env_key = "CODEX_API_KEY" +egress: + - api.openai.com +endpoint_env: OPENAI_BASE_URL +trace: codex-json +guidance: + names: [AGENTS.md, AGENTS.override.md] + resolution: codex-home + git-root-to-cwd + lazy-by-shell-read + imports: null + notes: >- + Documented for codex-cli 0.160: one AGENTS.override.md or AGENTS.md per + directory from the git root down to the working directory, joined root + first; an AGENTS.override.md replaces the AGENTS.md beside it. CLAUDE.md, + .github/copilot-instructions.md and .github/instructions are not read. + Measured on 0.160 with gpt-6-luna (oneDAL review task): root AGENTS.md + applied, subtree AGENTS.md applied only after the agent printed them with a + shell command; AGENTS.override.md hid the root AGENTS.md. diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index 4ceb72028..378ff7789 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -73,6 +73,7 @@ class KindFields: counts=("violations", "questions")), "need_graph": KindFields(subject=("artifact_name",), counts=("nodes", "edges")), "eval_coverage": KindFields(subject=("graph.artifact_name",), counts=("gaps", "evals", "problems")), + "reach_matrix": KindFields(time=("generated_at",), counts=("runs", "cells")), } #: Every schema_version an artifact may carry (current and superseded) -> its kind. diff --git a/dokimi/dokimi/schemas/reach_conformance.v1.json b/dokimi/dokimi/schemas/reach_conformance.v1.json new file mode 100644 index 000000000..4cb7ad1c5 --- /dev/null +++ b/dokimi/dokimi/schemas/reach_conformance.v1.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/reach_conformance.v1.json", + "title": "Reach conformance expectations v1", + "description": "What each harness's vendor documentation says each fixture repository loads, per reach scenario (data/reach-conformance/expectations.yaml, docs/reach-matrix.md). A file or scenario that is not listed is unknown: the documentation does not say.", + "type": "object", + "required": ["schema", "cases"], + "properties": { + "schema": {"const": "reach_conformance.v1"}, + "cases": { + "type": "object", + "description": "Keyed by the fixture repository's directory name.", + "additionalProperties": { + "type": "object", + "required": ["summary"], + "properties": {"summary": {"type": "string"}}, + "additionalProperties": {"$ref": "#/$defs/harness"} + } + } + }, + "$defs": { + "harness": { + "type": "object", + "required": ["why", "files"], + "additionalProperties": false, + "properties": { + "why": {"type": "string", "description": "The vendor documentation the states rest on, in short."}, + "files": { + "type": "object", + "description": "Keyed by the guidance file's path inside the fixture.", + "additionalProperties": { + "type": "object", + "description": "Keyed by reach scenario id: cwd=., cwd= or read=.", + "additionalProperties": {"enum": ["loaded", "not_loaded", "unknown"]} + } + } + } + } + } +} diff --git a/dokimi/dokimi/schemas/reach_matrix.v1.json b/dokimi/dokimi/schemas/reach_matrix.v1.json new file mode 100644 index 000000000..b14a59dbb --- /dev/null +++ b/dokimi/dokimi/schemas/reach_matrix.v1.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/reach_matrix.v1.json", + "title": "Reach conformance matrix v1", + "description": "The matrix of the reach conformance suite (docs/reach-matrix.md): for each harness run over each fixture repository, the state each guidance file was observed in, per reach scenario, next to what the harness's vendor documentation says (reach_conformance.v1). One matrix is written per harness and several are merged into one. A `skipped` run has no cells; a cell whose documented state is `unknown` makes no claim.", + "type": "object", + "required": ["schema_version", "generated_at", "expectations", "expectations_digest", "fixtures_digest", "runs", "cells", "summary"], + "properties": { + "schema_version": {"const": "reach_matrix.v1"}, + "generated_at": {"type": "string"}, + "expectations": {"const": "reach_conformance.v1", "description": "The schema of the expectations file the documented states come from."}, + "expectations_digest": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The sha256 of the expectations file's content as canonical JSON (sorted keys, no spaces); matrices with different digests are not merged."}, + "fixtures_digest": {"type": "string", "pattern": "^[0-9a-f]{64}$", "description": "The sha256 of the fixture tree the runs observed (each file's relative path and bytes, a link's target); matrices with different digests are not merged."}, + "runs": { + "type": "array", + "items": { + "type": "object", + "required": ["harness", "version", "model", "case", "status", "reason", "scenarios"], + "properties": { + "harness": {"type": "string"}, + "version": {"type": "string", "description": "The harness's own version as reach reports it; `unknown` when no receipt came back."}, + "model": {"type": "string", "description": "The model string passed to the harness; empty when none was configured."}, + "case": {"type": "string", "description": "The fixture repository's directory name."}, + "status": {"enum": ["ran", "errors", "failed", "skipped"], "description": "ran: every scenario answered; errors: a receipt came back with at least one scenario that gave no usable answer; failed: no usable receipt, or no scenario answered; skipped: the harness could not run here."}, + "reason": {"type": ["string", "null"]}, + "scenarios": {"type": "integer", "minimum": 0} + } + } + }, + "cells": { + "type": "array", + "items": { + "type": "object", + "required": ["harness", "version", "case", "scenario", "path", "observed", "documented", "agreement"], + "properties": { + "harness": {"type": "string"}, + "version": {"type": "string"}, + "case": {"type": "string"}, + "scenario": {"type": "string", "pattern": "^(cwd=|read=)"}, + "path": {"type": "string"}, + "alias_of": {"type": "string", "description": "Set when the path is a link; its observed state is the target's."}, + "observed": {"enum": ["loaded", "fetched", "not_loaded", "unknown"]}, + "documented": {"enum": ["loaded", "not_loaded", "unknown"]}, + "agreement": {"enum": ["agrees", "disagrees", "inconclusive", "no_claim", "alias"], "description": "agrees and disagrees compare observed with documented; inconclusive: observed is unknown or fetched; no_claim: the docs say nothing; alias: a link, not compared."} + } + } + }, + "summary": { + "type": "object", + "required": ["agrees", "disagrees", "inconclusive", "no_claim", "alias", "runs"], + "properties": { + "agrees": {"type": "integer", "minimum": 0}, + "disagrees": {"type": "integer", "minimum": 0}, + "inconclusive": {"type": "integer", "minimum": 0}, + "no_claim": {"type": "integer", "minimum": 0}, + "alias": {"type": "integer", "minimum": 0}, + "runs": { + "type": "object", + "required": ["ran", "errors", "failed", "skipped"], + "properties": { + "ran": {"type": "integer", "minimum": 0}, + "errors": {"type": "integer", "minimum": 0}, + "failed": {"type": "integer", "minimum": 0}, + "skipped": {"type": "integer", "minimum": 0} + } + } + } + } + } +} diff --git a/dokimi/mkdocs.yml b/dokimi/mkdocs.yml index 0f2d53b27..33c29ea68 100644 --- a/dokimi/mkdocs.yml +++ b/dokimi/mkdocs.yml @@ -129,6 +129,7 @@ nav: - repo-evaluation-contract.md - repo-evaluation-mergeability.md - repo-evaluation-agent-modes.md + - reach-matrix.md - preparing-a-repository.md - preparing-a-repository-evidence.md - Extending Dokimi: diff --git a/dokimi/pyproject.toml b/dokimi/pyproject.toml index c645cedb0..c97a10b0a 100644 --- a/dokimi/pyproject.toml +++ b/dokimi/pyproject.toml @@ -102,6 +102,7 @@ dokimi = [ "runner/spec.schema.json", "schemas/*.json", "repos/profiles/*.yaml", + "repos/conformance_profiles/*.yaml", "repos/support/*.py", "harnesses/swebench_specs.json", ] diff --git a/dokimi/scripts/reach_conformance.py b/dokimi/scripts/reach_conformance.py new file mode 100644 index 000000000..764a2b9f5 --- /dev/null +++ b/dokimi/scripts/reach_conformance.py @@ -0,0 +1,120 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Drive the reach conformance suite (docs/reach-matrix.md). + + python scripts/reach_conformance.py check + python scripts/reach_conformance.py run --harness codex= --out DIR + python scripts/reach_conformance.py merge DIR/*.matrix.json --out DIR/reach-matrix.json --md DIR/reach-matrix.md + +``run`` makes paid model calls for every harness that has a model, a credential and a CLI here; the rest are +written as skipped runs. Run it from the dokimi project directory with ``PYTHONPATH=.``: a script's own directory, +not the project, is on the import path, and ``make dev`` does not install the package. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from dokimi.artifacts import load_artifact +from dokimi.repos import conformance as cf + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_FIXTURES = ROOT / "data" / "reach-conformance" +DEFAULT_EXPECTATIONS = DEFAULT_FIXTURES / "expectations.yaml" + + +def _expectations(path: Path) -> dict: + return cf.load_expectations(path) + + +def cmd_check(args: argparse.Namespace) -> int: + problems = cf.check_expectations(_expectations(args.expectations), args.fixtures) + for problem in problems: + print(problem, file=sys.stderr) + if not problems: + print(f"ok: {len(cf.cases(args.fixtures))} fixtures, expectations agree") + return 1 if problems else 0 + + +def _harness_spec(text: str) -> tuple[cf.Harness, str]: + name, _, model = text.partition("=") + if name not in cf.HARNESSES: + raise argparse.ArgumentTypeError(f"unknown harness {name!r}; known: {', '.join(cf.HARNESSES)}") + return cf.HARNESSES[name], model + + +def cmd_run(args: argparse.Namespace) -> int: + expect = _expectations(args.expectations) + problems = cf.check_expectations(expect, args.fixtures) + if problems: + print("\n".join(problems), file=sys.stderr) + return 1 + names = [harness.name for harness, _ in args.harness] + if len(set(names)) != len(names): + print("✗ each --harness may be given once: a second run would overwrite the first's matrix", file=sys.stderr) + return 1 + cf.check_out_dir(args.out, args.fixtures) # before anything is created or cleared + failed = False + args.out.mkdir(parents=True, exist_ok=True) + for harness, model in args.harness: + matrix = cf.run_harness(harness, model, args.fixtures, expect, args.out / f"{harness.name}-runs", + timeout=args.timeout, budget=args.budget, env=os.environ) + cf.write_matrix(matrix, args.out / f"{harness.name}.matrix.json") + counts = matrix["summary"]["runs"] + print(f"{harness.name}: ran {counts['ran']}, errors {counts['errors']}, " + f"failed {counts['failed']}, skipped {counts['skipped']}") + failed = failed or counts["failed"] > 0 + return 1 if failed else 0 + + +def cmd_merge(args: argparse.Namespace) -> int: + expect = _expectations(args.expectations) + matrices = [load_artifact("reach_matrix", path) for path in args.inputs] + merged = cf.merge(matrices) + if merged["expectations_digest"] != cf.expectations_digest(expect): + print(f"✗ the matrices were built from another revision of {args.expectations}", file=sys.stderr) + return 1 + cf.write_matrix(merged, args.out) + summaries = {case: block["summary"] for case, block in expect["cases"].items()} + args.md.parent.mkdir(parents=True, exist_ok=True) + args.md.write_text(cf.render_markdown(merged, summaries), encoding="utf-8") + print(f"merged {len(matrices)} matrices: {len(merged['runs'])} runs, {len(merged['cells'])} cells") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--fixtures", type=Path, default=DEFAULT_FIXTURES) + parser.add_argument("--expectations", type=Path, default=DEFAULT_EXPECTATIONS) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("check", help="verify the expectations against the fixture tree") + run = sub.add_parser("run", help="run reach over the fixtures for each harness") + run.add_argument("--harness", action="append", type=_harness_spec, required=True, + metavar="NAME[=MODEL]", help="repeatable; no model makes the harness skipped") + run.add_argument("--out", type=Path, required=True, help="directory for .matrix.json") + run.add_argument("--timeout", type=int, default=cf.DEFAULT_TIMEOUT_S, help="per-scenario seconds") + run.add_argument("--budget", type=float, default=cf.DEFAULT_BUDGET_USD, + help="per-scenario USD cap for Claude Code only; Codex and Copilot have no budget flag") + merge = sub.add_parser("merge", help="merge matrices into one JSON and one markdown page") + merge.add_argument("inputs", nargs="+", type=Path) + merge.add_argument("--out", type=Path, required=True) + merge.add_argument("--md", type=Path, required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + commands = {"run": cmd_run, "merge": cmd_merge, "check": cmd_check} + try: + return commands[args.command](args) + except (cf.ConformanceError, OSError, ValueError) as exc: # an input the suite cannot use: say so, no traceback + print(f"✗ {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dokimi/tests/fixtures/artifacts/reach_conformance.v1.json b/dokimi/tests/fixtures/artifacts/reach_conformance.v1.json new file mode 100644 index 000000000..5b88bb037 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/reach_conformance.v1.json @@ -0,0 +1,22 @@ +{ + "schema": "reach_conformance.v1", + "cases": { + "nested-agents": { + "summary": "A root AGENTS.md and a packages/app/AGENTS.md below it.", + "codex": { + "why": "Codex walks from the git root down to cwd, one file per directory.", + "files": { + "AGENTS.md": {"cwd=.": "loaded", "cwd=packages/app": "loaded"}, + "packages/app/AGENTS.md": {"cwd=.": "not_loaded", "cwd=packages/app": "loaded"} + } + } + }, + "large-file": { + "summary": "A root AGENTS.md above the size cap.", + "claude-code": { + "why": "No size limit is documented.", + "files": {} + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json new file mode 100644 index 000000000..41e810f01 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/reach_matrix.v1.json @@ -0,0 +1,68 @@ +{ + "schema_version": "reach_matrix.v1", + "generated_at": "2026-10-31T04:37:00+00:00", + "expectations": "reach_conformance.v1", + "expectations_digest": "0000000000000000000000000000000000000000000000000000000000000000", + "fixtures_digest": "0000000000000000000000000000000000000000000000000000000000000000", + "runs": [ + { + "harness": "codex", + "version": "0.166.0", + "model": "example-model", + "case": "nested-agents", + "status": "ran", + "reason": null, + "scenarios": 2 + }, + { + "harness": "codex", + "version": "unknown", + "model": "", + "case": "nested-agents", + "status": "skipped", + "reason": "none of CODEX_API_KEY is set", + "scenarios": 0 + } + ], + "cells": [ + { + "harness": "codex", + "version": "0.166.0", + "case": "nested-agents", + "scenario": "cwd=packages/app", + "path": "packages/app/AGENTS.md", + "observed": "loaded", + "documented": "loaded", + "agreement": "agrees" + }, + { + "harness": "codex", + "version": "0.166.0", + "case": "nested-agents", + "scenario": "cwd=.", + "path": "AGENTS.md", + "observed": "not_loaded", + "documented": "loaded", + "agreement": "disagrees" + }, + { + "harness": "codex", + "version": "0.166.0", + "case": "nested-agents", + "scenario": "cwd=.", + "path": "LINK.md", + "alias_of": "AGENTS.md", + "observed": "loaded", + "documented": "unknown", + "agreement": "alias" + } + ], + "summary": { + "agrees": 1, + "disagrees": 1, + "inconclusive": 0, + "no_claim": 0, + "alias": 1, + "runs": {"ran": 1, "errors": 0, "failed": 0, "skipped": 1} + } +} diff --git a/dokimi/tests/test_packaging.py b/dokimi/tests/test_packaging.py index 1eac17184..cb5df4436 100644 --- a/dokimi/tests/test_packaging.py +++ b/dokimi/tests/test_packaging.py @@ -18,6 +18,7 @@ import dokimi from dokimi import _version, config, intents, registry from dokimi.cli import build_parser +from dokimi.repos import conformance from dokimi.runner import spec PROJECT = Path(__file__).resolve().parents[1] @@ -100,6 +101,7 @@ def _stray(package: Path) -> list[str]: intents._DEFAULT_INTENTS, config.DEFAULT_PRODUCTS_PATH, spec.DEFAULT_SCHEMA_PATH, + conformance.CODEX_PROFILE, ], ids=lambda p: p.name) def test_default_data_files_ship_in_the_wheel(path): assert path.is_file() diff --git a/dokimi/tests/test_reach_conformance.py b/dokimi/tests/test_reach_conformance.py new file mode 100644 index 000000000..73d7cd458 --- /dev/null +++ b/dokimi/tests/test_reach_conformance.py @@ -0,0 +1,742 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The reach conformance suite, offline: which harnesses can run, how observed and documented states agree, +the matrix and its markdown page, and the fixtures against their expectations. Harness runs are faked.""" + +from __future__ import annotations + +import importlib.resources +import importlib.util +import json +import os +import shutil +import tomllib +from pathlib import Path + +import pytest +import yaml + +from dokimi import process +from dokimi.artifacts import load_artifact, save_artifact, schema_errors +from dokimi.repos import conformance as cf +from dokimi.repos.harness_profile import load_profile +from dokimi.repos.receipt import build_receipt +from tests.test_repo_receipt import BUDGET, ROOT_ROW, _record, _reach, _tree + +PROJECT = Path(__file__).resolve().parents[1] +FIXTURES = PROJECT / "data" / "reach-conformance" +SCRIPT = PROJECT / "scripts" / "reach_conformance.py" +CODEX = cf.HARNESSES["codex"] +CLAUDE = cf.HARNESSES["claude-code"] +CASE_BLOCK = {"why": "docs", "files": {"AGENTS.md": {"cwd=.": "loaded"}, "CLAUDE.md": {"cwd=.": "not_loaded"}}} + + +def _expect(**blocks: dict) -> dict: + return {"schema": "reach_conformance.v1", "cases": {"toy": {"summary": "toy case", **blocks}}} + + +def _found(name: str, search: str) -> str: + return f"/usr/bin/{name}" + + +def _missing(name: str, search: str) -> None: + return None + + +def _load_cli(): + spec = importlib.util.spec_from_file_location("reach_conformance_cli", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +cli = _load_cli() + + +def _run(**fields) -> dict: + base = {"harness": "codex", "version": "0.166.0", "model": "m", "case": "toy", "status": "ran", + "reason": None, "scenarios": 1} + return {**base, **fields} + + +def _cell(**fields) -> dict: + base = {"harness": "codex", "version": "0.166.0", "case": "toy", "scenario": "cwd=.", "path": "AGENTS.md", + "observed": "loaded", "documented": "loaded", "agreement": "agrees"} + return {**base, **fields} + + +DIGEST = "a" * 64 +TREE = "c" * 64 + + +def _matrix(runs=None, cells=None, generated_at="2026-10-31T00:00:00+00:00", digest=DIGEST, tree=TREE) -> dict: + return cf.build_matrix([_run()] if runs is None else runs, [_cell()] if cells is None else cells, digest, tree, + generated_at=generated_at) + + +def test_unavailable_names_the_missing_piece(): + env = {"CODEX_API_KEY": "k"} + assert cf.unavailable(CODEX, "gpt", env, which=_found) is None + assert "no model" in cf.unavailable(CODEX, "", env, which=_found) + assert "CODEX_API_KEY" in cf.unavailable(CODEX, "gpt", {}, which=_found) + assert "not on PATH" in cf.unavailable(CODEX, "gpt", env, which=_missing) + + +def test_unavailable_looks_for_the_cli_on_the_path_the_run_gets(tmp_path, monkeypatch): + run_bin, parent_bin = tmp_path / "run", tmp_path / "parent" + for folder in (run_bin, parent_bin): + folder.mkdir() + (folder / "codex").write_text("#!/bin/sh\n", encoding="utf-8") + (folder / "codex").chmod(0o755) + empty = tmp_path / "empty" + empty.mkdir() + monkeypatch.setenv("PATH", str(parent_bin)) # this process's PATH has a codex; the run's does not + assert "not on PATH" in cf.unavailable(CODEX, "gpt", {"CODEX_API_KEY": "k", "PATH": str(empty)}) + assert cf.unavailable(CODEX, "gpt", {"CODEX_API_KEY": "k", "PATH": str(run_bin)}) is None + monkeypatch.setenv("PATH", str(empty)) # the other way round: the run's PATH has the codex, this process's has none + assert cf.unavailable(CODEX, "gpt", {"CODEX_API_KEY": "k", "PATH": str(run_bin)}) is None + + +def test_unavailable_forwards_only_the_credentials_the_profile_forwards(): + # codex's profile forwards CODEX_API_KEY alone, so OPENAI_API_KEY would not reach the run + assert cf.unavailable(CODEX, "gpt", {"OPENAI_API_KEY": "k"}, which=_found) is not None + assert cf.unavailable(CLAUDE, "m", {"AWS_BEARER_TOKEN_BEDROCK": "t"}, which=_found) is None + + +@pytest.mark.parametrize(("observed", "documented", "alias", "expected"), [ + ("loaded", "loaded", False, "agrees"), + ("not_loaded", "loaded", False, "disagrees"), + ("loaded", "not_loaded", False, "disagrees"), + ("loaded", "unknown", False, "no_claim"), + ("unknown", "unknown", False, "no_claim"), + ("unknown", "loaded", False, "inconclusive"), + ("fetched", "loaded", False, "inconclusive"), + ("loaded", "loaded", True, "alias"), +]) +def test_agreement_rules(observed, documented, alias, expected): + assert cf.agreement(observed, documented, alias) == expected + + +@pytest.mark.parametrize(("run", "scenario"), [ + ({"cwd": ".", "target": None}, "cwd=."), + ({"cwd": "packages/app", "target": None}, "cwd=packages/app"), + ({"cwd": ".", "target": "packages/app/REACH_PROBE.txt"}, "read=packages/app"), +]) +def test_scenario_id_names_the_cwd_or_the_read_directory(run, scenario): + assert cf.scenario_id(run) == scenario + + +def test_documented_is_unknown_unless_the_docs_name_the_file_and_scenario(): + expect = _expect(codex=CASE_BLOCK) + assert cf.documented(expect, "toy", "codex", "CLAUDE.md", "cwd=.") == "not_loaded" + assert cf.documented(expect, "toy", "codex", "CLAUDE.md", "cwd=sub") == "unknown" + assert cf.documented(expect, "toy", "claude-code", "AGENTS.md", "cwd=.") == "unknown" + assert cf.documented(expect, "other", "codex", "AGENTS.md", "cwd=.") == "unknown" + + +def test_cells_compare_each_source_with_the_documentation(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n", "sub/AGENTS.md": "s\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) + cells = cf.cells_of(receipt, "toy", _expect(**{"claude-code": { + "why": "docs", "files": {"AGENTS.md": {"cwd=.": "not_loaded"}}}})) + agents = next(c for c in cells if c["path"] == "AGENTS.md") + assert agents == {"harness": "claude-code", "version": "2.1.0", "case": "toy", "scenario": "cwd=.", + "path": "AGENTS.md", "observed": "loaded", "documented": "not_loaded", + "agreement": "disagrees"} + + +def test_a_symbolic_link_is_an_alias_of_its_target(tmp_path): + tree = _tree(tmp_path, {"AGENTS.md": "a\n"}) + os.symlink("AGENTS.md", tmp_path / "CLAUDE.md") + files = [{"path": "AGENTS.md", "head": "RC000001", "tail": "RC000002", "aliases": ["CLAUDE.md"]}] + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW], files=files), BUDGET, + tree) + link = next(c for c in cf.cells_of(receipt, "toy", _expect()) if c["path"] == "CLAUDE.md") + assert (link["agreement"], link["alias_of"], link["observed"]) == ("alias", "AGENTS.md", "loaded") + + +def test_build_matrix_counts_and_validates_against_the_schema(): + runs = [_run(), _run(harness="copilot", status="skipped", reason="no key", version="unknown")] + cells = [_cell(), _cell(agreement="disagrees", observed="not_loaded", documented="loaded", path="CLAUDE.md"), + _cell(agreement="alias", path="LINK.md", alias_of="AGENTS.md")] + matrix = cf.build_matrix(runs, cells, DIGEST, TREE, generated_at="2026-10-31T00:00:00+00:00") + assert matrix["summary"] == {"agrees": 1, "disagrees": 1, "inconclusive": 0, "no_claim": 0, "alias": 1, + "runs": {"ran": 1, "errors": 0, "failed": 0, "skipped": 1}} + assert schema_errors("reach_matrix.v1", matrix) == [] + + +def _fake_reach(monkeypatch, receipt: dict, *, returncode: int = 0, timed_out: bool = False, + write: bool = True, rows: list[dict] | None = None) -> list[list[str]]: + calls: list[list[str]] = [] + + def fake_run(argv, *, timeout, env, **_): + calls.append(list(argv)) + out = Path(argv[argv.index("--out") + 1]) + if write: + out.mkdir(parents=True, exist_ok=True) + save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) + if rows is not None: + (out / "reach.json").write_text(json.dumps({"scenarios": rows}), encoding="utf-8") + return process.Completed(argv=tuple(argv), returncode=returncode, timed_out=timed_out, + stderr_bytes=b"boom") + + monkeypatch.setattr(cf.process, "run", fake_run) + return calls + + +def _toy_fixture(tmp_path: Path) -> tuple[Path, dict]: + fixture = tmp_path / "toy" + tree = _tree(fixture, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + receipt = build_receipt(_reach([_record("AGENTS.md", "cwd=.", "loaded")], [ROOT_ROW]), BUDGET, tree) + return fixture, receipt + + +def test_run_case_reads_the_receipt_the_run_wrote(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + calls = _fake_reach(monkeypatch, receipt) + expect = _expect(**{"claude-code": {"why": "docs", "files": {"AGENTS.md": {"cwd=.": "loaded"}}}}) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "guidance.yaml", tmp_path / "out", expect, + timeout=30, budget=0.5, env={}) + assert record == _run(harness="claude-code", version="2.1.0", model="m", status="ran", + scenarios=len(receipt["receipts"])) + assert any(c["path"] == "AGENTS.md" and c["agreement"] == "agrees" for c in cells) + argv = calls[0] + assert argv[argv.index("--harness") + 1] == "claude-code" and argv[argv.index("--format") + 1] == "receipt" + + +def test_run_case_marks_a_run_with_a_failing_scenario_as_errors(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, returncode=1) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5, env={}) + assert (record["status"], record["reason"]) == ("errors", "exit 1: boom") + assert cells + + +AUTH_ERROR = "exit 1 (no answer): API Error: 403 Authentication failed" + + +def test_run_case_with_no_scenario_answered_is_failed_and_names_the_scenario_error(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, returncode=1, rows=[{**ROOT_ROW, "error": AUTH_ERROR}]) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5, env={}) + assert (record["status"], record["reason"], cells) == ("failed", AUTH_ERROR, []) + + +def test_run_case_with_one_scenario_unanswered_is_errors_and_names_its_error(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + rows = [ROOT_ROW, {**ROOT_ROW, "id": "read=sub", "error": AUTH_ERROR}] + _fake_reach(monkeypatch, receipt, returncode=1, rows=rows) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5, env={}) + assert (record["status"], record["reason"]) == ("errors", AUTH_ERROR) + assert cells + + +def test_run_case_scores_no_not_loaded_of_a_scenario_that_answered_in_prose(tmp_path, monkeypatch): + fixture = tmp_path / "toy" + tree = _tree(fixture, {"AGENTS.md": "a\n", "CLAUDE.md": "c\n"}) + row = {**ROOT_ROW, "prose": 14} # a refusal: words, no tokens + records = [_record("AGENTS.md", "cwd=.", "not_loaded"), _record("CLAUDE.md", "cwd=.", "loaded")] + _fake_reach(monkeypatch, build_receipt(_reach(records, [row]), BUDGET, tree), rows=[row]) + expect = _expect(**{"claude-code": {"why": "docs", "files": {"AGENTS.md": {"cwd=.": "loaded"}, + "CLAUDE.md": {"cwd=.": "loaded"}}}}) + record, cells = cf.run_case(CLAUDE, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", expect, + timeout=30, budget=0.5, env={}) + found = {c["path"]: (c["observed"], c["agreement"]) for c in cells if c["path"] in ("AGENTS.md", "CLAUDE.md")} + assert found == {"AGENTS.md": ("unknown", "inconclusive"), "CLAUDE.md": ("loaded", "agrees")} + assert record["status"] == "ran" + +def test_cli_run_exits_1_when_no_scenario_of_a_fixture_answered(tmp_path, monkeypatch, capsys): + _fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, returncode=1, rows=[{**ROOT_ROW, "error": AUTH_ERROR}]) + monkeypatch.setattr(cf.shutil, "which", lambda cmd, path=None: f"/usr/bin/{cmd}") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake") + assert cli.main(["run", "--harness", "claude-code=m", "--out", str(tmp_path / "o")]) == 1 + assert "errors 0, failed 8" in capsys.readouterr().out + + +@pytest.mark.parametrize(("kwargs", "reason"), [ + ({"write": False}, "exit 0"), + ({"timed_out": True, "returncode": process.TIMEOUT_RC}, "timed out"), +]) +def test_run_case_without_a_receipt_is_failed(tmp_path, monkeypatch, kwargs, reason): + fixture, receipt = _toy_fixture(tmp_path) + _fake_reach(monkeypatch, receipt, **kwargs) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5, env={}) + assert record["status"] == "failed" and reason in record["reason"] + assert cells == [] + + +@pytest.mark.parametrize("text", [ + '{"schema_version": "effective_instructions.v1"}', # fails the schema + '{"schema_version": "effective_instructions.v1", "receipts": [', # truncated JSON + "[1, 2]", # not an object +]) +def test_run_case_with_a_malformed_receipt_is_failed_not_fatal(tmp_path, monkeypatch, text): + fixture, _receipt = _toy_fixture(tmp_path) + + def fake_run(argv, *, timeout, env, **_): + out = Path(argv[argv.index("--out") + 1]) + out.mkdir(parents=True, exist_ok=True) + (out / cf.RECEIPT_NAME).write_text(text, encoding="utf-8") + return process.Completed(argv=tuple(argv), returncode=0) + + monkeypatch.setattr(cf.process, "run", fake_run) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), + timeout=30, budget=0.5, env={}) + assert record["status"] == "failed" and record["reason"].startswith("invalid receipt") + assert cells == [] + + +def test_run_harness_skips_every_fixture_when_the_harness_cannot_run(tmp_path, monkeypatch): + def must_not_run(*_args, **_kwargs): + raise AssertionError("an unavailable harness must not be run") + + monkeypatch.setattr(cf.process, "run", must_not_run) + matrix = cf.run_harness(CODEX, "gpt", FIXTURES, cf.load_expectations(FIXTURES / "expectations.yaml"), + tmp_path, env={}, which=_found) + assert {r["case"] for r in matrix["runs"]} == set(cf.cases(FIXTURES)) + assert all(r["status"] == "skipped" and "CODEX_API_KEY" in r["reason"] for r in matrix["runs"]) + assert matrix["cells"] == [] and matrix["summary"]["runs"]["skipped"] == len(matrix["runs"]) + + +def test_merge_joins_matrices_and_keeps_the_latest_time(): + first = _matrix(generated_at="2026-10-30T00:00:00+00:00") + second = _matrix(runs=[_run(harness="claude-code", version="2.1.0")], + cells=[_cell(harness="claude-code", version="2.1.0")], + generated_at="2026-10-31T00:00:00+00:00") + merged = cf.merge([first, second]) + assert len(merged["runs"]) == 2 and len(merged["cells"]) == 2 + assert merged["generated_at"] == "2026-10-31T00:00:00+00:00" + assert schema_errors("reach_matrix.v1", merged) == [] + + +def test_merge_refuses_a_harness_version_on_one_fixture_twice(): + with pytest.raises(cf.ConformanceError, match="codex 0.166.0 on toy"): + cf.merge([_matrix(), _matrix()]) + + +def test_merge_refuses_matrices_from_two_revisions_of_the_expectations(): + second = _matrix(runs=[_run(harness="claude-code")], cells=[_cell(harness="claude-code")], digest="b" * 64) + with pytest.raises(cf.ConformanceError, match="2 revisions of the expectations"): + cf.merge([_matrix(), second]) + + +def test_merge_refuses_matrices_observed_over_two_fixture_trees(tmp_path, monkeypatch): + expect = cf.load_expectations(FIXTURES / "expectations.yaml") + edited = tmp_path / "fixtures" + shutil.copytree(FIXTURES, edited, symlinks=True) + (edited / "nested-agents" / "packages" / "app" / "CLAUDE.md").write_text("x\n", encoding="utf-8") + monkeypatch.setattr(cf.process, "run", lambda *a, **k: pytest.fail("a skipped harness must not run")) + pristine, changed = (cf.run_harness(CODEX, "", fx, expect, tmp_path / name, env={}, which=_found) + for fx, name in ((FIXTURES, "a"), (edited, "b"))) + assert pristine["fixtures_digest"] == cf.fixtures_digest(FIXTURES) != changed["fixtures_digest"] + with pytest.raises(cf.ConformanceError, match="2 fixture trees"): + cf.merge([pristine, changed]) + + +def test_fixtures_digest_follows_each_file_and_link(tmp_path): + tree = tmp_path / "t" + (tree / "a").mkdir(parents=True) + (tree / "a" / "AGENTS.md").write_text("x\n", encoding="utf-8") + base = cf.fixtures_digest(tree) + assert cf.fixtures_digest(tree) == base + (tree / "a" / "AGENTS.md").write_text("y\n", encoding="utf-8") + edited = cf.fixtures_digest(tree) + (tree / "a" / "AGENTS.md").rename(tree / "a" / "CLAUDE.md") + assert len({base, edited, cf.fixtures_digest(tree)}) == 3 + + +def test_fixtures_digest_counts_a_link_by_its_target_text(tmp_path): + tree = tmp_path / "t" + tree.mkdir() + (tree / "real.md").write_text("same\n", encoding="utf-8") + (tree / "other.md").write_text("same\n", encoding="utf-8") + os.symlink("real.md", tree / "LINK.md") + linked = cf.fixtures_digest(tree) + (tree / "LINK.md").unlink() + os.symlink("other.md", tree / "LINK.md") # the same bytes behind it: only the link's text differs + retargeted = cf.fixtures_digest(tree) + assert retargeted != linked + (tree / "LINK.md").unlink() + assert cf.fixtures_digest(tree) not in {linked, retargeted} # a missing link is a change too + + +def test_fixtures_digest_frames_each_path_and_its_data(tmp_path): + one, two = tmp_path / "one", tmp_path / "two" + one.mkdir() + two.mkdir() + (one / "a").write_bytes(b"Fb") # path "a" then data "F" + "Fb": without length prefixes, "aFFb" + (two / "aF").write_bytes(b"b") # path "aF" then data "F" + "b": the same bytes, a different tree + assert cf.fixtures_digest(one) != cf.fixtures_digest(two) + + +def test_fixtures_digest_does_not_depend_on_the_walk_order(tmp_path, monkeypatch): + tree = tmp_path / "t" + tree.mkdir() + for name in ("a.md", "b.md", "c.md"): + (tree / name).write_text(name, encoding="utf-8") + os.symlink("a.md", tree / "L.md") + ordinary = cf.fixtures_digest(tree) + walk = type(tree).rglob + monkeypatch.setattr(type(tree), "rglob", lambda self, pattern: reversed(list(walk(self, pattern)))) + assert cf.fixtures_digest(tree) == ordinary # the same tree, walked in the other order: one digest + + +def test_expectations_digest_follows_the_content_not_the_key_order(): + expect = cf.load_expectations(FIXTURES / "expectations.yaml") + reordered = dict(reversed(list(expect.items()))) + assert cf.expectations_digest(reordered) == cf.expectations_digest(expect) + assert cf.expectations_digest({**expect, "note": "edited"}) != cf.expectations_digest(expect) + matrix = cf.run_harness(CODEX, "", FIXTURES, expect, Path("unused"), env={}, which=_found) + assert matrix["expectations_digest"] == cf.expectations_digest(expect) + + +def test_merge_keeps_two_failed_runs_of_one_harness(): + failed = _run(status="failed", reason="timed out", version="unknown", scenarios=0) + merged = cf.merge([_matrix(runs=[failed], cells=[]), _matrix(runs=[failed], cells=[])]) + assert [r["status"] for r in merged["runs"]] == ["failed", "failed"] + assert merged["cells"] == [] + + +def test_merge_refuses_two_ran_runs_even_when_their_version_is_unknown(): + ran = _run(version="unknown") + with pytest.raises(cf.ConformanceError, match="codex unknown on toy"): + cf.merge([_matrix(runs=[ran]), _matrix(runs=[ran])]) + + +def test_merge_refuses_nothing(): + with pytest.raises(cf.ConformanceError, match="nothing to merge"): + cf.merge([]) + + +def test_render_markdown_shows_observed_documented_and_links(): + matrix = _matrix( + runs=[_run(), _run(harness="copilot", version="1.0.105", status="skipped", reason="a|b")], + cells=[_cell(), _cell(path="CLAUDE.md", observed="not_loaded", documented="loaded", agreement="disagrees"), + _cell(path="LINK.md", observed="loaded", documented="unknown", agreement="alias", alias_of="AGENTS.md")], + ) + md = cf.render_markdown(matrix, {"toy": "toy case"}) + assert md.isascii() + assert "## toy" in md and "toy case" in md + assert "| file | scenario | codex 0.166.0 |" in md + assert "| AGENTS.md | cwd=. | loaded |" in md + assert "| CLAUDE.md | cwd=. | not_loaded (docs: loaded) |" in md + assert "link, loaded" in md + assert "a\\|b" in md + assert "copilot 1.0.105" not in md.split("## toy")[1] + + +def test_expectations_match_the_fixture_tree(): + expect = cf.load_expectations(FIXTURES / "expectations.yaml") + assert cf.check_expectations(expect, FIXTURES) == [] + assert {"large-file", "symlinked-file", "subtree-claude-shadow"} <= set(cf.cases(FIXTURES)) + + +def test_check_expectations_reports_each_kind_of_drift(tmp_path): + fixtures = tmp_path / "fixtures" + _tree(fixtures / "toy", {"AGENTS.md": "a\n"}) + (fixtures / "extra").mkdir() + expect = {"schema": "reach_conformance.v1", "cases": { + "toy": {"summary": "s", "codex": {"why": "w", "files": { + "CLAUDE.md": {"cwd=.": "loaded"}, "AGENTS.md": {"cwd=nope": "loaded", "cwd": "loaded"}}}}, + "gone": {"summary": "s"}, + }} + problems = "\n".join(cf.check_expectations(expect, fixtures)) + assert "no expectations for fixture extra" in problems + assert "expectations name no fixture gone" in problems + assert "codex names CLAUDE.md, which the fixture does not hold" in problems + assert "codex names scenario cwd=nope" in problems + assert "codex names scenario cwd, which" in problems # no "=": the empty directory is the fixture root, not a scenario + + +def test_check_expectations_names_a_harness_key_that_no_harness_has(tmp_path): + fixtures = tmp_path / "fixtures" + _tree(fixtures / "toy", {"AGENTS.md": "a\n"}) + expect = {"schema": "reach_conformance.v1", "cases": {"toy": {"summary": "s", "claud": { + "why": "w", "files": {"AGENTS.md": {"cwd=.": "loaded"}}}}}} + assert "toy: claud is not a harness" in cf.check_expectations(expect, fixtures) + + +def test_check_expectations_names_only_the_scenarios_reach_runs(tmp_path): + fixtures = tmp_path / "fixtures" + _tree(fixtures / "toy", {"AGENTS.md": "a\n", "sub/AGENTS.md": "b\n", "packages/app.py": "x\n"}) + expect = {"schema": "reach_conformance.v1", "cases": {"toy": {"summary": "s", "codex": {"why": "w", "files": { + "AGENTS.md": {"cwd=.": "loaded", "cwd=sub": "loaded", "read=sub": "loaded", "read=.": "loaded"}, + "sub/AGENTS.md": {"cwd=packages": "loaded"}}}}}} + problems = "\n".join(cf.check_expectations(expect, fixtures)) + assert "toy: codex names scenario read=., which" in problems # the root is never a read= directory + assert "toy: codex names scenario cwd=packages, which" in problems # packages holds no guidance file + assert "cwd=sub" not in problems and "read=sub" not in problems + + +def test_manifest_names_are_the_guidance_globs_of_the_conformance_manifest(): + manifest = yaml.safe_load((FIXTURES / "guidance.yaml").read_text(encoding="utf-8")) + assert sorted(glob.removeprefix("**/") for glob in manifest["guidance"]["globs"]) == sorted(cf.MANIFEST_NAMES) + + +def test_load_expectations_rejects_another_schema(tmp_path): + path = tmp_path / "expectations.yaml" + path.write_text("schema: other.v1\ncases: {}\n", encoding="utf-8") + with pytest.raises(cf.ConformanceError): + cf.load_expectations(path) + + +def test_cli_check_passes_on_the_fixtures_and_fails_on_drift(tmp_path, capsys): + assert cli.main(["check"]) == 0 + (tmp_path / "undeclared").mkdir() + assert cli.main(["--fixtures", str(tmp_path), "--expectations", str(FIXTURES / "expectations.yaml"), + "check"]) == 1 + assert "no expectations for fixture undeclared" in capsys.readouterr().err + + +def test_cli_run_rejects_an_unknown_harness(tmp_path): + with pytest.raises(SystemExit): + cli.main(["run", "--harness", "nope=x", "--out", str(tmp_path)]) + + +def test_cli_run_budget_help_says_it_caps_claude_code_only(capsys): + with pytest.raises(SystemExit): + cli.main(["run", "--help"]) + assert "Claude Code only" in " ".join(capsys.readouterr().out.split()) + + +def test_render_markdown_legend_names_the_inconclusive_and_link_cells(): + matrix = _matrix(runs=[_run()], cells=[_cell(observed="unknown", documented="loaded", agreement="inconclusive")]) + legend = cf.render_markdown(matrix).split("Cell legend", 1)[1].split("\n\n", 1)[0] + assert "inconclusive" in legend and "link" in legend + + +def test_cli_merge_writes_json_and_markdown(tmp_path): + digest = cf.expectations_digest(cf.load_expectations(FIXTURES / "expectations.yaml")) + left, right = tmp_path / "left.json", tmp_path / "right.json" + cf.write_matrix(_matrix(digest=digest), left) + cf.write_matrix(_matrix(runs=[_run(harness="claude-code", version="2.1.0")], + cells=[_cell(harness="claude-code", version="2.1.0")], digest=digest), right) + out, md = tmp_path / "reach-matrix.json", tmp_path / "reach-matrix.md" + assert cli.main(["merge", str(left), str(right), "--out", str(out), "--md", str(md)]) == 0 + assert load_artifact("reach_matrix", out)["summary"]["agrees"] == 2 + assert md.read_text(encoding="utf-8").startswith("# Reach matrix") + + +def test_cli_merge_refuses_matrices_from_another_revision_of_the_expectations(tmp_path, capsys): + left = tmp_path / "left.json" + cf.write_matrix(_matrix(), left) + out, md = tmp_path / "reach-matrix.json", tmp_path / "reach-matrix.md" + assert cli.main(["merge", str(left), "--out", str(out), "--md", str(md)]) == 1 + assert "another revision" in capsys.readouterr().err + assert not out.exists() and not md.exists() + + +def test_merge_accepts_one_harness_version_on_two_fixtures(): + merged = cf.merge([_matrix(), _matrix(runs=[_run(case="other")], cells=[_cell(case="other")])]) + assert sorted(r["case"] for r in merged["runs"]) == ["other", "toy"] + + +def test_merge_accepts_two_versions_of_one_harness_on_one_fixture(): + merged = cf.merge([_matrix(), _matrix(runs=[_run(version="0.167.0")], cells=[_cell(version="0.167.0")])]) + assert sorted(r["version"] for r in merged["runs"]) == ["0.166.0", "0.167.0"] + + +def test_merge_refuses_two_errors_runs_of_one_harness_version(): + errors = _run(status="errors", reason="exit 1") + with pytest.raises(cf.ConformanceError, match="codex 0.166.0 on toy"): + cf.merge([_matrix(runs=[errors]), _matrix(runs=[errors])]) + + +def test_merge_accepts_a_skipped_run_beside_a_ran_run_of_one_version(): + skipped = _run(status="skipped", reason="no model configured for this harness", scenarios=0) + merged = cf.merge([_matrix(runs=[skipped], cells=[]), _matrix(runs=[_run()], cells=[_cell()])]) + assert sorted(r["status"] for r in merged["runs"]) == ["ran", "skipped"] + + +def test_render_markdown_keeps_a_multiline_reason_on_one_row(): + matrix = _matrix(runs=[_run(status="errors", reason="exit 1: a\nb | c")], cells=[]) + lines = cf.render_markdown(matrix).splitlines() + assert "| toy | codex | 0.166.0 | m | errors | exit 1: a b \\| c |" in lines + + +def test_render_markdown_adds_no_column_for_a_failed_run(): + matrix = _matrix(runs=[_run(), _run(status="failed", version="unknown", reason="timed out", scenarios=0)]) + header = [line for line in cf.render_markdown(matrix).splitlines() if line.startswith("| file | scenario")] + assert header == ["| file | scenario | codex 0.166.0 |"] + + +def test_run_case_does_not_read_a_receipt_an_earlier_run_left(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + out = tmp_path / "out" + save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) + _fake_reach(monkeypatch, receipt, write=False) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", out, _expect(), + timeout=30, budget=0.5, env={}) + assert record["status"] == "failed" and cells == [] + + +@pytest.mark.parametrize("layout", ["same", "parent", "inside", "through-link"]) +def test_run_harness_refuses_an_out_directory_that_holds_the_fixtures(tmp_path, monkeypatch, layout): + suite = tmp_path / "suite" + fixtures = suite / "codex-runs" # the out directory of a run is out/-runs: this is out_dir = fixtures + _tree(fixtures / "toy", {"AGENTS.md": "a\n"}) + (fixtures / "guidance.yaml").write_text("guidance: {}\n", encoding="utf-8") + out = {"same": fixtures, "parent": suite, "inside": fixtures / "toy", "through-link": tmp_path / "link"}[layout] + if layout == "through-link": + out.symlink_to(suite, target_is_directory=True) + + def no_run(*args, **kwargs): + raise AssertionError("a run must not start") + + monkeypatch.setattr(cf, "run_case", no_run) + with pytest.raises(cf.ConformanceError, match="overlap"): + cf.run_harness(CODEX, "m", fixtures, _expect(), out, env={"CODEX_API_KEY": "k", "PATH": "/usr/bin"}, + which=_found) + assert (fixtures / "toy" / "AGENTS.md").read_text(encoding="utf-8") == "a\n" + assert (fixtures / "guidance.yaml").is_file() + + +def test_cli_run_refuses_an_out_directory_that_holds_the_fixtures(monkeypatch, capsys): + def no_call(*args, **kwargs): + raise AssertionError("run_harness must not be called") + + monkeypatch.setattr(cli.cf, "run_harness", no_call) + assert cli.main(["run", "--harness", "codex=m", "--out", str(FIXTURES.parent)]) == 1 # out holds the fixtures + assert "overlap" in capsys.readouterr().err + assert not (FIXTURES.parent / "codex-runs").exists() + + +def test_cli_run_refuses_a_harness_given_twice_before_any_call(tmp_path, monkeypatch, capsys): + def no_call(*args, **kwargs): + raise AssertionError("run_harness must not be called") + + monkeypatch.setattr(cli.cf, "run_harness", no_call) + out = tmp_path / "out" + assert cli.main(["run", "--harness", "codex=a", "--harness", "codex=b", "--out", str(out)]) == 1 + assert "may be given once" in capsys.readouterr().err + assert not out.exists() + + +@pytest.mark.parametrize(("status", "exit_code"), [("failed", 1), ("errors", 0)]) +def test_cli_run_exits_1_only_when_a_run_failed(tmp_path, monkeypatch, status, exit_code): + run = _run(status=status, reason="exit 1", version="unknown" if status == "failed" else "0.166.0", + scenarios=0 if status == "failed" else 1) + monkeypatch.setattr(cli.cf, "run_harness", lambda *args, **kwargs: cf.build_matrix([run], [], DIGEST, TREE)) + assert cli.main(["run", "--harness", "codex=m", "--out", str(tmp_path)]) == exit_code + assert (tmp_path / "codex.matrix.json").is_file() + + +def test_cli_merge_creates_both_output_directories_when_missing(tmp_path): + digest = cf.expectations_digest(cf.load_expectations(FIXTURES / "expectations.yaml")) + left = tmp_path / "left.json" + cf.write_matrix(_matrix(digest=digest), left) + out, md = tmp_path / "a" / "b" / "reach-matrix.json", tmp_path / "c" / "reach-matrix.md" + assert cli.main(["merge", str(left), "--out", str(out), "--md", str(md)]) == 0 + assert out.is_file() and md.is_file() + + +def test_cli_reports_unusable_inputs_without_a_traceback(tmp_path, capsys): + digest = cf.expectations_digest(cf.load_expectations(FIXTURES / "expectations.yaml")) + left, bad = tmp_path / "left.json", tmp_path / "bad.json" + cf.write_matrix(_matrix(digest=digest), left) + bad.write_text("{}", encoding="utf-8") + out, md = tmp_path / "reach-matrix.json", tmp_path / "reach-matrix.md" + cases = [ + (["--expectations", str(tmp_path / "missing.yaml"), "check"], "missing.yaml"), + (["merge", str(left), str(left), "--out", str(out), "--md", str(md)], "two matrices hold codex"), + (["merge", str(bad), "--out", str(out), "--md", str(md)], "reach_matrix"), + ] + for argv, text in cases: + assert cli.main(argv) == 1 + err = capsys.readouterr().err + assert text in err and "Traceback" not in err + + +def test_run_case_hands_codex_the_suite_profile_with_its_provider(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + calls = _fake_reach(monkeypatch, receipt) + cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", tmp_path / "out", _expect(), timeout=30, budget=0.5, env={}) + argv = calls[0] + assert argv[argv.index("--harness") + 1] == str(cf.CODEX_PROFILE) + + +def test_the_codex_suite_profile_authenticates_with_the_key_it_forwards(tmp_path): + suite = load_profile(CODEX.profile_ref) + builtin = load_profile("codex") + assert "CODEX_API_KEY" in suite.env_allow + assert (suite.invoke, suite.env_allow, suite.egress) == (builtin.invoke, builtin.env_allow, builtin.egress) + suite.write_config(tmp_path, {"config_dir": tmp_path.as_posix(), "support_dir": tmp_path.as_posix()}) + config = tomllib.loads((tmp_path / "config.toml").read_text(encoding="utf-8")) + provider = config["model_providers"][config["model_provider"]] # the provider the run selects + assert provider["env_key"] == "CODEX_API_KEY" + assert provider["base_url"] == "https://api.openai.com/v1" + + +def test_the_codex_suite_profile_is_the_builtin_one_plus_its_provider(): + suite = yaml.safe_load(cf.CODEX_PROFILE.read_text(encoding="utf-8")) + builtin = yaml.safe_load((cf.CODEX_PROFILE.parents[1] / "profiles" / "codex.yaml").read_text(encoding="utf-8")) + assert list(suite.pop("config_files")) == ["config.toml"] + assert suite == builtin # the nightly measures the Codex setup `repo reach --harness codex` runs + + +def test_the_codex_suite_profile_loads_through_the_package_resources(): + # The installed-wheel case: the file is found through the package, not through the source tree. + shipped = importlib.resources.files("dokimi.repos") / "conformance_profiles" / "codex.yaml" + with importlib.resources.as_file(shipped) as path: + assert load_profile(path).name == "codex" + assert path.read_bytes() == cf.CODEX_PROFILE.read_bytes() + + +def test_run_harness_gives_each_run_exactly_the_environment_it_is_given(tmp_path, monkeypatch): + fixtures = tmp_path / "fixtures" + _toy_fixture(fixtures) + seen: list[dict[str, str]] = [] + + def fake_run(argv, *, timeout, env, **_): + seen.append(env.resolve()) + return process.Completed(argv=tuple(argv), returncode=1) + + monkeypatch.setattr(cf.process, "run", fake_run) + given = {"CODEX_API_KEY": "k", "PATH": "/usr/bin"} + matrix = cf.run_harness(CODEX, "gpt", fixtures, _expect(), tmp_path / "runs", env=given, which=_found) + assert seen == [given] + assert [r["status"] for r in matrix["runs"]] == ["failed"] + + +def test_run_case_fails_when_an_earlier_receipt_cannot_be_cleared(tmp_path, monkeypatch): + fixture, receipt = _toy_fixture(tmp_path) + out = tmp_path / "out" + save_artifact("effective_instructions", receipt, out / cf.RECEIPT_NAME) + calls = _fake_reach(monkeypatch, receipt) + + def refuse(path, *args, **kwargs): + raise PermissionError(13, "Permission denied", str(path)) + + monkeypatch.setattr(cf.shutil, "rmtree", refuse) + record, cells = cf.run_case(CODEX, "m", fixture, tmp_path / "g.yaml", out, _expect(), + timeout=30, budget=0.5, env={}) + assert record["status"] == "failed" and "cannot clear" in record["reason"] + assert cells == [] and calls == [] # the old receipt is never read, and no probe runs + + +def test_merge_orders_matrices_by_the_instant_they_name_not_their_text(): + # 10:00 at +02:00 is 08:00 UTC, an hour before 09:00 UTC, though its text sorts higher + earlier = _matrix(generated_at="2026-11-02T10:00:00+02:00") + later = _matrix(runs=[_run(harness="claude-code", version="2.1.0")], + cells=[_cell(harness="claude-code", version="2.1.0")], generated_at="2026-11-02T09:00:00+00:00") + assert cf.merge([earlier, later])["generated_at"] == "2026-11-02T09:00:00+00:00" + + +@pytest.mark.parametrize("stamp", ["2026-11-02 10:00", "not a time"]) +def test_merge_refuses_a_timestamp_it_cannot_order(stamp): + with pytest.raises(cf.ConformanceError, match="generated_at"): + cf.merge([_matrix(generated_at=stamp)]) + + +def test_render_markdown_escapes_every_table_field_not_only_the_reason(): + matrix = _matrix(runs=[_run(model="provider|model")], cells=[_cell(path="A|B.md", scenario="cwd=x|y")]) + lines = cf.render_markdown(matrix).splitlines() + assert "| toy | codex | 0.166.0 | provider\\|model | ran | - |" in lines + assert "| file | scenario | codex 0.166.0 |" in lines + assert "| A\\|B.md | cwd=x\\|y | loaded |" in lines diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index 12e3a2f8c..91ae33c11 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -53,6 +53,7 @@ "redteam.v1": ({}, "questions.json", None, None), "need_graph.v1": ({}, "dpnp-io", None, None), "eval_coverage.v1": ({}, "dpnp-io", None, None), + "reach_matrix.v1": ({}, None, None, "2026-10-31T04:37:00+00:00"), } diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 243e6ebd6..ca384b305 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -33,6 +33,7 @@ "questions.v2": {"questions": 3}, "repo_budget.v1": {"entries": 9}, "effective_instructions.v1": {"receipts": 1}, + "reach_matrix.v1": {"runs": 2, "cells": 3}, "repo_environment.v1": {"instructions": 4, "findings": 13}, "repo_readiness.v1": {"files": 2, "claims": 52}, "result.v1": {"cells": 6},