diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index 558dcf9b..789eada7 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -52,6 +52,10 @@ jobs: release: runs-on: ubuntu-24.04 timeout-minutes: 30 + outputs: + version: ${{ steps.vars.outputs.version }} + dist_owner: ${{ steps.vars.outputs.dist_owner }} + dist_name: ${{ steps.vars.outputs.dist_name }} steps: - name: Checkout monorepo uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -165,6 +169,24 @@ jobs: fi fi + - name: Build and validate Codex submission from selected source + if: inputs.plugin == 'codex' + env: + RELEASE_VERSION: ${{ steps.vars.outputs.version }} + run: | + set -euo pipefail + make test-codex-packaging codex-submission + test -f "dist/submissions/braintrust-$RELEASE_VERSION.zip" + + - name: Save Codex submission ZIP + if: inputs.plugin == 'codex' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: braintrust-mcp-${{ inputs.source_sha }}-${{ steps.vars.outputs.version }} + path: dist/submissions/braintrust-${{ steps.vars.outputs.version }}.zip* + if-no-files-found: error + overwrite: true + - name: Record monorepo release id: source-release if: ${{ inputs.record }} @@ -226,6 +248,52 @@ jobs: --title "$DIST_TAG" \ --generate-notes + # Separate job: if asset upload fails after release creation, Re-run failed + # jobs retries attachment without repeating deployment/tag preflight. + attach-codex-submission: + needs: release + if: inputs.plugin == 'codex' && inputs.record + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Download validated submission from release build + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: braintrust-mcp-${{ inputs.source_sha }}-${{ needs.release.outputs.version }} + path: submission + + - name: Generate distribution-repo token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} + private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} + owner: ${{ needs.release.outputs.dist_owner }} + repositories: ${{ needs.release.outputs.dist_name }} + permission-contents: write + + - name: Attach MCP-only ZIP to distribution release + working-directory: submission + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + DIST_REPO: ${{ inputs.dist_repo }} + VERSION: ${{ needs.release.outputs.version }} + run: | + set -euo pipefail + python3 - "$VERSION" <<'PY' + import hashlib + from pathlib import Path + import sys + archive = Path(f"braintrust-{sys.argv[1]}.zip") + expected = f"{hashlib.sha256(archive.read_bytes()).hexdigest()} {archive.name}\n" + if archive.with_suffix(".zip.sha256").read_text() != expected: + sys.exit("Submission ZIP checksum mismatch") + PY + gh release upload "v$VERSION" "braintrust-$VERSION.zip" "braintrust-$VERSION.zip.sha256" \ + --repo "$DIST_REPO" --clobber + # Post-deploy end-to-end smoke test (codex only): install the just-deployed # plugin from the marketplace on Linux + both macOS arches and assert a real # session traces. It's a verification, not a gate — the deploy already happened. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbf65f51..9c3c80f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,14 @@ jobs: fi - name: Build and validate plugins run: make test + - name: Test Codex packaging and release contracts + run: make test-codex-packaging codex-submission + - name: Save MCP submission ZIP + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: braintrust-mcp-submission + path: dist/submissions/braintrust-*.zip* + if-no-files-found: error - name: Package Cursor extension working-directory: dist/cursor run: | @@ -349,6 +357,11 @@ jobs: npm exec --prefix "${{ runner.temp }}/coding-agents" -- claude --version npm exec --prefix "${{ runner.temp }}/coding-agents" -- opencode --version npm exec --prefix "${{ runner.temp }}/coding-agents" -- pi --version + - name: Check installed Codex MCP loading + if: runner.os == 'Linux' + env: + CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex + run: make test-codex-mcp-loading - name: Check formatting if: runner.os == 'Linux' run: cargo fmt --manifest-path bt-daemon/Cargo.toml -- --check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12077288..8dae338c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: printf 'paths<.zip` to the distribution GitHub Release. OpenAI submission remains manual. + Human approval and merge into `main` authorize tagging the merge commit and publishing this version to the production destination. This preparation does not tag or deploy anything. - name: Summarize release pull request diff --git a/.gitignore b/.gitignore index 659c91a1..e35d0d29 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build output dist/ +__pycache__/ # Local credentials and environment overrides .env diff --git a/Makefile b/Makefile index 1a2c3ce8..59240d20 100644 --- a/Makefile +++ b/Makefile @@ -60,3 +60,16 @@ $(PUBLISH_RULES): publish-%: clean: @rm -rf "$(DIST)" + +# MCP-only package for manual OpenAI submission. Never publish automatically. +.PHONY: codex-submission test-codex-packaging test-codex-mcp-loading +codex-submission: validate-codex + python3 src/plugins/codex/package.py zip "$(DIST)/codex" "$(DIST)/submissions" + +test-codex-packaging: + python3 -m unittest discover -s src/plugins/codex/tests -p 'test_*.py' -v + +# Requires Codex on PATH (or the integration harness's CODEX_BIN override). +# Loads the installed MCP entry without OAuth, inference, or tool calls. +test-codex-mcp-loading: validate-codex + node src/plugins/codex/tests/mcp-loading.mjs "$(DIST)/codex" diff --git a/README.md b/README.md index 927de449..94bbe5ba 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,12 @@ This monorepo contains the plugins and the shared Rust tracing daemon embedded in the Braintrust CLI (`bt`). Plugins forward native events locally; the daemon builds traces, journals events for recovery, and delivers spans to Braintrust. +This repository also packages the [Braintrust +plugin](src/plugins/codex/content/plugins/braintrust/README.md) for submission to +the official ChatGPT and Codex plugin directory. It connects to Braintrust's +hosted MCP server through OAuth and requires no `bt` installation. Installing it +does not enable session tracing. + ## Get started Install your coding agent and the @@ -82,6 +88,9 @@ JavaScript packages, and stable Rust for the daemon. Run from the monorepo root: ```bash make build # Build all plugins into dist/ make build-codex # Build one plugin +make codex-submission # Validate and build the MCP-only OpenAI ZIP +make test-codex-packaging # Test Codex packaging and release contracts +make test-codex-mcp-loading # Check installed MCP loading with a local Codex CLI make test # Build and validate plugin packages and forwarders cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked ``` diff --git a/scripts/set-plugin-version.py b/scripts/set-plugin-version.py index 77688ee3..22ffdfdf 100644 --- a/scripts/set-plugin-version.py +++ b/scripts/set-plugin-version.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 """Set a release version on every distributed version surface for an agent. -Versioning is per-plugin: every plugin under an agent carries its own -.-plugin/plugin.json with a `version` field. The marketplace manifest is -NOT touched. +Versioning is per distribution: portable root manifests and compatibility +manifests share one version. Marketplace manifests are NOT touched. Only version values are rewritten, so surrounding files keep their formatting. @@ -14,12 +13,15 @@ import re import sys -# agent -> glob of its per-plugin manifests (relative to repo root) +# agent -> globs of its per-plugin manifests (relative to repo root) MANIFEST_GLOBS = { - "claude": "src/plugins/claude/content/plugins/*/.claude-plugin/plugin.json", - "codex": "src/plugins/codex/content/plugins/*/.codex-plugin/plugin.json", - "grok": "src/plugins/grok/content/.grok-plugin/plugin.json", - "cursor": "src/plugins/cursor/content/.cursor-plugin/plugin.json", + "claude": ["src/plugins/claude/content/plugins/*/.claude-plugin/plugin.json"], + "codex": [ + "src/plugins/codex/content/plugins/*/.codex-plugin/plugin.json", + "src/plugins/codex/content/plugins/*/plugin.json", + ], + "grok": ["src/plugins/grok/content/.grok-plugin/plugin.json"], + "cursor": ["src/plugins/cursor/content/.cursor-plugin/plugin.json"], } VERSION_RE = re.compile(r'("version"\s*:\s*")[^"]*(")') @@ -27,13 +29,18 @@ def main() -> None: if len(sys.argv) != 3: sys.exit("usage: set-plugin-version.py ") agent, version = sys.argv[1], sys.argv[2].lstrip("v") - pattern = MANIFEST_GLOBS.get(agent) - if not pattern: + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + sys.exit("version must be MAJOR.MINOR.PATCH") + patterns = MANIFEST_GLOBS.get(agent) + if not patterns: sys.exit(f"unknown agent '{agent}' (known: {', '.join(MANIFEST_GLOBS)})") - manifests = sorted(glob.glob(pattern)) - if not manifests: - sys.exit(f"no plugin manifests found for '{agent}' ({pattern})") + manifests = [] + for pattern in patterns: + matches = sorted(glob.glob(pattern)) + if not matches: + sys.exit(f"no plugin manifests found for '{agent}' ({pattern})") + manifests.extend(matches) surfaces = [(path, VERSION_RE, "version") for path in manifests] if agent == "cursor": diff --git a/src/plugins/codex/build.sh b/src/plugins/codex/build.sh index 750b1214..80891a1c 100755 --- a/src/plugins/codex/build.sh +++ b/src/plugins/codex/build.sh @@ -4,6 +4,7 @@ # # The Codex marketplace consumes a repo whose ROOT is the marketplace: # .agents/plugins/marketplace.json marketplace manifest +# plugins/braintrust/ hosted MCP package (portable manifest) # plugins/trace-codex/ tracing plugin (thin daemon hook shims) # # The tracing plugin contains no tracing runtime or platform-specific binary; diff --git a/src/plugins/codex/content/.agents/plugins/marketplace.json b/src/plugins/codex/content/.agents/plugins/marketplace.json index 6ae6fecc..f7b47e9b 100644 --- a/src/plugins/codex/content/.agents/plugins/marketplace.json +++ b/src/plugins/codex/content/.agents/plugins/marketplace.json @@ -4,6 +4,18 @@ "displayName": "Braintrust Codex Plugin" }, "plugins": [ + { + "name": "braintrust", + "source": { + "source": "local", + "path": "./plugins/braintrust" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Developer Tools" + }, { "name": "trace-codex", "source": { diff --git a/src/plugins/codex/content/AGENTS.md b/src/plugins/codex/content/AGENTS.md index bd55923b..22cf47e7 100644 --- a/src/plugins/codex/content/AGENTS.md +++ b/src/plugins/codex/content/AGENTS.md @@ -4,26 +4,39 @@ Guidelines for AI agents working in this repo. ## Repo purpose -This repo distributes one [Codex marketplace plugin](https://developers.openai.com/codex/plugins): +This generated repo distributes two independently opt-in +[Codex marketplace plugins](https://developers.openai.com/plugins/build/plugins): + +- `plugins/braintrust/` — hosted MCP with OAuth DCR, portable root `plugin.json` + and `mcp.json`. No hooks, apps, skills, credentials, or local runtime. - `plugins/trace-codex/` — an opt-in plugin that traces Codex sessions to Braintrust (session, turn, and tool spans) via Codex lifecycle hooks. -The plugin is listed in `.agents/plugins/marketplace.json` (the repo marketplace). +Both plugins are listed in `.agents/plugins/marketplace.json`. Preserve the +marketplace name `braintrust-codex-plugins` and tracing identity +`trace-codex@braintrust-codex-plugins`. -Key files for the tracing plugin (see [`plugins/trace-codex/AGENTS.md`](plugins/trace-codex/AGENTS.md) for its architecture): +Key files for the tracing plugin: -- `plugins/trace-codex/AGENTS.md` — architecture and contributor guide for this plugin +- `plugins/trace-codex/README.md` — architecture and setup guide - `plugins/trace-codex/.codex-plugin/plugin.json` — plugin manifest - `plugins/trace-codex/hooks/hooks.json` — lifecycle hook config -- `plugins/trace-codex/src/` — the hook client + event server (compiled to `bin/codex-hook`) +- Tracing hooks forward to `bt trace hook --source codex`; there is no bundled runtime. ## Making changes +- **MCP metadata**: edit `plugins/braintrust/plugin.json` and `mcp.json` in the + source monorepo. Keep portable identity and OpenAI presentation in one manifest. + Keep server-level auth extensions out of the Codex `mcp.json`: current Codex + rejects them. The monorepo ZIP builder adds OAuth DCR metadata only to the + OpenAI submission archive and validates both formats separately. - **Plugin metadata**: edit `plugins/trace-codex/.codex-plugin/plugin.json`. - **Marketplace**: edit `.agents/plugins/marketplace.json` to change the plugin entry, category, or install policy. ## Releasing a new version -1. Bump `"version"` in the relevant plugin's `.codex-plugin/plugin.json`. -2. Commit, tag, and create a GitHub release (see README for exact commands). -3. Do **not** skip the git tag — releases are tracked via tags so users can see a changelog. +Make all source changes in `braintrustdata/braintrust-coding-agent-plugins`. +Both packages share the Codex distribution version. Use the monorepo's reviewed +Release Plugin workflow; do not edit or tag this generated repository manually. +The distribution release carries an MCP-only ZIP for manual OpenAI submission. +Template settings belong to portal setup, not speculative manifest fields. diff --git a/src/plugins/codex/content/README.md b/src/plugins/codex/content/README.md index de130ae8..13cb953f 100644 --- a/src/plugins/codex/content/README.md +++ b/src/plugins/codex/content/README.md @@ -1,11 +1,31 @@ -# Braintrust tracing for Codex +# Braintrust plugins for Codex > **This repository is generated.** It is built from > [braintrustdata/braintrust-coding-agent-plugins](https://github.com/braintrustdata/braintrust-coding-agent-plugins). > Don't edit files here — make changes and file issues in that repository, and they > will be rebuilt into this one. -Trace Codex sessions in Braintrust with the `trace-codex` plugin. +This marketplace contains two independently opt-in plugins: + +- **Braintrust**, `braintrust@braintrust-codex-plugins`: hosted MCP tools with + OAuth. No `bt` dependency or tracing hooks. See the + [MCP installation and endpoint guide](plugins/braintrust/README.md). +- **Trace Codex**, `trace-codex@braintrust-codex-plugins`: session tracing via `bt`. + +For MCP, add this marketplace and select Braintrust: + +```bash +codex plugin marketplace add braintrustdata/braintrust-codex-plugin +codex plugin add braintrust@braintrust-codex-plugins +``` + +The GitHub package connects to NA (US). EU and custom deployments require an +admin-configured local/team copy; GitHub installs do not inherit public portal +template settings. The planned public-directory listing uses one template, +pending OpenAI eligibility and endpoint-template confirmation. Its concrete NA +review URL is not a verified install-time default. + +The sections below describe optional tracing setup only. ## Quickstart @@ -49,5 +69,4 @@ bt trace disable codex ``` See the [tracing plugin guide](plugins/trace-codex/README.md) for hook behavior, -metadata, and tags. This marketplace does not configure the Braintrust MCP -server; use the agent's connector or MCP configuration for MCP access. +metadata, and tags. MCP sign-in and `bt` tracing sign-in are independent. diff --git a/src/plugins/codex/content/install.sh b/src/plugins/codex/content/install.sh index b082774c..f35f4d70 100755 --- a/src/plugins/codex/content/install.sh +++ b/src/plugins/codex/content/install.sh @@ -17,7 +17,8 @@ # correct config entries. # # Usage: -# ./install.sh # install all plugins in this repo +# ./install.sh # install tracing (legacy default) +# ./install.sh braintrust # install only the hosted MCP plugin # ./install.sh trace-codex # install just one plugin (by folder) # # Use ./uninstall.sh to remove. @@ -28,14 +29,11 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MARKETPLACE="braintrust-codex-plugins" -# Plugin folders to install. Default: every folder under plugins/. +# Keep the existing tracing-only default. MCP is selected explicitly. if [ "$#" -gt 0 ]; then PLUGINS=("$@") else - PLUGINS=() - for d in "$REPO_ROOT"/plugins/*/; do - PLUGINS+=("$(basename "$d")") - done + PLUGINS=("trace-codex") fi if ! command -v codex >/dev/null 2>&1; then @@ -80,6 +78,9 @@ build_plugin() { for folder in "${PLUGINS[@]}"; do plugin_src="$REPO_ROOT/plugins/$folder" manifest="$plugin_src/.codex-plugin/plugin.json" + if [ -f "$plugin_src/plugin.json" ]; then + manifest="$plugin_src/plugin.json" + fi if [ ! -f "$manifest" ]; then echo "Skipping '$folder': no manifest at $manifest" >&2 @@ -108,7 +109,7 @@ done echo "" echo "Done. Next steps:" echo " 1. Restart Codex (or start a new session) so it loads the plugins." -echo " 2. Plugin hooks are non-managed: run /hooks in the Codex CLI and trust them" -echo " before they will fire." +echo " 2. For Braintrust MCP, complete OAuth sign-in in the client." +echo " For tracing, run /hooks and trust the tracing hooks before they fire." echo "" echo "After editing plugin files, re-run ./install.sh to re-sync the cached copy." diff --git a/src/plugins/codex/content/plugins/braintrust/LICENSE b/src/plugins/codex/content/plugins/braintrust/LICENSE new file mode 100644 index 00000000..f8a1f4ae --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Braintrust Data, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/plugins/codex/content/plugins/braintrust/README.md b/src/plugins/codex/content/plugins/braintrust/README.md new file mode 100644 index 00000000..1c36eb23 --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/README.md @@ -0,0 +1,86 @@ +# Braintrust + +Query traces, investigate failures, manage prompts and datasets, and run +evaluations using the hosted [Braintrust MCP server](https://www.braintrust.dev/docs/integrations/developer-tools/mcp). +Requires a Braintrust account with access to your organization's data plane. +Tools use that account's permissions and can read or change Braintrust data. + +This package contains a remote Streamable HTTP connection with OAuth Dynamic +Client Registration (DCR). The client discovers registration and authorization +endpoints. No API key, client secret, authorization header, local process, `bt` +installation, hooks, or desktop execution is required by this package. +MCP authorization is independent of optional `bt` session tracing. + +Codex discovers OAuth from the server. Its package declares only the HTTP +transport and server URL because current Codex versions reject explicit OAuth +metadata in `mcp.json`. The OpenAI submission ZIP adds the documented OAuth DCR +declaration during packaging; it does not change the GitHub package. See the +[Codex compatibility issue](https://github.com/openai/codex/issues/53658) and +[OpenAI authentication fields](https://developers.openai.com/plugins/build/plugins#configure-mcp-authentication). + +## GitHub marketplace + +After this version is released to the generated distribution repository: + +```bash +codex plugin marketplace add braintrustdata/braintrust-codex-plugin +codex plugin add braintrust@braintrust-codex-plugins +``` + +You can also select **Braintrust** in a supported client's marketplace browser. +Complete the client's OAuth flow, then start a new conversation. Installing +Braintrust does not install or enable the separate tracing plugin. + +The GitHub package's `mcp.json` has a concrete NA URL. It does not implement +public-directory template configuration or an endpoint picker. Current Codex +plugin policy settings document enablement and tool approval, not a server URL +override. To use EU or a custom endpoint today, have your admin copy this +package into a local/team marketplace and replace only +`mcpServers.braintrust.url` in that copy's `mcp.json` with the complete URL below. +The canonical release validator intentionally pins the NA review configuration; +an organization-specific copy needs its own endpoint validation. Avoid editing +the installed cache: marketplace upgrades can replace it. Do not enable both +the NA package and a duplicate connection for the same tools. + +| Data plane | MCP URL | +| --- | --- | +| NA (called US in Braintrust docs) | `https://api.braintrust.dev/mcp` | +| EU | `https://api-eu.braintrust.dev/mcp` | +| Custom / self-hosted | Copy the complete MCP URL from **Settings > Data plane**. | + +Braintrust documents the custom URL as the configured API base URL with `/mcp` +appended. Preserve any deployment-specific path prefix or other supplied URL +details; do not reduce every custom URL to `https:///mcp`. + +## Public-directory template listing (pending external setup) + +The intended public listing is one template-based Braintrust plugin for all +data planes. The package supplies a concrete NA endpoint for review and scans. +It does **not** establish NA as the installation default. Preselect/prefill NA +only if OpenAI confirms the installation UI supports it. + +OpenAI describes **Example MCP Server URL** and **Template MCP Server URL** as +submission settings. The candidate template `https://{data_plane_host}/mcp` +needs OpenAI confirmation: the published example varies a subdomain, not an +arbitrary full host. That candidate also cannot represent an API base with a +path prefix. Support for those custom URL shapes and parameter defaults must +be confirmed before configuring the listing. Do not place a template URL or +invent template/default fields in `mcp.json` or `plugin.json`. + +Template submissions are restricted to trusted developers with an established +OpenAI relationship. Portal access, eligibility, supported parameter shapes, +and authenticated installation evidence are still prerequisites. Portal +template settings are not automatically applied to GitHub marketplace installs. +This package has not been approved or submitted. + +## Support and package assets + +See [Braintrust troubleshooting](https://www.braintrust.dev/docs/kb/troubleshooting-guides) +or contact [support@braintrust.dev](mailto:support@braintrust.dev). +[Privacy policy](https://www.braintrust.dev/legal/privacy-policy) · +[Terms of service](https://www.braintrust.dev/legal/terms-of-service). + +`assets/icon.png` is the existing 200×200 Braintrust icon copied verbatim from +this monorepo's `src/plugins/cursor/content/icon.png`; the brand color and legal +URLs also match existing Braintrust plugin metadata. No synthetic screenshots +or invented contact details are included. diff --git a/src/plugins/codex/content/plugins/braintrust/assets/icon.png b/src/plugins/codex/content/plugins/braintrust/assets/icon.png new file mode 100644 index 00000000..fb64bacc Binary files /dev/null and b/src/plugins/codex/content/plugins/braintrust/assets/icon.png differ diff --git a/src/plugins/codex/content/plugins/braintrust/mcp.json b/src/plugins/codex/content/plugins/braintrust/mcp.json new file mode 100644 index 00000000..d5b9bcbd --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/mcp.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", + "mcpServers": { + "braintrust": { + "type": "streamable-http", + "url": "https://api.braintrust.dev/mcp" + } + } +} diff --git a/src/plugins/codex/content/plugins/braintrust/plugin.json b/src/plugins/codex/content/plugins/braintrust/plugin.json new file mode 100644 index 00000000..a6f1a69d --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/plugin.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "braintrust", + "version": "2.1.1", + "description": "Explore Braintrust traces, manage prompts and datasets, and run evaluations.", + "author": { + "name": "Braintrust", + "email": "support@braintrust.dev", + "url": "https://www.braintrust.dev/" + }, + "homepage": "https://www.braintrust.dev/docs/integrations/developer-tools/mcp", + "repository": "https://github.com/braintrustdata/braintrust-coding-agent-plugins", + "license": "MIT", + "keywords": ["braintrust", "evaluations", "observability", "mcp"], + "extensions": { + "com.openai": { + "interface": { + "displayName": "Braintrust", + "shortDescription": "Explore traces and run evals", + "longDescription": "Query production traces, investigate failures, manage prompts and datasets, and run evaluations in Braintrust. Sign in with a Braintrust account to access your organization's data. Tools use your account's permissions and can read or modify data. This package connects to Braintrust's US data plane.", + "developerName": "Braintrust", + "category": "Developer Tools", + "capabilities": ["Read", "Write"], + "websiteURL": "https://www.braintrust.dev/", + "supportURL": "https://www.braintrust.dev/docs/kb/troubleshooting-guides", + "privacyPolicyURL": "https://www.braintrust.dev/legal/privacy-policy", + "termsOfServiceURL": "https://www.braintrust.dev/legal/terms-of-service", + "defaultPrompt": [ + "Show me my Braintrust projects.", + "Summarize recent errors in my Braintrust project.", + "Compare the results of two experiments in my Braintrust project." + ], + "brandColor": "#2C1FEB", + "composerIcon": "./assets/icon.png", + "logo": "./assets/icon.png" + } + } + } +} diff --git a/src/plugins/codex/content/uninstall.sh b/src/plugins/codex/content/uninstall.sh index 49ebf799..09e726b0 100755 --- a/src/plugins/codex/content/uninstall.sh +++ b/src/plugins/codex/content/uninstall.sh @@ -9,8 +9,7 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MARKETPLACE="braintrust-codex-plugins" -CODEX_HOME="${CODEX_HOME:-$HOME/.codex}" -CACHE_DIR="$CODEX_HOME/plugins/cache/$MARKETPLACE" +CACHE_DIR="${CODEX_HOME:-$HOME/.codex}/plugins/cache/$MARKETPLACE" if ! command -v codex >/dev/null 2>&1; then echo "Error: 'codex' CLI not found." >&2 @@ -21,6 +20,9 @@ echo "Uninstalling Braintrust Codex plugins..." for d in "$REPO_ROOT"/plugins/*/; do manifest="$d/.codex-plugin/plugin.json" + if [ -f "$d/plugin.json" ]; then + manifest="$d/plugin.json" + fi [ -f "$manifest" ] || continue # Installable name is the manifest `name`, which may differ from the folder. name="$(grep -o '"name"[[:space:]]*:[[:space:]]*"[^"]*"' "$manifest" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')" diff --git a/src/plugins/codex/package.py b/src/plugins/codex/package.py new file mode 100644 index 00000000..cd3d8276 --- /dev/null +++ b/src/plugins/codex/package.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Offline distribution and MCP-only submission validation/ZIP creation. + +This is a narrow package contract, not a general OpenAI schema validator. +Template settings are portal configuration, not speculative JSON fields. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import struct +import tempfile +from urllib.parse import urlsplit +import zipfile + + +SCHEMA_BASE = "https://agent-plugins.org/schemas/1.0.0/" +PUBLIC_FILES = frozenset({"plugin.json", "mcp.json", "README.md", "LICENSE", "assets/icon.png"}) +EVENTS = { + "Interrupt", "PermissionRequest", "PostCompact", "PostToolUse", "PreCompact", + "PreToolUse", "SessionEnd", "SessionStart", "Stop", "SubagentStart", "SubagentStop", + "UserPromptSubmit", +} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def read_json(path): + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, f"duplicate JSON key: {key}") + result[key] = value + return result + return json.loads(path.read_text(), object_pairs_hook=unique) + + +def keys(value, allowed): + require(isinstance(value, dict), "expected an object") + require(set(value) == set(allowed), f"unexpected/missing fields: {set(value) ^ set(allowed)}") + + +def version(value): + require(isinstance(value, str) and re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", value), "expected MAJOR.MINOR.PATCH version") + + +def https_url(value): + require(isinstance(value, str), "URL must be a string") + url = urlsplit(value) + require(url.scheme == "https" and url.hostname and not url.username and not url.password + and not url.query and not url.fragment and not re.search(r"[{}\s]", value), + f"expected concrete credential-free HTTPS URL: {value}") + + +def inventory(root): + require(root.is_dir() and not root.is_symlink(), f"missing or symlinked package: {root}") + files = set() + for path in root.rglob("*"): + require(not path.is_symlink(), f"symlink not allowed: {path}") + require(path.is_dir() or path.is_file(), f"special file not allowed: {path}") + if path.is_file(): + files.add(path.relative_to(root).as_posix()) + return files + + +def submission_mcp_config(config): + """Add portal auth metadata without changing the Codex-compatible source. + + Codex currently rejects server-level extensions, even though OpenAI's + submission format documents them. Keep this projection in the ZIP builder. + """ + # Round trip to leave the caller's validated distribution config untouched. + config = json.loads(json.dumps(config)) + config["mcpServers"]["braintrust"]["extensions"] = { + "com.openai": {"auth": {"type": "oauth", "client": {"mode": "dcr"}}}, + } + return config + + +def validate_mcp_config(config, *, submission): + # Exact contracts reject headers, credentials, processes, speculative + # template fields, and submission-only metadata in the Codex distribution. + expected = { + "$schema": SCHEMA_BASE + "mcp.schema.json", + "mcpServers": {"braintrust": { + "type": "streamable-http", "url": "https://api.braintrust.dev/mcp", + }}, + } + if submission: + expected = submission_mcp_config(expected) + message = "Submission MCP must use the concrete NA URL, Streamable HTTP, and discovery-based OAuth DCR" + else: + message = "Codex MCP must use the concrete NA URL and Streamable HTTP without server extensions or credentials" + require(config == expected, message) + + +def validate_mcp_package(root, *, submission=False): + require(inventory(root) == PUBLIC_FILES, "public package files must match MCP-only allowlist (no hooks, apps, skills, scripts, or marketplace)") + require({p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_dir()} == {"assets"}, "unexpected public package directory") + manifest = read_json(root / "plugin.json") + keys(manifest, {"$schema", "name", "version", "description", "author", "homepage", "repository", "license", "keywords", "extensions"}) + require(manifest["$schema"] == SCHEMA_BASE + "plugin.schema.json", "wrong portable plugin schema") + require(manifest["name"] == "braintrust", "MCP identity must be braintrust") + version(manifest["version"]) + require(manifest["license"] == "MIT", "expected MIT package license") + keys(manifest["author"], {"name", "email", "url"}) + require(manifest["author"]["name"] == "Braintrust" and manifest["author"]["email"] == "support@braintrust.dev", "unexpected publisher") + for value in [manifest["homepage"], manifest["repository"], manifest["author"]["url"]]: + https_url(value) + require(isinstance(manifest["description"], str) and manifest["description"].strip(), "missing description") + require(isinstance(manifest["keywords"], list) and all(isinstance(k, str) and k.strip() for k in manifest["keywords"]), "invalid keywords") + keys(manifest["extensions"], {"com.openai"}) + overlay = manifest["extensions"]["com.openai"] + keys(overlay, {"interface"}) # No hooks/apps/mcpServers/desktop requirements. + interface = overlay["interface"] + keys(interface, {"displayName", "shortDescription", "longDescription", "developerName", "category", "capabilities", "websiteURL", "supportURL", "privacyPolicyURL", "termsOfServiceURL", "defaultPrompt", "brandColor", "composerIcon", "logo"}) + require(interface["displayName"] == interface["developerName"] == "Braintrust", "unexpected display name/publisher") + require(interface["category"] == "Developer Tools", "unexpected listing category") + for field, limit in [("displayName", 30), ("shortDescription", 30), ("longDescription", 4000), ("developerName", 80)]: + value = interface[field] + require(isinstance(value, str) and value.strip() and len(value) <= limit and not re.search(r"[\x00-\x1f]", value), f"invalid {field}") + for field in ["websiteURL", "supportURL", "privacyPolicyURL", "termsOfServiceURL"]: + https_url(interface[field]) + require(len(interface[field]) <= 1024, f"{field} too long") + require(interface["capabilities"] == ["Read", "Write"], "unexpected capabilities") + prompts = interface["defaultPrompt"] + require(isinstance(prompts, list) and 1 <= len(prompts) <= 3 and all(isinstance(p, str) and p.strip() and len(p) <= 128 and "@" not in p for p in prompts), "invalid starter prompts") + require(len(set(prompts)) == len(prompts), "duplicate starter prompts") + require(interface["brandColor"] == "#2C1FEB", "unexpected brand color") + for field in ["composerIcon", "logo"]: + require(interface[field] == "./assets/icon.png", f"unexpected {field} path") + data = (root / "assets/icon.png").read_bytes() + require(24 <= len(data) <= 5 * 1024 * 1024 and data[:8] == b"\x89PNG\r\n\x1a\n" and data[12:16] == b"IHDR", "invalid PNG asset") + width, height = struct.unpack(">II", data[16:24]) + require(width == height and 48 <= width <= 4096, "listing icon must be square, 48..4096px") + validate_mcp_config(read_json(root / "mcp.json"), submission=submission) + return manifest + + +def validate_distribution(root): + inventory(root) + marketplace = read_json(root / ".agents/plugins/marketplace.json") + keys(marketplace, {"name", "interface", "plugins"}) + require(marketplace["name"] == "braintrust-codex-plugins", "marketplace identity changed") + require(marketplace["interface"] == {"displayName": "Braintrust Codex Plugin"}, "marketplace display identity changed") + entries = marketplace["plugins"] + require(isinstance(entries, list) and len(entries) == 2 and {p["name"] for p in entries} == {"braintrust", "trace-codex"}, "expected exactly MCP and tracing entries") + require({p.name for p in (root / "plugins").iterdir()} == {"braintrust", "trace-codex"}, "unexpected plugin paths") + for entry in entries: + keys(entry, {"name", "source", "policy", "category"}) + name = entry["name"] + require(entry["source"] == {"source": "local", "path": f"./plugins/{name}"}, "unexpected marketplace source/path") + require(entry["policy"] == {"installation": "AVAILABLE", "authentication": "ON_INSTALL"}, "plugins must be separately opt-in") + require(entry["category"] == ("Developer Tools" if name == "braintrust" else "Observability"), "wrong marketplace category") + public = validate_mcp_package(root / "plugins/braintrust") + tracing = root / "plugins/trace-codex" + manifest = read_json(tracing / ".codex-plugin/plugin.json") + require(manifest["name"] == "trace-codex" and manifest["hooks"] == "./hooks/hooks.json", "tracing identity/hooks changed") + version(manifest["version"]) + require(manifest["version"] == public["version"], "Codex packages must share a release version") + require(not ({"apps", "mcpServers", "skills"} & set(manifest)), "unexpected tracing components") + require("plugin.json" not in inventory(tracing), "conflicting tracing manifest") + require(not any(p.name in {"mcp.json", ".mcp.json", ".app.json", "package.json", "pnpm-lock.yaml", "tsconfig.json"} + or p.name.startswith("codex-hook") for p in tracing.rglob("*")), "legacy tracing runtime or MCP/app configuration") + hooks = read_json(tracing / "hooks/hooks.json") + keys(hooks, {"hooks"}) + require(set(hooks["hooks"]) == EVENTS, "unexpected tracing events") + for definitions in hooks["hooks"].values(): + require(isinstance(definitions, list) and definitions, "empty tracing hook definitions") + for definition in definitions: + require(isinstance(definition["hooks"], list) and definition["hooks"], "empty tracing hooks") + for hook in definition["hooks"]: + require(hook["type"] == "command" and hook["command"] == hook["commandWindows"] == "bt trace hook --source codex", "hooks must use daemon forwarders") + for path in root.rglob("*"): + if path.name in {"mcp.json", ".mcp.json", ".app.json"}: + require(path == root / "plugins/braintrust/mcp.json", "unexpected distribution MCP/app configuration") + return public + + +def validate_zip(path): + with zipfile.ZipFile(path) as archive, tempfile.TemporaryDirectory() as temp: + infos = archive.infolist() + require(len(infos) == len(PUBLIC_FILES) and {i.filename for i in infos} == PUBLIC_FILES, + "ZIP must contain only the MCP package with plugin.json at archive root") + for info in infos: + require(stat.S_IFMT(info.external_attr >> 16) == stat.S_IFREG, "ZIP entry must be a regular file") + require(info.file_size <= 5 * 1024 * 1024, "oversized ZIP entry") + archive.extractall(temp) # Paths were checked against exact allowlist. + return validate_mcp_package(Path(temp), submission=True) + + +def build_zip(root, output): + manifest = validate_distribution(root) + package = root / "plugins/braintrust" + output.mkdir(parents=True, exist_ok=True) + path = output / f"braintrust-{manifest['version']}.zip" + # Fixed order, timestamps, permissions, no compression-version dependency. + with tempfile.TemporaryDirectory(dir=output) as temp: + staged = Path(temp) / path.name + with zipfile.ZipFile(staged, "w", compression=zipfile.ZIP_STORED) as archive: + for name in sorted(PUBLIC_FILES): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + data = (package / name).read_bytes() + if name == "mcp.json": + data = (json.dumps(submission_mcp_config(read_json(package / name)), indent=2) + "\n").encode() + archive.writestr(info, data) + validate_zip(staged) + os.replace(staged, path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + path.with_suffix(".zip.sha256").write_text(f"{digest} {path.name}\n") + return path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["validate", "validate-public", "validate-zip", "zip"]) + parser.add_argument("path", type=Path) + parser.add_argument("output", type=Path, nargs="?") + args = parser.parse_args() + try: + if args.command == "zip": + require(args.output is not None, "zip requires output directory") + print(build_zip(args.path, args.output)) + else: + {"validate": validate_distribution, "validate-public": lambda path: validate_mcp_package(path, submission=True), "validate-zip": validate_zip}[args.command](args.path) + print(f"validate: {args.command} OK ({args.path})") + except (ValueError, KeyError, TypeError, OSError, zipfile.BadZipFile) as error: + parser.exit(1, f"validate: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/src/plugins/codex/tests/mcp-loading.mjs b/src/plugins/codex/tests/mcp-loading.mjs new file mode 100644 index 00000000..0f7c5f86 --- /dev/null +++ b/src/plugins/codex/tests/mcp-loading.mjs @@ -0,0 +1,114 @@ +// Exercise Codex's MCP parser, which plugin add/list alone does not validate. +// This reads installed plugin metadata; it never signs in or calls MCP tools. +import assert from "node:assert/strict" +import { cp, mkdtemp, mkdir, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { createInterface } from "node:readline" +import test from "node:test" +import { + execFileBackground, + spawnBackground, +} from "../../../runtime/js-daemon-client/src/background-process.ts" + +const distribution = resolve(process.argv[2] ?? "dist/codex") +const codex = process.env.CODEX_BIN || "codex" + +async function readPlugin(env, cwd, marketplace) { + const child = spawnBackground(codex, ["app-server", "--stdio"], { + env, + cwd, + stdio: ["pipe", "pipe", "pipe"], + }) + const pending = new Map() + let stderr = "" + child.stderr.setEncoding("utf8").on("data", (data) => { stderr += data }) + const lines = createInterface({ input: child.stdout }) + const fail = (error) => { + for (const request of pending.values()) request.reject(error) + pending.clear() + } + child.on("error", fail) + const exited = new Promise((done) => child.once("close", () => { + fail(new Error(`Codex app-server exited: ${stderr}`)) + done() + })) + lines.on("line", (line) => { + try { + const message = JSON.parse(line) + const request = pending.get(message.id) + if (request) { + pending.delete(message.id) + if (message.error) request.reject(new Error(JSON.stringify(message.error))) + else request.resolve(message.result) + } + } catch (error) { fail(error) } + }) + let id = 0 + async function call(method, params) { + const requestId = ++id + let timer + try { + return await new Promise((resolve, reject) => { + timer = setTimeout(() => reject(new Error(`Timeout: ${method}\n${stderr}`)), 10_000) + pending.set(requestId, { resolve, reject }) + child.stdin.write(JSON.stringify({ id: requestId, method, params }) + "\n") + }) + } finally { + clearTimeout(timer) + pending.delete(requestId) + } + } + try { + await call("initialize", { + clientInfo: { name: "braintrust_mcp_loading_test", version: "1.0.0" }, + capabilities: { experimentalApi: true }, + }) + child.stdin.write(JSON.stringify({ method: "initialized", params: {} }) + "\n") + const result = await call("plugin/read", { + marketplacePath: join(marketplace, ".agents/plugins/marketplace.json"), + pluginName: "braintrust", + }) + return result.plugin + } finally { + const killTimer = setTimeout(() => child.kill("SIGKILL"), 1000) + killTimer.unref() + child.kill() + await exited + clearTimeout(killTimer) + lines.close() + } +} + +test("installed Braintrust MCP server loads in Codex", { timeout: 40_000 }, async () => { + const root = await mkdtemp(join(tmpdir(), "braintrust-mcp-loading-")) + try { + const marketplace = join(root, "marketplace") + const home = join(root, "codex") + await mkdir(home) + await cp(distribution, marketplace, { recursive: true }) + const env = { ...process.env, CODEX_HOME: home } + // Separate profile and cwd prevent loading user credentials or repo hooks. + const options = { env, cwd: root, encoding: "utf8", timeout: 10_000 } + await execFileBackground(codex, ["plugin", "marketplace", "add", marketplace], options) + const install = await execFileBackground(codex, [ + "plugin", "add", "braintrust@braintrust-codex-plugins", "--json", + ], options) + const cached = JSON.parse(install.stdout).installedPath + const config = JSON.parse(await readFile(join(cached, "mcp.json"), "utf8")) + assert.equal(config.mcpServers.braintrust.type, "streamable-http") + const plugin = await readPlugin(env, root, marketplace) + assert.equal(plugin.summary.installed, true) + assert.deepEqual(plugin.mcpServers, ["braintrust"], "MCP entry was dropped by Codex's parser") + assert.deepEqual(plugin.hooks, []) + assert.deepEqual(plugin.apps, []) + const listed = await execFileBackground(codex, [ + "plugin", "list", "--marketplace", "braintrust-codex-plugins", "--json", + ], options) + assert.deepEqual(JSON.parse(listed.stdout).installed.map((p) => p.pluginId), [ + "braintrust@braintrust-codex-plugins", + ]) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/src/plugins/codex/tests/test_packaging.py b/src/plugins/codex/tests/test_packaging.py new file mode 100644 index 00000000..f7783efa --- /dev/null +++ b/src/plugins/codex/tests/test_packaging.py @@ -0,0 +1,304 @@ +"""Failure cases for package isolation and the reviewed-release boundary.""" +import glob +import importlib.util +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest +import zipfile + + +REPO = Path(__file__).resolve().parents[4] +CODEX = REPO / "src/plugins/codex" +spec = importlib.util.spec_from_file_location("codex_package", CODEX / "package.py") +package = importlib.util.module_from_spec(spec) +spec.loader.exec_module(package) + + +def run_step(workflow, name): + """Extract the actual workflow shell body so tests exercise release code.""" + text = (REPO / ".github/workflows" / workflow).read_text() + tail = text.split(f"- name: {name}\n", 1)[1].split(" run: |\n", 1)[1] + lines = [] + for line in tail.splitlines(): + if line and not line.startswith(" "): + break + lines.append(line[10:]) + return "\n".join(lines) + + +class PackagingTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.dist = self.root / "dist" + shutil.copytree(CODEX / "content", self.dist) + self.public = self.dist / "plugins/braintrust" + + def edit(self, path, fn): + data = json.loads(path.read_text()) + fn(data) + path.write_text(json.dumps(data)) + + def test_valid_distribution_and_reproducible_zip(self): + first = package.build_zip(self.dist, self.root / "one") + # A timestamp change must not change the archive bytes. + for path in self.public.rglob("*"): + os.utime(path, (1700000000, 1700000000)) + second = package.build_zip(self.dist, self.root / "two") + self.assertEqual(first.read_bytes(), second.read_bytes()) + manifest = package.validate_zip(first) + self.assertEqual(first.name, f"braintrust-{manifest['version']}.zip") + with zipfile.ZipFile(first) as archive: + self.assertEqual(set(archive.namelist()), package.PUBLIC_FILES) + self.assertEqual(json.loads(archive.read("plugin.json"))["name"], "braintrust") + + def test_accidental_public_components_rejected_before_packaging(self): + for name in ["hooks/hooks.json", ".app.json", ".codex-plugin/plugin.json", "skills/workflow/SKILL.md", "trace-codex/README.md", ".agents/plugins/marketplace.json", "credentials.json"]: + with self.subTest(name=name): + target = self.public / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("{}") + with self.assertRaisesRegex(ValueError, "allowlist"): + package.build_zip(self.dist, self.root / "output") + target.unlink() + parent = target.parent + while parent != self.public: + parent.rmdir() + parent = parent.parent + + def test_symlink_rejected(self): + icon = self.public / "assets/icon.png" + icon.unlink() + icon.symlink_to(CODEX / "content/plugins/braintrust/assets/icon.png") + with self.assertRaisesRegex(ValueError, "symlink"): + package.validate_mcp_package(self.public) + + def test_zip_injection_rejected(self): + original = package.build_zip(self.dist, self.root / "valid") + for name in ["hooks/hooks.json", "../plugin.json", "braintrust/plugin.json", "plugin.json"]: + with self.subTest(name=name): + bad = self.root / "bad.zip" + shutil.copyfile(original, bad) + with zipfile.ZipFile(bad, "a") as archive: + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + archive.writestr(name, "{}") + with self.assertRaisesRegex(ValueError, "archive root"): + package.validate_zip(bad) + + def test_whole_marketplace_zip_rejected(self): + bad = self.root / "marketplace.zip" + with zipfile.ZipFile(bad, "w") as archive: + for path in self.dist.rglob("*"): + if path.is_file(): + archive.write(path, path.relative_to(self.dist)) + with self.assertRaisesRegex(ValueError, "archive root"): + package.validate_zip(bad) + + def test_zip_symlink_rejected(self): + original = package.build_zip(self.dist, self.root / "valid") + bad = self.root / "symlink.zip" + with zipfile.ZipFile(original) as source, zipfile.ZipFile(bad, "w") as target: + for info in source.infolist(): + if info.filename == "assets/icon.png": + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + target.writestr(info, source.read(info.filename)) + with self.assertRaisesRegex(ValueError, "regular file"): + package.validate_zip(bad) + + def test_codex_transport_and_template_misconfiguration_rejected(self): + path = self.public / "mcp.json" + original = path.read_text() + mutations = [ + lambda s: s.update(url="https://{data_plane_host}/mcp"), + lambda s: s.update(url="https://api-eu.braintrust.dev/mcp"), + lambda s: s.update(type="sse"), + lambda s: s.update(type="stdio", command="bt"), + lambda s: s.update(headers={"Authorization": "Bearer fake-test-secret"}), + lambda s: s.update(default="api.braintrust.dev"), + ] + for index, mutation in enumerate(mutations): + with self.subTest(index=index): + path.write_text(original) + self.edit(path, lambda d: mutation(d["mcpServers"]["braintrust"])) + with self.assertRaisesRegex(ValueError, "Codex MCP"): + package.validate_mcp_package(self.public) + + def test_zip_adds_submission_auth_without_mutating_distribution(self): + path = self.public / "mcp.json" + original = path.read_bytes() + archive = package.build_zip(self.dist, self.root / "output") + with zipfile.ZipFile(archive) as zipped: + config = json.loads(zipped.read("mcp.json")) + self.assertEqual(config["mcpServers"]["braintrust"], { + "type": "streamable-http", + "url": "https://api.braintrust.dev/mcp", + "extensions": {"com.openai": {"auth": {"type": "oauth", "client": {"mode": "dcr"}}}}, + }) + self.assertEqual(path.read_bytes(), original) + self.assertNotIn("extensions", json.loads(original)["mcpServers"]["braintrust"]) + package.validate_distribution(self.dist) + + def test_codex_rejects_submission_auth_metadata(self): + path = self.public / "mcp.json" + path.write_text(json.dumps(package.submission_mcp_config(json.loads(path.read_text())))) + with self.assertRaisesRegex(ValueError, "without server extensions"): + package.validate_distribution(self.dist) + + def test_submission_rejects_missing_or_invalid_auth(self): + archive = package.build_zip(self.dist, self.root / "valid") + mutations = [ + lambda s: s.pop("extensions"), + lambda s: s["extensions"]["com.openai"]["auth"].update(clientSecret="fake-test-secret"), + lambda s: s["extensions"]["com.openai"]["auth"].update(type="none"), + lambda s: s["extensions"]["com.openai"]["auth"]["client"].update(mode="provided", clientId="test"), + lambda s: s.update(headers={"Authorization": "Bearer fake-test-secret"}), + lambda s: s.update(url="https://{data_plane_host}/mcp"), + ] + for index, mutation in enumerate(mutations): + with self.subTest(index=index): + bad = self.root / "bad.zip" + with zipfile.ZipFile(archive) as source, zipfile.ZipFile(bad, "w") as target: + for info in source.infolist(): + data = source.read(info.filename) + if info.filename == "mcp.json": + config = json.loads(data) + mutation(config["mcpServers"]["braintrust"]) + data = json.dumps(config).encode() + target.writestr(info, data) + with self.assertRaisesRegex(ValueError, "OAuth DCR"): + package.validate_zip(bad) + + def test_manifest_hook_app_and_template_fields_rejected(self): + path = self.public / "plugin.json" + original = path.read_text() + for field in ["hooks", "apps", "mcpServers", "template", "desktopRequired"]: + with self.subTest(field=field): + path.write_text(original) + self.edit(path, lambda d: d["extensions"]["com.openai"].update({field: "bad"})) + with self.assertRaisesRegex(ValueError, "fields"): + package.validate_mcp_package(self.public) + + def test_asset_reference_and_missing_asset_rejected(self): + self.edit(self.public / "plugin.json", lambda d: d["extensions"]["com.openai"]["interface"].update(logo="../trace-codex/icon.png")) + with self.assertRaisesRegex(ValueError, "logo path"): + package.validate_mcp_package(self.public) + (self.public / "assets/icon.png").unlink() + with self.assertRaisesRegex(ValueError, "allowlist"): + package.validate_mcp_package(self.public) + + def test_duplicate_json_keys_rejected(self): + path = self.public / "mcp.json" + path.write_text(path.read_text().replace('"type": "streamable-http"', '"type": "stdio", "type": "streamable-http"')) + with self.assertRaisesRegex(ValueError, "duplicate JSON"): + package.validate_mcp_package(self.public) + + def test_marketplace_escape_identity_and_implicit_install_rejected(self): + path = self.dist / ".agents/plugins/marketplace.json" + original = path.read_text() + for mutation in [ + lambda d: d.update(name="renamed"), + lambda d: d["plugins"][0]["source"].update(path="../outside"), + lambda d: d["plugins"][0]["policy"].update(installation="INSTALLED_BY_DEFAULT"), + lambda d: d["plugins"].append(d["plugins"][0]), + ]: + path.write_text(original) + self.edit(path, mutation) + with self.assertRaises(ValueError): + package.validate_distribution(self.dist) + + def test_tracing_hooks_remain_daemon_only(self): + path = self.dist / "plugins/trace-codex/hooks/hooks.json" + self.edit(path, lambda d: d["hooks"]["SessionStart"][0]["hooks"][0].update(command="other-runtime")) + with self.assertRaisesRegex(ValueError, "daemon forwarders"): + package.validate_distribution(self.dist) + + def test_mismatched_package_versions_rejected(self): + self.edit(self.public / "plugin.json", lambda d: d.update(version="999.0.0")) + with self.assertRaisesRegex(ValueError, "share a release version"): + package.validate_distribution(self.dist) + + def prepare_source(self): + target = self.root / "src/plugins/codex/content" + target.parent.mkdir(parents=True) + shutil.copytree(self.dist, target) + (self.root / "scripts").mkdir() + shutil.copyfile(REPO / "scripts/set-plugin-version.py", self.root / "scripts/set-plugin-version.py") + return target + + def test_stamping_and_preparation_paths_cover_both_packages(self): + source = self.prepare_source() + result = subprocess.run([sys.executable, "scripts/set-plugin-version.py", "codex", "v9.8.7"], cwd=self.root, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + package.validate_distribution(source) + output = self.root / "outputs" + env = dict(os.environ, REQUESTED_VERSION="v9.8.7", PLUGIN="codex", SOURCE_REF="refs/heads/main", GITHUB_OUTPUT=str(output)) + subprocess.run(["bash", "-euc", run_step("release.yml", "Validate release request")], env=env, cwd=self.root, check=True, capture_output=True) + patterns = output.read_text().split("paths<> "$ATTACH_LOG"\n') + gh.chmod(0o755) + env = dict(os.environ, PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], VERSION=package.validate_zip(archive)["version"], DIST_REPO="braintrustdata/braintrust-codex-plugin", ATTACH_LOG=str(self.root / "log")) + for _ in range(2): + subprocess.run(["bash", "-euc", script], cwd=archive.parent, env=env, check=True, capture_output=True) + self.assertEqual(len((self.root / "log").read_text().splitlines()), 2) + archive.write_bytes(b"tampered") + result = subprocess.run(["bash", "-euc", script], cwd=archive.parent, env=env, capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len((self.root / "log").read_text().splitlines()), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/plugins/codex/validate.sh b/src/plugins/codex/validate.sh index b9376cac..9a7f5a86 100755 --- a/src/plugins/codex/validate.sh +++ b/src/plugins/codex/validate.sh @@ -1,92 +1,4 @@ #!/usr/bin/env bash -# -# validate.sh — Sanity-check a built Codex dist tree in $1 before publishing. -# -# Fails (non-zero) on the mistakes that would ship a broken marketplace: -# - missing marketplace manifest / plugin manifests / daemon shims / hooks -# - malformed JSON in any manifest -# - marketplace entries whose `source.path` does not exist in the tree -# -# Usage: validate.sh - +# Validate the built marketplace, including each package's separate contract. set -euo pipefail - -TARGET_DIR="${1:?usage: validate.sh }" -fail() { echo "validate: $*" >&2; exit 1; } - -# jq if available (preferred), else python3 as a fallback JSON checker. -check_json() { - if command -v jq >/dev/null 2>&1; then - jq empty "$1" >/dev/null 2>&1 || fail "invalid JSON: $1" - else - python3 -c 'import json,sys; json.load(open(sys.argv[1]))' "$1" 2>/dev/null \ - || fail "invalid JSON: $1" - fi -} - -MARKETPLACE="$TARGET_DIR/.agents/plugins/marketplace.json" -[[ -f "$MARKETPLACE" ]] || fail "missing $MARKETPLACE" -check_json "$MARKETPLACE" - -# Required files for each shipped plugin. -required=( - "plugins/trace-codex/.codex-plugin/plugin.json" - "plugins/trace-codex/hooks/hooks.json" -) -for rel in "${required[@]}"; do - [[ -f "$TARGET_DIR/$rel" ]] || fail "missing $rel" - case "$rel" in *.json) check_json "$TARGET_DIR/$rel";; esac -done - -python3 - "$MARKETPLACE" "$TARGET_DIR/plugins" <<'PY' \ - || fail "Codex marketplace must contain only the tracing plugin" -import json -import sys -from pathlib import Path - -with open(sys.argv[1]) as f: - plugins = json.load(f)["plugins"] - -assert [plugin["name"] for plugin in plugins] == ["trace-codex"] -assert sorted(path.name for path in Path(sys.argv[2]).iterdir() if path.is_dir()) == ["trace-codex"] -PY - -if find "$TARGET_DIR" -name '.mcp.json' -print -quit | grep -q .; then - fail "Codex dist still contains an MCP proxy configuration" -fi - -python3 - "$TARGET_DIR/plugins/trace-codex/hooks/hooks.json" <<'PY' \ - || fail "Codex hooks do not all use the daemon forwarders" -import json -import sys - -with open(sys.argv[1]) as f: - hooks = json.load(f)["hooks"] - -expected_events = { - "Interrupt", "PermissionRequest", "PostCompact", "PostToolUse", "PreCompact", - "PreToolUse", "SessionEnd", "SessionStart", "Stop", "SubagentStart", "SubagentStop", - "UserPromptSubmit", -} -assert set(hooks) == expected_events -for definitions in hooks.values(): - for definition in definitions: - for hook in definition["hooks"]: - assert hook["type"] == "command" - assert hook["command"] == "bt trace hook --source codex" - assert hook["commandWindows"] == "bt trace hook --source codex" -PY -if find "$TARGET_DIR/plugins/trace-codex" -type f \( \ - -name 'package.json' -o -name 'pnpm-lock.yaml' -o -name 'tsconfig.json' -o \ - -name 'codex-hook-*' -o -name 'codex-hook.sh' \) -print -quit | grep -q .; then - fail "Codex tracing plugin still contains a legacy tracing runtime" -fi - -# Every marketplace entry's source path must exist in the built tree. -if command -v jq >/dev/null 2>&1; then - while IFS= read -r p; do - [[ -d "$TARGET_DIR/$p" ]] || fail "marketplace source path not found: $p" - done < <(jq -r '.plugins[].source.path' "$MARKETPLACE") -fi - -echo "validate: codex dist OK ($TARGET_DIR)" +python3 "$(dirname "${BASH_SOURCE[0]}")/package.py" validate "${1:?usage: validate.sh }"