From fca5c4f77e0469064f0fe466f41828fc3fe76062 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 27 Oct 2026 15:35:51 -0400 Subject: [PATCH 1/3] feat(codex): prepare braintrust for the official plugin directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > ๐Ÿค– **This text was AI-generated.** Add the package needed to publish Braintrust in the official ChatGPT and Codex plugin directory. It lets users query traces, manage prompts and datasets, and run evaluations from a conversation. Once approved and published, install Braintrust from the directory, sign in with your Braintrust account, and select it in a new conversation. Try "Show me my Braintrust projects" or "Summarize recent errors in my project." The plugin uses your account's permissions and needs no API key or local software. Installing it does not enable session tracing; bt trace enable codex continues to install only the existing tracing plugin. Keep both plugins in braintrustdata/braintrust-codex-plugin, built from this monorepo with the same release version. Extend the existing release approval process to create a ZIP containing only the new MCP plugin: ```text approve and merge release PR -> attach braintrust-.zip to braintrust-codex-plugin's release -> download ZIP and upload it to OpenAI for review -> publish in the official directory after approval ``` Attach a checksum alongside the ZIP and allow attachment retries without repeating deployment. CI and test releases provide downloadable workflow artifacts. Build a local ZIP with make codex-submission. OpenAI needs a working server address to test the plugin. The package provides Braintrust's North America (US) address, https://api.braintrust.dev/mcp. Organizations using Europe or their own Braintrust deployment need a different address. We plan one public listing where organization admins can configure that address, rather than separate regional plugins. OpenAI must still confirm Braintrust's access to this feature, the address formats it supports, and whether North America can be preselected. GitHub installs currently use the North America address; admins needing another address must configure their own package copy. OpenAI submission and publication remain manual. Before submission, complete the separate server review and sign-in/tool checks with authorized test accounts. This change prepares the package; it does not publish a directory listing. Docs: - [Braintrust MCP setup](https://www.braintrust.dev/docs/integrations/developer-tools/mcp) - [Connect and test in ChatGPT](https://developers.openai.com/plugins/deploy/connect-chatgpt) - [Submit and publish the plugin](https://developers.openai.com/plugins/deploy/submission) - [Configurable server addresses](https://developers.openai.com/plugins/deploy/app-review#template-mcp-server-urls) --- .github/workflows/_release.yml | 68 +++++ .github/workflows/ci.yml | 8 + .github/workflows/release.yml | 4 +- .gitignore | 1 + Makefile | 8 + README.md | 6 + scripts/set-plugin-version.py | 33 ++- src/plugins/codex/build.sh | 1 + .../content/.agents/plugins/marketplace.json | 12 + src/plugins/codex/content/AGENTS.md | 26 +- src/plugins/codex/content/README.md | 27 +- src/plugins/codex/content/install.sh | 17 +- .../codex/content/plugins/braintrust/LICENSE | 21 ++ .../content/plugins/braintrust/README.md | 79 ++++++ .../plugins/braintrust/assets/icon.png | Bin 0 -> 6298 bytes .../codex/content/plugins/braintrust/mcp.json | 19 ++ .../content/plugins/braintrust/plugin.json | 39 +++ src/plugins/codex/content/uninstall.sh | 6 +- src/plugins/codex/package.py | 215 ++++++++++++++ src/plugins/codex/tests/test_packaging.py | 262 ++++++++++++++++++ src/plugins/codex/validate.sh | 92 +----- 21 files changed, 818 insertions(+), 126 deletions(-) create mode 100644 src/plugins/codex/content/plugins/braintrust/LICENSE create mode 100644 src/plugins/codex/content/plugins/braintrust/README.md create mode 100644 src/plugins/codex/content/plugins/braintrust/assets/icon.png create mode 100644 src/plugins/codex/content/plugins/braintrust/mcp.json create mode 100644 src/plugins/codex/content/plugins/braintrust/plugin.json create mode 100644 src/plugins/codex/package.py create mode 100644 src/plugins/codex/tests/test_packaging.py diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index 558dcf9b..789eada7 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -52,6 +52,10 @@ jobs: release: runs-on: ubuntu-24.04 timeout-minutes: 30 + outputs: + version: ${{ steps.vars.outputs.version }} + dist_owner: ${{ steps.vars.outputs.dist_owner }} + dist_name: ${{ steps.vars.outputs.dist_name }} steps: - name: Checkout monorepo uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -165,6 +169,24 @@ jobs: fi fi + - name: Build and validate Codex submission from selected source + if: inputs.plugin == 'codex' + env: + RELEASE_VERSION: ${{ steps.vars.outputs.version }} + run: | + set -euo pipefail + make test-codex-packaging codex-submission + test -f "dist/submissions/braintrust-$RELEASE_VERSION.zip" + + - name: Save Codex submission ZIP + if: inputs.plugin == 'codex' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: braintrust-mcp-${{ inputs.source_sha }}-${{ steps.vars.outputs.version }} + path: dist/submissions/braintrust-${{ steps.vars.outputs.version }}.zip* + if-no-files-found: error + overwrite: true + - name: Record monorepo release id: source-release if: ${{ inputs.record }} @@ -226,6 +248,52 @@ jobs: --title "$DIST_TAG" \ --generate-notes + # Separate job: if asset upload fails after release creation, Re-run failed + # jobs retries attachment without repeating deployment/tag preflight. + attach-codex-submission: + needs: release + if: inputs.plugin == 'codex' && inputs.record + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Download validated submission from release build + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: braintrust-mcp-${{ inputs.source_sha }}-${{ needs.release.outputs.version }} + path: submission + + - name: Generate distribution-repo token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.BRAINTRUST_BOT_APP_ID }} + private-key: ${{ secrets.BRAINTRUST_BOT_PRIVATE_KEY }} + owner: ${{ needs.release.outputs.dist_owner }} + repositories: ${{ needs.release.outputs.dist_name }} + permission-contents: write + + - name: Attach MCP-only ZIP to distribution release + working-directory: submission + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + DIST_REPO: ${{ inputs.dist_repo }} + VERSION: ${{ needs.release.outputs.version }} + run: | + set -euo pipefail + python3 - "$VERSION" <<'PY' + import hashlib + from pathlib import Path + import sys + archive = Path(f"braintrust-{sys.argv[1]}.zip") + expected = f"{hashlib.sha256(archive.read_bytes()).hexdigest()} {archive.name}\n" + if archive.with_suffix(".zip.sha256").read_text() != expected: + sys.exit("Submission ZIP checksum mismatch") + PY + gh release upload "v$VERSION" "braintrust-$VERSION.zip" "braintrust-$VERSION.zip.sha256" \ + --repo "$DIST_REPO" --clobber + # Post-deploy end-to-end smoke test (codex only): install the just-deployed # plugin from the marketplace on Linux + both macOS arches and assert a real # session traces. It's a verification, not a gate โ€” the deploy already happened. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbf65f51..fe2a65c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,14 @@ jobs: fi - name: Build and validate plugins run: make test + - name: Test Codex packaging and release contracts + run: make test-codex-packaging codex-submission + - name: Save MCP submission ZIP + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: braintrust-mcp-submission + path: dist/submissions/braintrust-*.zip* + if-no-files-found: error - name: Package Cursor extension working-directory: dist/cursor run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12077288..8dae338c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: printf 'paths<.zip` to the distribution GitHub Release. OpenAI submission remains manual. + Human approval and merge into `main` authorize tagging the merge commit and publishing this version to the production destination. This preparation does not tag or deploy anything. - name: Summarize release pull request diff --git a/.gitignore b/.gitignore index 659c91a1..e35d0d29 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build output dist/ +__pycache__/ # Local credentials and environment overrides .env diff --git a/Makefile b/Makefile index 1a2c3ce8..74bfa2d6 100644 --- a/Makefile +++ b/Makefile @@ -60,3 +60,11 @@ $(PUBLISH_RULES): publish-%: clean: @rm -rf "$(DIST)" + +# MCP-only package for manual OpenAI submission. Never publish automatically. +.PHONY: codex-submission test-codex-packaging +codex-submission: validate-codex + python3 src/plugins/codex/package.py zip "$(DIST)/codex" "$(DIST)/submissions" + +test-codex-packaging: + python3 -m unittest discover -s src/plugins/codex/tests -p 'test_*.py' -v diff --git a/README.md b/README.md index 927de449..508faee7 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,10 @@ This monorepo contains the plugins and the shared Rust tracing daemon embedded in the Braintrust CLI (`bt`). Plugins forward native events locally; the daemon builds traces, journals events for recovery, and delivers spans to Braintrust. +The Codex marketplace also includes an independent [Braintrust MCP +package](src/plugins/codex/content/plugins/braintrust/README.md) for hosted tools +with OAuth. It requires no `bt` installation and does not enable session tracing. + ## Get started Install your coding agent and the @@ -82,6 +86,8 @@ JavaScript packages, and stable Rust for the daemon. Run from the monorepo root: ```bash make build # Build all plugins into dist/ make build-codex # Build one plugin +make codex-submission # Validate and build the MCP-only OpenAI ZIP +make test-codex-packaging # Test Codex packaging and release contracts make test # Build and validate plugin packages and forwarders cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked ``` diff --git a/scripts/set-plugin-version.py b/scripts/set-plugin-version.py index 77688ee3..22ffdfdf 100644 --- a/scripts/set-plugin-version.py +++ b/scripts/set-plugin-version.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 """Set a release version on every distributed version surface for an agent. -Versioning is per-plugin: every plugin under an agent carries its own -.-plugin/plugin.json with a `version` field. The marketplace manifest is -NOT touched. +Versioning is per distribution: portable root manifests and compatibility +manifests share one version. Marketplace manifests are NOT touched. Only version values are rewritten, so surrounding files keep their formatting. @@ -14,12 +13,15 @@ import re import sys -# agent -> glob of its per-plugin manifests (relative to repo root) +# agent -> globs of its per-plugin manifests (relative to repo root) MANIFEST_GLOBS = { - "claude": "src/plugins/claude/content/plugins/*/.claude-plugin/plugin.json", - "codex": "src/plugins/codex/content/plugins/*/.codex-plugin/plugin.json", - "grok": "src/plugins/grok/content/.grok-plugin/plugin.json", - "cursor": "src/plugins/cursor/content/.cursor-plugin/plugin.json", + "claude": ["src/plugins/claude/content/plugins/*/.claude-plugin/plugin.json"], + "codex": [ + "src/plugins/codex/content/plugins/*/.codex-plugin/plugin.json", + "src/plugins/codex/content/plugins/*/plugin.json", + ], + "grok": ["src/plugins/grok/content/.grok-plugin/plugin.json"], + "cursor": ["src/plugins/cursor/content/.cursor-plugin/plugin.json"], } VERSION_RE = re.compile(r'("version"\s*:\s*")[^"]*(")') @@ -27,13 +29,18 @@ def main() -> None: if len(sys.argv) != 3: sys.exit("usage: set-plugin-version.py ") agent, version = sys.argv[1], sys.argv[2].lstrip("v") - pattern = MANIFEST_GLOBS.get(agent) - if not pattern: + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + sys.exit("version must be MAJOR.MINOR.PATCH") + patterns = MANIFEST_GLOBS.get(agent) + if not patterns: sys.exit(f"unknown agent '{agent}' (known: {', '.join(MANIFEST_GLOBS)})") - manifests = sorted(glob.glob(pattern)) - if not manifests: - sys.exit(f"no plugin manifests found for '{agent}' ({pattern})") + manifests = [] + for pattern in patterns: + matches = sorted(glob.glob(pattern)) + if not matches: + sys.exit(f"no plugin manifests found for '{agent}' ({pattern})") + manifests.extend(matches) surfaces = [(path, VERSION_RE, "version") for path in manifests] if agent == "cursor": diff --git a/src/plugins/codex/build.sh b/src/plugins/codex/build.sh index 750b1214..80891a1c 100755 --- a/src/plugins/codex/build.sh +++ b/src/plugins/codex/build.sh @@ -4,6 +4,7 @@ # # The Codex marketplace consumes a repo whose ROOT is the marketplace: # .agents/plugins/marketplace.json marketplace manifest +# plugins/braintrust/ hosted MCP package (portable manifest) # plugins/trace-codex/ tracing plugin (thin daemon hook shims) # # The tracing plugin contains no tracing runtime or platform-specific binary; diff --git a/src/plugins/codex/content/.agents/plugins/marketplace.json b/src/plugins/codex/content/.agents/plugins/marketplace.json index 6ae6fecc..f7b47e9b 100644 --- a/src/plugins/codex/content/.agents/plugins/marketplace.json +++ b/src/plugins/codex/content/.agents/plugins/marketplace.json @@ -4,6 +4,18 @@ "displayName": "Braintrust Codex Plugin" }, "plugins": [ + { + "name": "braintrust", + "source": { + "source": "local", + "path": "./plugins/braintrust" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Developer Tools" + }, { "name": "trace-codex", "source": { diff --git a/src/plugins/codex/content/AGENTS.md b/src/plugins/codex/content/AGENTS.md index bd55923b..696280fb 100644 --- a/src/plugins/codex/content/AGENTS.md +++ b/src/plugins/codex/content/AGENTS.md @@ -4,26 +4,36 @@ Guidelines for AI agents working in this repo. ## Repo purpose -This repo distributes one [Codex marketplace plugin](https://developers.openai.com/codex/plugins): +This generated repo distributes two independently opt-in +[Codex marketplace plugins](https://developers.openai.com/plugins/build/plugins): + +- `plugins/braintrust/` โ€” hosted MCP with OAuth DCR, portable root `plugin.json` + and `mcp.json`. No hooks, apps, skills, credentials, or local runtime. - `plugins/trace-codex/` โ€” an opt-in plugin that traces Codex sessions to Braintrust (session, turn, and tool spans) via Codex lifecycle hooks. -The plugin is listed in `.agents/plugins/marketplace.json` (the repo marketplace). +Both plugins are listed in `.agents/plugins/marketplace.json`. Preserve the +marketplace name `braintrust-codex-plugins` and tracing identity +`trace-codex@braintrust-codex-plugins`. -Key files for the tracing plugin (see [`plugins/trace-codex/AGENTS.md`](plugins/trace-codex/AGENTS.md) for its architecture): +Key files for the tracing plugin: -- `plugins/trace-codex/AGENTS.md` โ€” architecture and contributor guide for this plugin +- `plugins/trace-codex/README.md` โ€” architecture and setup guide - `plugins/trace-codex/.codex-plugin/plugin.json` โ€” plugin manifest - `plugins/trace-codex/hooks/hooks.json` โ€” lifecycle hook config -- `plugins/trace-codex/src/` โ€” the hook client + event server (compiled to `bin/codex-hook`) +- Tracing hooks forward to `bt trace hook --source codex`; there is no bundled runtime. ## Making changes +- **MCP metadata**: edit `plugins/braintrust/plugin.json` and `mcp.json` in the + source monorepo. Keep portable identity and OpenAI presentation in one manifest. - **Plugin metadata**: edit `plugins/trace-codex/.codex-plugin/plugin.json`. - **Marketplace**: edit `.agents/plugins/marketplace.json` to change the plugin entry, category, or install policy. ## Releasing a new version -1. Bump `"version"` in the relevant plugin's `.codex-plugin/plugin.json`. -2. Commit, tag, and create a GitHub release (see README for exact commands). -3. Do **not** skip the git tag โ€” releases are tracked via tags so users can see a changelog. +Make all source changes in `braintrustdata/braintrust-coding-agent-plugins`. +Both packages share the Codex distribution version. Use the monorepo's reviewed +Release Plugin workflow; do not edit or tag this generated repository manually. +The distribution release carries an MCP-only ZIP for manual OpenAI submission. +Template settings belong to portal setup, not speculative manifest fields. diff --git a/src/plugins/codex/content/README.md b/src/plugins/codex/content/README.md index de130ae8..13cb953f 100644 --- a/src/plugins/codex/content/README.md +++ b/src/plugins/codex/content/README.md @@ -1,11 +1,31 @@ -# Braintrust tracing for Codex +# Braintrust plugins for Codex > **This repository is generated.** It is built from > [braintrustdata/braintrust-coding-agent-plugins](https://github.com/braintrustdata/braintrust-coding-agent-plugins). > Don't edit files here โ€” make changes and file issues in that repository, and they > will be rebuilt into this one. -Trace Codex sessions in Braintrust with the `trace-codex` plugin. +This marketplace contains two independently opt-in plugins: + +- **Braintrust**, `braintrust@braintrust-codex-plugins`: hosted MCP tools with + OAuth. No `bt` dependency or tracing hooks. See the + [MCP installation and endpoint guide](plugins/braintrust/README.md). +- **Trace Codex**, `trace-codex@braintrust-codex-plugins`: session tracing via `bt`. + +For MCP, add this marketplace and select Braintrust: + +```bash +codex plugin marketplace add braintrustdata/braintrust-codex-plugin +codex plugin add braintrust@braintrust-codex-plugins +``` + +The GitHub package connects to NA (US). EU and custom deployments require an +admin-configured local/team copy; GitHub installs do not inherit public portal +template settings. The planned public-directory listing uses one template, +pending OpenAI eligibility and endpoint-template confirmation. Its concrete NA +review URL is not a verified install-time default. + +The sections below describe optional tracing setup only. ## Quickstart @@ -49,5 +69,4 @@ bt trace disable codex ``` See the [tracing plugin guide](plugins/trace-codex/README.md) for hook behavior, -metadata, and tags. This marketplace does not configure the Braintrust MCP -server; use the agent's connector or MCP configuration for MCP access. +metadata, and tags. MCP sign-in and `bt` tracing sign-in are independent. diff --git a/src/plugins/codex/content/install.sh b/src/plugins/codex/content/install.sh index b082774c..f35f4d70 100755 --- a/src/plugins/codex/content/install.sh +++ b/src/plugins/codex/content/install.sh @@ -17,7 +17,8 @@ # correct config entries. # # Usage: -# ./install.sh # install all plugins in this repo +# ./install.sh # install tracing (legacy default) +# ./install.sh braintrust # install only the hosted MCP plugin # ./install.sh trace-codex # install just one plugin (by folder) # # Use ./uninstall.sh to remove. @@ -28,14 +29,11 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MARKETPLACE="braintrust-codex-plugins" -# Plugin folders to install. Default: every folder under plugins/. +# Keep the existing tracing-only default. MCP is selected explicitly. if [ "$#" -gt 0 ]; then PLUGINS=("$@") else - PLUGINS=() - for d in "$REPO_ROOT"/plugins/*/; do - PLUGINS+=("$(basename "$d")") - done + PLUGINS=("trace-codex") fi if ! command -v codex >/dev/null 2>&1; then @@ -80,6 +78,9 @@ build_plugin() { for folder in "${PLUGINS[@]}"; do plugin_src="$REPO_ROOT/plugins/$folder" manifest="$plugin_src/.codex-plugin/plugin.json" + if [ -f "$plugin_src/plugin.json" ]; then + manifest="$plugin_src/plugin.json" + fi if [ ! -f "$manifest" ]; then echo "Skipping '$folder': no manifest at $manifest" >&2 @@ -108,7 +109,7 @@ done echo "" echo "Done. Next steps:" echo " 1. Restart Codex (or start a new session) so it loads the plugins." -echo " 2. Plugin hooks are non-managed: run /hooks in the Codex CLI and trust them" -echo " before they will fire." +echo " 2. For Braintrust MCP, complete OAuth sign-in in the client." +echo " For tracing, run /hooks and trust the tracing hooks before they fire." echo "" echo "After editing plugin files, re-run ./install.sh to re-sync the cached copy." diff --git a/src/plugins/codex/content/plugins/braintrust/LICENSE b/src/plugins/codex/content/plugins/braintrust/LICENSE new file mode 100644 index 00000000..f8a1f4ae --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Braintrust Data, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/plugins/codex/content/plugins/braintrust/README.md b/src/plugins/codex/content/plugins/braintrust/README.md new file mode 100644 index 00000000..5f4e964a --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/README.md @@ -0,0 +1,79 @@ +# Braintrust + +Query traces, investigate failures, manage prompts and datasets, and run +evaluations using the hosted [Braintrust MCP server](https://www.braintrust.dev/docs/integrations/developer-tools/mcp). +Requires a Braintrust account with access to your organization's data plane. +Tools use that account's permissions and can read or change Braintrust data. + +This package contains a remote Streamable HTTP connection with OAuth Dynamic +Client Registration (DCR). The client discovers registration and authorization +endpoints. No API key, client secret, authorization header, local process, `bt` +installation, hooks, or desktop execution is required by this package. +MCP authorization is independent of optional `bt` session tracing. + +## GitHub marketplace + +After this version is released to the generated distribution repository: + +```bash +codex plugin marketplace add braintrustdata/braintrust-codex-plugin +codex plugin add braintrust@braintrust-codex-plugins +``` + +You can also select **Braintrust** in a supported client's marketplace browser. +Complete the client's OAuth flow, then start a new conversation. Installing +Braintrust does not install or enable the separate tracing plugin. + +The GitHub package's `mcp.json` has a concrete NA URL. It does not implement +public-directory template configuration or an endpoint picker. Current Codex +plugin policy settings document enablement and tool approval, not a server URL +override. To use EU or a custom endpoint today, have your admin copy this +package into a local/team marketplace and replace only +`mcpServers.braintrust.url` in that copy's `mcp.json` with the complete URL below. +The canonical release validator intentionally pins the NA review configuration; +an organization-specific copy needs its own endpoint validation. Avoid editing +the installed cache: marketplace upgrades can replace it. Do not enable both +the NA package and a duplicate connection for the same tools. + +| Data plane | MCP URL | +| --- | --- | +| NA (called US in Braintrust docs) | `https://api.braintrust.dev/mcp` | +| EU | `https://api-eu.braintrust.dev/mcp` | +| Custom / self-hosted | Copy the complete MCP URL from **Settings > Data plane**. | + +Braintrust documents the custom URL as the configured API base URL with `/mcp` +appended. Preserve any deployment-specific path prefix or other supplied URL +details; do not reduce every custom URL to `https:///mcp`. + +## Public-directory template listing (pending external setup) + +The intended public listing is one template-based Braintrust plugin for all +data planes. The package supplies a concrete NA endpoint for review and scans. +It does **not** establish NA as the installation default. Preselect/prefill NA +only if OpenAI confirms the installation UI supports it. + +OpenAI describes **Example MCP Server URL** and **Template MCP Server URL** as +submission settings. The candidate template `https://{data_plane_host}/mcp` +needs OpenAI confirmation: the published example varies a subdomain, not an +arbitrary full host. That candidate also cannot represent an API base with a +path prefix. Support for those custom URL shapes and parameter defaults must +be confirmed before configuring the listing. Do not place a template URL or +invent template/default fields in `mcp.json` or `plugin.json`. + +Template submissions are restricted to trusted developers with an established +OpenAI relationship. Portal access, eligibility, supported parameter shapes, +and authenticated installation evidence are still prerequisites. Portal +template settings are not automatically applied to GitHub marketplace installs. +This package has not been approved or submitted. + +## Support and package assets + +See [Braintrust troubleshooting](https://www.braintrust.dev/docs/kb/troubleshooting-guides) +or contact [support@braintrust.dev](mailto:support@braintrust.dev). +[Privacy policy](https://www.braintrust.dev/legal/privacy-policy) ยท +[Terms of service](https://www.braintrust.dev/legal/terms-of-service). + +`assets/icon.png` is the existing 200ร—200 Braintrust icon copied verbatim from +this monorepo's `src/plugins/cursor/content/icon.png`; the brand color and legal +URLs also match existing Braintrust plugin metadata. No synthetic screenshots +or invented contact details are included. diff --git a/src/plugins/codex/content/plugins/braintrust/assets/icon.png b/src/plugins/codex/content/plugins/braintrust/assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..fb64baccb89896550c0b6d1fce3ef41a6ff1e13a GIT binary patch literal 6298 zcmai(Wl&tv)8;Sk1PdM*+#Q03;2sDNoWU(XfWch`0t^JV!8O4p!CgX-Fjxp0ENE~D z?(gK?-G6QEZq!uB)A=kEOdU;N#=N`_{$D!`jl#meBOdcS)`?5 z4MNyg6qtON{Cf9Xr`Vx!K&xzt(^n`Vla#db+x8X z{RpT_54HE&Vw=6VGPh6~?HYD$0^x0UY@Yj?8aNeS+o1L6Sn!FNdFXn%``4dc>bNr()hq7@n;_Zdg`jKdt|Bm>ZBlGLJ;2s4QP zv*5KC(BZ1Tr}*_WnP;Pq2aqHeZD63brSZM?W zKzM4d51plrVDoZaP!JYd%CX+QCdfbo7C~)nv>Ji^hae?daN>;28a`{+(jl!rXwAdi zRWo=tIq2ZkGR6Rv{{Dy^b=V<6RI>8Ivl61g2iUa4%SrTH2|##0_nVvqX$b$q!Q+a% z4-p|VFH6yiA=P&m{Eo07UmxM$zpQ=IKhxgg^hpFmPxJ_ZiD-!Z?S9u6O)Yu>pFM8I z8`yN=zEaO`VOW=?PkbP)QKB2jk0eam#fMO&%qU;QoYP)CW6*;%)QB9*9lL95=&Me= zWYEzNl*kRu@v5MM{gYEX?m>>(ch&=s+ieo-)*=PX)nU8qO7AGj5eSA6$y^h9gwVlQ z;ar2T;E4Ks=!=_e6vx-8gB-jrx%qBC^QYsvqSZaj9epAlL0jq?2`mnXz{2lRgui9e zdFv$mdua`b?7YwXAb8J~==Sk4sai_JAyT2OF{fgvPHsO#LX@ADG=2=B3;nu3rp+VK zFr`7jETIP>8w71xkGvQug%;z=?d(Im?aNqE>75*De^X8y<%CmziQ9nl5kAu+`jpk8 zHYXK`t(HMU{qmNGD`G5mw^`scqGolPqu%ynJ=z$S6$8~@;@wp~ zqm+^?0<8KBu^x$s@XIjdDg`(GOh8Hufv=Ng#d0tk&`{K!;Rtyn8{tOH5lw_y0+ORl zgkoFD;4(&_{QnIj2DB1stD5M^)zrV;X2gL$vc#QvvGflst-%pS=+eBjY7ae&4s@&o zE_N}khUVRF>QfS2v-F>Bb|)|9gHr0_&WIU}F;~d|UC!mVL6jf*10yrM`HWMucB$6F z95JY|yaC@x$|v~G_x&4wP@QvessyO#zs#QLl8G4a>mPl|7)7Q~O@`L2;nwO3u75jF z=>Mqo$~TFC>y%gLKu2K5;*Y!Z!au&hdcokv6>ckdeV6L;Bov%1md8u4o}v}V-0 zz)V{PG6&Sj&fhY-Pd;j18|%oXqc`!WmV#?%dVdyJ(K4=i2;~F|1kMporO3f~Rm4)I zbeLv6fUVRt%?;f%U}EWL0>jjCmhrpI!wk6!LR4t(jqCy{?7H6a@Ex6I#4cTNy}z2 ztOz5sLhC^q7}UTJA3HOY(oBUyl*AIRiYAG^Lb-8R6Ri5St6#A!4|Mb{@(YS+riQx+ zOE~cTJ&}8fUS-r~xWO1_CMX*JSSX?XEL+2WOY(glw_)P(-N7JmCoPhR4*bcweXF}( zkWJ0@y}#wSY@*4cs}wf`YIDw|w@D1Z&Es9NEO+PD&JCwV@cYTwD(nyO+iQj{6_%mg zxu-MCjP})MLd!&xopytQ;W#bkwsf)-q?^37+07t%f|``Dw(Pf^F)ijRQ2}ZTP&XRj zT2q0FklSUs$#=il4^K`aw@CW+Go~8|QUr9rvrf9KJO_NZ)R=*mp|tv+;0J3Wx%R)p z=T7T%`}c3fX-GsDu|aPkbSKYUXl_*s!#xMN!$ihxV1Hr`4_9A{)A$UO9)Je)^rJ|M z|MI{L%0P;>+Qc83CYmMMJID3DB#gDVK7}9Fai)l2Z+0EwJymeYY^1hSWInvFdD`xH z=2vUCioBY99&oqP9MZV70&1dx77a%w(^5B`RyP)rH) zyydl?CP6iJf-qa4ZFhV zd@kMsRGY9}avm3gi=eVp9`AMD z3YME7;h3b)Pss1!vsB$W*`-|q>uAdFy|c0`nWgyVQRzCxbjo@W-D8)&y-@DMllv!V zu4S=rj>;HfEp@s4l1%hz^kYr}g%2MCkcvdUA*-Zs9S8<<_G++Y(?q9`DVN(L*aQE5 zJ=RdOUu?#Wn4YmGDgJ})GzilnwpJ8AyfrD@j?HuY`J+Y)wQN_a-fM$DWqph5m3DU< zIBT1_7)259$E0L^1`dV1=5596Gf>du!jd=_JuK!RTyrGm6@%06NQFc5kt$;#wR>9a zbj}9$C%LI80vXI7;DXMain~M&k^ps#BIhD5a&Q5;xis`?voRWC{eQ(d6DU}|b zfaSn|Jv!k>9gP?p_oURwxglFOBMF_~N&aE6Zxy#5+C-?p$9P@;bA zkMU|(HwydJXi3tH#a~N+zDY2pO!4eB^VWUU5(gmO%^Sh+|HOt=o`QOd-hlxaQxt;X z$0woe9E`*bNg@=Yxmj|R&R+;a@Ol;2eD2P0{R%>KNA8=^K+O}K8m29m+)Z!MoWa#8 zY)`3BV1u**-iyqy!wx+~isj{b40L7s1=Q)57DAMK4hmWX{tYj^y+*69F7+X@|ERur z=O%+PkWhvo4YNW>hsf)K*jkT#p1wM6oHyXp#236D3prx=)JxDwJaR=pq)_kR2g^ix zFYO5&nS69HoynJ}4%?WXU`OSMa+8jep=j&bLt-|b`f^KZZ{cfaIVbjO5O%YS9VJDh zEp_;jD;}$>iP9E@s0cW*?xnabu2wMZhl<)&&Imoq7Dq6sRggBaeBD+T!>EmnE-N|; z$T8NI5SVfMj55EkIidx|4UD*84zae24ARA*TF7f5h`2P!Q|~~t_W8{4kL43#f^S+t zi#ctDa|J~hZwZ()h^I=chnbhdChXly<{?F7{h|%;bs|rhXvj-=>YOmKn~k&8XnQQ` zQ^I{X2z0tl5rIDJWZADBLM>$tX|7p;?&&$#YL*5n5Zze%iooW;s8My7hB&A-$a`fYaQvy64BJ5)$zO1knhDbjGmm`y@H>2) z1OGHTNbz1#pw@=@X1WZhOX1xof8}o5NEbUBWP(RKrhkJ32-DU3yk<}=+Ax!4uw$@x z8@}g##z*G@;+0RF9{?a{PU2-!y^3R)7LkALUoo#Z110~vY_<8K;`SXD)NDyR;!Thp z&vxwb9meFf;kn=LjI_63m;P9!#3TiZ6RQHLUtKU_F6u9R2`M?}6W!*k?8#R`h%xFb zLL^)`;Xh7NYv3lBXN1A~#ebVx4V0C^=8rj3A0Tzt-felYvZ-aKXu3xj8WvG;U#{$lgH zFn&3+)9qxQGIztsFd{UkUD8$uV;Y;-M_+Ph`fy%=%ZFxu|LCW%L?(**CD*blZV}ni zU;w1g$;9rR-sj8s(Pe^p#E=eiI+oSC?J&76!0 zoRp1`_)u-eUz3^q%mp@Z&sA|=gjjW0>@J$zl+FpG0ZJ;)7uLaJ4KEcY`oc;Ascfk) zfip5veV+JheUrv7+AMvR!V`J@M))~e1ycZ`Nm4gXMZv1e$JB)p14X8%(xOUONPpHL z0*)0eu4#06^DnZBQ#~m5Px3!_b}+)I8WE?6a*GT2d^(N+;Gj)Asp^hf>cNzc;igI@CvizdB;n1W}zU8muY{@cZHjq{Hd7icP{of4_0p zav~ZRI#t8}@wKA1Q?U5oX>j^VL9~9y0S`nCZMG+@x)cp8-^liKA_zV4(2~C>t@RwO zx#^^?=D9x=OSuU?>C z0IpF3yGYv5Ie#b>`6kIDpZ$v)ISd#Vp$<|P_uV+JN7XKVQ>~dGyAwq!YaoT3(=I+? z$Rg&n=ZUMLzj1S5THK2p4seSxKfs{JVCd%EG)Czd@Nn!B!R&^ zlB=X!>>{ZHW7z$I>fCadD&J(+sQN-?We4;hveA(I1qH7osP6FMw<07mXkxiFtx0*sd5zr$ z^BE>$Hs2Bg?WrszT?b+P0SJ7J)1e)J`ssGLeJ6Z31hhhmf%nC3>H`?C0ZYB7Os(HA zkVcnK^h$423e-g+NuU;$nd1-m*dlsjAX^b`@#sd2_czsD3Wl|~>m$yy220r)=NF#z zq#>AoHQK~!OoG?%=`F+7*v}met&Jb>nRFdwCS#u#FLkkH9?Ls<-~bjquI^=8ZxbaH zd%T_r)c$t&{#3ICra28Fh`_3bu@ENLZ_FAZ69($WjKdyr@C?=I(-#BJ^@!~L$RC0; zP<>n0EY>U(`AnUib95%Z#W6nlBv~Y|LV>^+33~ZYb&iQ4D}q3C`ZEpefF1lrB3MAJ za68;lRaE1}(2$Zqv0O)VEc2rD&AYJvK-L?^&;)sxHI8Wu<}INd2g}K#pKr#Awikz( zViqtbI*nFKYX>`A)7n#F@Z5W#7&J6&A2eQu_mqbJ@x|mr=U+Umo=?1#A}R8SqvA_e$bFlJUPvi~q}@ z$rWAx3pAKiH1bYsKDVZwr4?uYf3!jHSu(gK4JXq&9HZQFxl%Jv(%pPdIe%0o3xV_Pdf}5O|l>H6O32JuYrCcOHgqqjD>I77)JHy6G7+? z?MvbnHWMOckJAVl@76rIB@K{frE)@hPOmlw3V?Z!^r!b%Ww41=_i|PF`Pd?AWVzt9 z1nT1$avcv-*+{7xy|SpZYIq2E4b!I*XN`GLi0jbgoli`-cnt^)7Q*UjK{o0*iH{u{ zN*jzGsDmD`D5E)IHn7oelAd}P84W`)P(n4lH|zj$uK0KwggNi|(LS=kWwq1wP@FD% z)&912U7F>YQR2U%Q)uekd;2x?*h1W+A62GdFob@}_XT<3LmeOh_h_ycW=IEb9#x>RwQjdAB&?7#(o$3ImqEN63R?n#2 z+rM~7&dezGp0$Uyc^>>UXtQHOwp{y9kblG*>WT_W!+NxS<$e@iEKUhl6Fv-3`L9lj?5M(#~MwZY}>ajXh-Dd@R)e3IN(eyoU z%5!mIa=r3qXKuz~mpil@V|8gd_<#{pwng1zXJv10F11clgTwI)2#Y)KQNQO~jaKLm zBfxXP@;jJc|1I^()a24xa*()EBi4=P?64PA&vtLW@&0!r#;VhxjXqb#?E9dm&9a_H Zl=lZGURvS$rpS8*Kvhvwp+?RE`d?jW_p1N^ literal 0 HcmV?d00001 diff --git a/src/plugins/codex/content/plugins/braintrust/mcp.json b/src/plugins/codex/content/plugins/braintrust/mcp.json new file mode 100644 index 00000000..0c119693 --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/mcp.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", + "mcpServers": { + "braintrust": { + "type": "streamable-http", + "url": "https://api.braintrust.dev/mcp", + "extensions": { + "com.openai": { + "auth": { + "type": "oauth", + "client": { + "mode": "dcr" + } + } + } + } + } + } +} diff --git a/src/plugins/codex/content/plugins/braintrust/plugin.json b/src/plugins/codex/content/plugins/braintrust/plugin.json new file mode 100644 index 00000000..4adc1eef --- /dev/null +++ b/src/plugins/codex/content/plugins/braintrust/plugin.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "braintrust", + "version": "2.1.1", + "description": "Query Braintrust traces, manage prompts and datasets, and run evaluations through the hosted MCP server.", + "author": { + "name": "Braintrust", + "email": "support@braintrust.dev", + "url": "https://www.braintrust.dev/" + }, + "homepage": "https://www.braintrust.dev/docs/integrations/developer-tools/mcp", + "repository": "https://github.com/braintrustdata/braintrust-coding-agent-plugins", + "license": "MIT", + "keywords": ["braintrust", "evaluations", "observability", "mcp"], + "extensions": { + "com.openai": { + "interface": { + "displayName": "Braintrust", + "shortDescription": "Explore traces and run evals", + "longDescription": "Connect to Braintrust to query production traces, investigate failures, manage prompts and datasets, and run evaluations. For developers building and evaluating AI applications. Requires a Braintrust account and access to the selected organization's data plane. Tools act with your connected account's permissions and can read or modify Braintrust data. This package uses the NA (US) endpoint; public-directory template endpoint selection requires separate OpenAI setup. MCP sign-in is independent of optional bt session tracing.", + "developerName": "Braintrust", + "category": "Developer Tools", + "capabilities": ["Read", "Write"], + "websiteURL": "https://www.braintrust.dev/", + "supportURL": "https://www.braintrust.dev/docs/kb/troubleshooting-guides", + "privacyPolicyURL": "https://www.braintrust.dev/legal/privacy-policy", + "termsOfServiceURL": "https://www.braintrust.dev/legal/terms-of-service", + "defaultPrompt": [ + "Show me my Braintrust projects.", + "Summarize recent errors in my Braintrust project.", + "Compare the results of two experiments in my Braintrust project." + ], + "brandColor": "#2C1FEB", + "composerIcon": "./assets/icon.png", + "logo": "./assets/icon.png" + } + } + } +} diff --git a/src/plugins/codex/content/uninstall.sh b/src/plugins/codex/content/uninstall.sh index 49ebf799..09e726b0 100755 --- a/src/plugins/codex/content/uninstall.sh +++ b/src/plugins/codex/content/uninstall.sh @@ -9,8 +9,7 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MARKETPLACE="braintrust-codex-plugins" -CODEX_HOME="${CODEX_HOME:-$HOME/.codex}" -CACHE_DIR="$CODEX_HOME/plugins/cache/$MARKETPLACE" +CACHE_DIR="${CODEX_HOME:-$HOME/.codex}/plugins/cache/$MARKETPLACE" if ! command -v codex >/dev/null 2>&1; then echo "Error: 'codex' CLI not found." >&2 @@ -21,6 +20,9 @@ echo "Uninstalling Braintrust Codex plugins..." for d in "$REPO_ROOT"/plugins/*/; do manifest="$d/.codex-plugin/plugin.json" + if [ -f "$d/plugin.json" ]; then + manifest="$d/plugin.json" + fi [ -f "$manifest" ] || continue # Installable name is the manifest `name`, which may differ from the folder. name="$(grep -o '"name"[[:space:]]*:[[:space:]]*"[^"]*"' "$manifest" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')" diff --git a/src/plugins/codex/package.py b/src/plugins/codex/package.py new file mode 100644 index 00000000..62895220 --- /dev/null +++ b/src/plugins/codex/package.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Offline distribution and MCP-only submission validation/ZIP creation. + +This is a narrow package contract, not a general OpenAI schema validator. +Template settings are portal configuration, not speculative JSON fields. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import struct +import tempfile +from urllib.parse import urlsplit +import zipfile + + +SCHEMA_BASE = "https://agent-plugins.org/schemas/1.0.0/" +PUBLIC_FILES = frozenset({"plugin.json", "mcp.json", "README.md", "LICENSE", "assets/icon.png"}) +EVENTS = { + "Interrupt", "PermissionRequest", "PostCompact", "PostToolUse", "PreCompact", + "PreToolUse", "SessionEnd", "SessionStart", "Stop", "SubagentStart", "SubagentStop", + "UserPromptSubmit", +} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def read_json(path): + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, f"duplicate JSON key: {key}") + result[key] = value + return result + return json.loads(path.read_text(), object_pairs_hook=unique) + + +def keys(value, allowed): + require(isinstance(value, dict), "expected an object") + require(set(value) == set(allowed), f"unexpected/missing fields: {set(value) ^ set(allowed)}") + + +def version(value): + require(isinstance(value, str) and re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", value), "expected MAJOR.MINOR.PATCH version") + + +def https_url(value): + require(isinstance(value, str), "URL must be a string") + url = urlsplit(value) + require(url.scheme == "https" and url.hostname and not url.username and not url.password + and not url.query and not url.fragment and not re.search(r"[{}\s]", value), + f"expected concrete credential-free HTTPS URL: {value}") + + +def inventory(root): + require(root.is_dir() and not root.is_symlink(), f"missing or symlinked package: {root}") + files = set() + for path in root.rglob("*"): + require(not path.is_symlink(), f"symlink not allowed: {path}") + require(path.is_dir() or path.is_file(), f"special file not allowed: {path}") + if path.is_file(): + files.add(path.relative_to(root).as_posix()) + return files + + +def validate_public(root): + require(inventory(root) == PUBLIC_FILES, "public package files must match MCP-only allowlist (no hooks, apps, skills, scripts, or marketplace)") + require({p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_dir()} == {"assets"}, "unexpected public package directory") + manifest = read_json(root / "plugin.json") + keys(manifest, {"$schema", "name", "version", "description", "author", "homepage", "repository", "license", "keywords", "extensions"}) + require(manifest["$schema"] == SCHEMA_BASE + "plugin.schema.json", "wrong portable plugin schema") + require(manifest["name"] == "braintrust", "MCP identity must be braintrust") + version(manifest["version"]) + require(manifest["license"] == "MIT", "expected MIT package license") + keys(manifest["author"], {"name", "email", "url"}) + require(manifest["author"]["name"] == "Braintrust" and manifest["author"]["email"] == "support@braintrust.dev", "unexpected publisher") + for value in [manifest["homepage"], manifest["repository"], manifest["author"]["url"]]: + https_url(value) + require(isinstance(manifest["description"], str) and manifest["description"].strip(), "missing description") + require(isinstance(manifest["keywords"], list) and all(isinstance(k, str) and k.strip() for k in manifest["keywords"]), "invalid keywords") + keys(manifest["extensions"], {"com.openai"}) + overlay = manifest["extensions"]["com.openai"] + keys(overlay, {"interface"}) # No hooks/apps/mcpServers/desktop requirements. + interface = overlay["interface"] + keys(interface, {"displayName", "shortDescription", "longDescription", "developerName", "category", "capabilities", "websiteURL", "supportURL", "privacyPolicyURL", "termsOfServiceURL", "defaultPrompt", "brandColor", "composerIcon", "logo"}) + require(interface["displayName"] == interface["developerName"] == "Braintrust", "unexpected display name/publisher") + require(interface["category"] == "Developer Tools", "unexpected listing category") + for field, limit in [("displayName", 30), ("shortDescription", 30), ("longDescription", 4000), ("developerName", 80)]: + value = interface[field] + require(isinstance(value, str) and value.strip() and len(value) <= limit and not re.search(r"[\x00-\x1f]", value), f"invalid {field}") + for field in ["websiteURL", "supportURL", "privacyPolicyURL", "termsOfServiceURL"]: + https_url(interface[field]) + require(len(interface[field]) <= 1024, f"{field} too long") + require(interface["capabilities"] == ["Read", "Write"], "unexpected capabilities") + prompts = interface["defaultPrompt"] + require(isinstance(prompts, list) and 1 <= len(prompts) <= 3 and all(isinstance(p, str) and p.strip() and len(p) <= 128 and "@" not in p for p in prompts), "invalid starter prompts") + require(len(set(prompts)) == len(prompts), "duplicate starter prompts") + require(interface["brandColor"] == "#2C1FEB", "unexpected brand color") + for field in ["composerIcon", "logo"]: + require(interface[field] == "./assets/icon.png", f"unexpected {field} path") + data = (root / "assets/icon.png").read_bytes() + require(24 <= len(data) <= 5 * 1024 * 1024 and data[:8] == b"\x89PNG\r\n\x1a\n" and data[12:16] == b"IHDR", "invalid PNG asset") + width, height = struct.unpack(">II", data[16:24]) + require(width == height and 48 <= width <= 4096, "listing icon must be square, 48..4096px") + config = read_json(root / "mcp.json") + # Exact contract forbids headers, tokens, processes, env interpolation, and + # speculative template/default fields at every nesting level. + require(config == { + "$schema": SCHEMA_BASE + "mcp.schema.json", + "mcpServers": {"braintrust": { + "type": "streamable-http", "url": "https://api.braintrust.dev/mcp", + "extensions": {"com.openai": {"auth": {"type": "oauth", "client": {"mode": "dcr"}}}}, + }}, + }, "MCP must use the concrete NA review URL, Streamable HTTP, and discovery-based OAuth DCR; templates belong in portal setup") + return manifest + + +def validate_distribution(root): + inventory(root) + marketplace = read_json(root / ".agents/plugins/marketplace.json") + keys(marketplace, {"name", "interface", "plugins"}) + require(marketplace["name"] == "braintrust-codex-plugins", "marketplace identity changed") + require(marketplace["interface"] == {"displayName": "Braintrust Codex Plugin"}, "marketplace display identity changed") + entries = marketplace["plugins"] + require(isinstance(entries, list) and len(entries) == 2 and {p["name"] for p in entries} == {"braintrust", "trace-codex"}, "expected exactly MCP and tracing entries") + require({p.name for p in (root / "plugins").iterdir()} == {"braintrust", "trace-codex"}, "unexpected plugin paths") + for entry in entries: + keys(entry, {"name", "source", "policy", "category"}) + name = entry["name"] + require(entry["source"] == {"source": "local", "path": f"./plugins/{name}"}, "unexpected marketplace source/path") + require(entry["policy"] == {"installation": "AVAILABLE", "authentication": "ON_INSTALL"}, "plugins must be separately opt-in") + require(entry["category"] == ("Developer Tools" if name == "braintrust" else "Observability"), "wrong marketplace category") + public = validate_public(root / "plugins/braintrust") + tracing = root / "plugins/trace-codex" + manifest = read_json(tracing / ".codex-plugin/plugin.json") + require(manifest["name"] == "trace-codex" and manifest["hooks"] == "./hooks/hooks.json", "tracing identity/hooks changed") + version(manifest["version"]) + require(manifest["version"] == public["version"], "Codex packages must share a release version") + require(not ({"apps", "mcpServers", "skills"} & set(manifest)), "unexpected tracing components") + require("plugin.json" not in inventory(tracing), "conflicting tracing manifest") + require(not any(p.name in {"mcp.json", ".mcp.json", ".app.json", "package.json", "pnpm-lock.yaml", "tsconfig.json"} + or p.name.startswith("codex-hook") for p in tracing.rglob("*")), "legacy tracing runtime or MCP/app configuration") + hooks = read_json(tracing / "hooks/hooks.json") + keys(hooks, {"hooks"}) + require(set(hooks["hooks"]) == EVENTS, "unexpected tracing events") + for definitions in hooks["hooks"].values(): + require(isinstance(definitions, list) and definitions, "empty tracing hook definitions") + for definition in definitions: + require(isinstance(definition["hooks"], list) and definition["hooks"], "empty tracing hooks") + for hook in definition["hooks"]: + require(hook["type"] == "command" and hook["command"] == hook["commandWindows"] == "bt trace hook --source codex", "hooks must use daemon forwarders") + for path in root.rglob("*"): + if path.name in {"mcp.json", ".mcp.json", ".app.json"}: + require(path == root / "plugins/braintrust/mcp.json", "unexpected distribution MCP/app configuration") + return public + + +def validate_zip(path): + with zipfile.ZipFile(path) as archive, tempfile.TemporaryDirectory() as temp: + infos = archive.infolist() + require(len(infos) == len(PUBLIC_FILES) and {i.filename for i in infos} == PUBLIC_FILES, + "ZIP must contain only the MCP package with plugin.json at archive root") + for info in infos: + require(stat.S_IFMT(info.external_attr >> 16) == stat.S_IFREG, "ZIP entry must be a regular file") + require(info.file_size <= 5 * 1024 * 1024, "oversized ZIP entry") + archive.extractall(temp) # Paths were checked against exact allowlist. + return validate_public(Path(temp)) + + +def build_zip(root, output): + manifest = validate_distribution(root) + package = root / "plugins/braintrust" + output.mkdir(parents=True, exist_ok=True) + path = output / f"braintrust-{manifest['version']}.zip" + # Fixed order, timestamps, permissions, no compression-version dependency. + with tempfile.TemporaryDirectory(dir=output) as temp: + staged = Path(temp) / path.name + with zipfile.ZipFile(staged, "w", compression=zipfile.ZIP_STORED) as archive: + for name in sorted(PUBLIC_FILES): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + archive.writestr(info, (package / name).read_bytes()) + validate_zip(staged) + os.replace(staged, path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + path.with_suffix(".zip.sha256").write_text(f"{digest} {path.name}\n") + return path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["validate", "validate-public", "validate-zip", "zip"]) + parser.add_argument("path", type=Path) + parser.add_argument("output", type=Path, nargs="?") + args = parser.parse_args() + try: + if args.command == "zip": + require(args.output is not None, "zip requires output directory") + print(build_zip(args.path, args.output)) + else: + {"validate": validate_distribution, "validate-public": validate_public, "validate-zip": validate_zip}[args.command](args.path) + print(f"validate: {args.command} OK ({args.path})") + except (ValueError, KeyError, TypeError, OSError, zipfile.BadZipFile) as error: + parser.exit(1, f"validate: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/src/plugins/codex/tests/test_packaging.py b/src/plugins/codex/tests/test_packaging.py new file mode 100644 index 00000000..e6ce703f --- /dev/null +++ b/src/plugins/codex/tests/test_packaging.py @@ -0,0 +1,262 @@ +"""Failure cases for package isolation and the reviewed-release boundary.""" +import glob +import importlib.util +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest +import zipfile + + +REPO = Path(__file__).resolve().parents[4] +CODEX = REPO / "src/plugins/codex" +spec = importlib.util.spec_from_file_location("codex_package", CODEX / "package.py") +package = importlib.util.module_from_spec(spec) +spec.loader.exec_module(package) + + +def run_step(workflow, name): + """Extract the actual workflow shell body so tests exercise release code.""" + text = (REPO / ".github/workflows" / workflow).read_text() + tail = text.split(f"- name: {name}\n", 1)[1].split(" run: |\n", 1)[1] + lines = [] + for line in tail.splitlines(): + if line and not line.startswith(" "): + break + lines.append(line[10:]) + return "\n".join(lines) + + +class PackagingTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.dist = self.root / "dist" + shutil.copytree(CODEX / "content", self.dist) + self.public = self.dist / "plugins/braintrust" + + def edit(self, path, fn): + data = json.loads(path.read_text()) + fn(data) + path.write_text(json.dumps(data)) + + def test_valid_distribution_and_reproducible_zip(self): + first = package.build_zip(self.dist, self.root / "one") + # A timestamp change must not change the archive bytes. + for path in self.public.rglob("*"): + os.utime(path, (1700000000, 1700000000)) + second = package.build_zip(self.dist, self.root / "two") + self.assertEqual(first.read_bytes(), second.read_bytes()) + manifest = package.validate_zip(first) + self.assertEqual(first.name, f"braintrust-{manifest['version']}.zip") + with zipfile.ZipFile(first) as archive: + self.assertEqual(set(archive.namelist()), package.PUBLIC_FILES) + self.assertEqual(json.loads(archive.read("plugin.json"))["name"], "braintrust") + + def test_accidental_public_components_rejected_before_packaging(self): + for name in ["hooks/hooks.json", ".app.json", ".codex-plugin/plugin.json", "skills/workflow/SKILL.md", "trace-codex/README.md", ".agents/plugins/marketplace.json", "credentials.json"]: + with self.subTest(name=name): + target = self.public / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("{}") + with self.assertRaisesRegex(ValueError, "allowlist"): + package.build_zip(self.dist, self.root / "output") + target.unlink() + parent = target.parent + while parent != self.public: + parent.rmdir() + parent = parent.parent + + def test_symlink_rejected(self): + icon = self.public / "assets/icon.png" + icon.unlink() + icon.symlink_to(CODEX / "content/plugins/braintrust/assets/icon.png") + with self.assertRaisesRegex(ValueError, "symlink"): + package.validate_public(self.public) + + def test_zip_injection_rejected(self): + original = package.build_zip(self.dist, self.root / "valid") + for name in ["hooks/hooks.json", "../plugin.json", "braintrust/plugin.json", "plugin.json"]: + with self.subTest(name=name): + bad = self.root / "bad.zip" + shutil.copyfile(original, bad) + with zipfile.ZipFile(bad, "a") as archive: + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + archive.writestr(name, "{}") + with self.assertRaisesRegex(ValueError, "archive root"): + package.validate_zip(bad) + + def test_whole_marketplace_zip_rejected(self): + bad = self.root / "marketplace.zip" + with zipfile.ZipFile(bad, "w") as archive: + for path in self.dist.rglob("*"): + if path.is_file(): + archive.write(path, path.relative_to(self.dist)) + with self.assertRaisesRegex(ValueError, "archive root"): + package.validate_zip(bad) + + def test_zip_symlink_rejected(self): + original = package.build_zip(self.dist, self.root / "valid") + bad = self.root / "symlink.zip" + with zipfile.ZipFile(original) as source, zipfile.ZipFile(bad, "w") as target: + for info in source.infolist(): + if info.filename == "assets/icon.png": + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + target.writestr(info, source.read(info.filename)) + with self.assertRaisesRegex(ValueError, "regular file"): + package.validate_zip(bad) + + def test_mcp_auth_transport_and_template_misconfiguration_rejected(self): + path = self.public / "mcp.json" + original = path.read_text() + mutations = [ + lambda s: s.update(url="https://{data_plane_host}/mcp"), + lambda s: s.update(url="https://api-eu.braintrust.dev/mcp"), + lambda s: s.update(type="sse"), + lambda s: s.update(type="stdio", command="bt"), + lambda s: s.update(headers={"Authorization": "Bearer fake-test-secret"}), + lambda s: s.update(default="api.braintrust.dev"), + lambda s: s["extensions"]["com.openai"]["auth"].update(clientSecret="fake-test-secret"), + lambda s: s["extensions"]["com.openai"]["auth"].update(type="none"), + lambda s: s["extensions"]["com.openai"]["auth"]["client"].update(mode="provided", clientId="test"), + ] + for index, mutation in enumerate(mutations): + with self.subTest(index=index): + path.write_text(original) + self.edit(path, lambda d: mutation(d["mcpServers"]["braintrust"])) + with self.assertRaisesRegex(ValueError, "OAuth DCR"): + package.validate_public(self.public) + + def test_manifest_hook_app_and_template_fields_rejected(self): + path = self.public / "plugin.json" + original = path.read_text() + for field in ["hooks", "apps", "mcpServers", "template", "desktopRequired"]: + with self.subTest(field=field): + path.write_text(original) + self.edit(path, lambda d: d["extensions"]["com.openai"].update({field: "bad"})) + with self.assertRaisesRegex(ValueError, "fields"): + package.validate_public(self.public) + + def test_asset_reference_and_missing_asset_rejected(self): + self.edit(self.public / "plugin.json", lambda d: d["extensions"]["com.openai"]["interface"].update(logo="../trace-codex/icon.png")) + with self.assertRaisesRegex(ValueError, "logo path"): + package.validate_public(self.public) + (self.public / "assets/icon.png").unlink() + with self.assertRaisesRegex(ValueError, "allowlist"): + package.validate_public(self.public) + + def test_duplicate_json_keys_rejected(self): + path = self.public / "mcp.json" + path.write_text(path.read_text().replace('"type": "streamable-http"', '"type": "stdio", "type": "streamable-http"')) + with self.assertRaisesRegex(ValueError, "duplicate JSON"): + package.validate_public(self.public) + + def test_marketplace_escape_identity_and_implicit_install_rejected(self): + path = self.dist / ".agents/plugins/marketplace.json" + original = path.read_text() + for mutation in [ + lambda d: d.update(name="renamed"), + lambda d: d["plugins"][0]["source"].update(path="../outside"), + lambda d: d["plugins"][0]["policy"].update(installation="INSTALLED_BY_DEFAULT"), + lambda d: d["plugins"].append(d["plugins"][0]), + ]: + path.write_text(original) + self.edit(path, mutation) + with self.assertRaises(ValueError): + package.validate_distribution(self.dist) + + def test_tracing_hooks_remain_daemon_only(self): + path = self.dist / "plugins/trace-codex/hooks/hooks.json" + self.edit(path, lambda d: d["hooks"]["SessionStart"][0]["hooks"][0].update(command="other-runtime")) + with self.assertRaisesRegex(ValueError, "daemon forwarders"): + package.validate_distribution(self.dist) + + def test_mismatched_package_versions_rejected(self): + self.edit(self.public / "plugin.json", lambda d: d.update(version="999.0.0")) + with self.assertRaisesRegex(ValueError, "share a release version"): + package.validate_distribution(self.dist) + + def prepare_source(self): + target = self.root / "src/plugins/codex/content" + target.parent.mkdir(parents=True) + shutil.copytree(self.dist, target) + (self.root / "scripts").mkdir() + shutil.copyfile(REPO / "scripts/set-plugin-version.py", self.root / "scripts/set-plugin-version.py") + return target + + def test_stamping_and_preparation_paths_cover_both_packages(self): + source = self.prepare_source() + result = subprocess.run([sys.executable, "scripts/set-plugin-version.py", "codex", "v9.8.7"], cwd=self.root, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + package.validate_distribution(source) + output = self.root / "outputs" + env = dict(os.environ, REQUESTED_VERSION="v9.8.7", PLUGIN="codex", SOURCE_REF="refs/heads/main", GITHUB_OUTPUT=str(output)) + subprocess.run(["bash", "-euc", run_step("release.yml", "Validate release request")], env=env, cwd=self.root, check=True, capture_output=True) + patterns = output.read_text().split("paths<> "$ATTACH_LOG"\n') + gh.chmod(0o755) + env = dict(os.environ, PATH=str(bin_dir) + os.pathsep + os.environ["PATH"], VERSION=package.validate_zip(archive)["version"], DIST_REPO="braintrustdata/braintrust-codex-plugin", ATTACH_LOG=str(self.root / "log")) + for _ in range(2): + subprocess.run(["bash", "-euc", script], cwd=archive.parent, env=env, check=True, capture_output=True) + self.assertEqual(len((self.root / "log").read_text().splitlines()), 2) + archive.write_bytes(b"tampered") + result = subprocess.run(["bash", "-euc", script], cwd=archive.parent, env=env, capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len((self.root / "log").read_text().splitlines()), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/plugins/codex/validate.sh b/src/plugins/codex/validate.sh index b9376cac..9a7f5a86 100755 --- a/src/plugins/codex/validate.sh +++ b/src/plugins/codex/validate.sh @@ -1,92 +1,4 @@ #!/usr/bin/env bash -# -# validate.sh โ€” Sanity-check a built Codex dist tree in $1 before publishing. -# -# Fails (non-zero) on the mistakes that would ship a broken marketplace: -# - missing marketplace manifest / plugin manifests / daemon shims / hooks -# - malformed JSON in any manifest -# - marketplace entries whose `source.path` does not exist in the tree -# -# Usage: validate.sh - +# Validate the built marketplace, including each package's separate contract. set -euo pipefail - -TARGET_DIR="${1:?usage: validate.sh }" -fail() { echo "validate: $*" >&2; exit 1; } - -# jq if available (preferred), else python3 as a fallback JSON checker. -check_json() { - if command -v jq >/dev/null 2>&1; then - jq empty "$1" >/dev/null 2>&1 || fail "invalid JSON: $1" - else - python3 -c 'import json,sys; json.load(open(sys.argv[1]))' "$1" 2>/dev/null \ - || fail "invalid JSON: $1" - fi -} - -MARKETPLACE="$TARGET_DIR/.agents/plugins/marketplace.json" -[[ -f "$MARKETPLACE" ]] || fail "missing $MARKETPLACE" -check_json "$MARKETPLACE" - -# Required files for each shipped plugin. -required=( - "plugins/trace-codex/.codex-plugin/plugin.json" - "plugins/trace-codex/hooks/hooks.json" -) -for rel in "${required[@]}"; do - [[ -f "$TARGET_DIR/$rel" ]] || fail "missing $rel" - case "$rel" in *.json) check_json "$TARGET_DIR/$rel";; esac -done - -python3 - "$MARKETPLACE" "$TARGET_DIR/plugins" <<'PY' \ - || fail "Codex marketplace must contain only the tracing plugin" -import json -import sys -from pathlib import Path - -with open(sys.argv[1]) as f: - plugins = json.load(f)["plugins"] - -assert [plugin["name"] for plugin in plugins] == ["trace-codex"] -assert sorted(path.name for path in Path(sys.argv[2]).iterdir() if path.is_dir()) == ["trace-codex"] -PY - -if find "$TARGET_DIR" -name '.mcp.json' -print -quit | grep -q .; then - fail "Codex dist still contains an MCP proxy configuration" -fi - -python3 - "$TARGET_DIR/plugins/trace-codex/hooks/hooks.json" <<'PY' \ - || fail "Codex hooks do not all use the daemon forwarders" -import json -import sys - -with open(sys.argv[1]) as f: - hooks = json.load(f)["hooks"] - -expected_events = { - "Interrupt", "PermissionRequest", "PostCompact", "PostToolUse", "PreCompact", - "PreToolUse", "SessionEnd", "SessionStart", "Stop", "SubagentStart", "SubagentStop", - "UserPromptSubmit", -} -assert set(hooks) == expected_events -for definitions in hooks.values(): - for definition in definitions: - for hook in definition["hooks"]: - assert hook["type"] == "command" - assert hook["command"] == "bt trace hook --source codex" - assert hook["commandWindows"] == "bt trace hook --source codex" -PY -if find "$TARGET_DIR/plugins/trace-codex" -type f \( \ - -name 'package.json' -o -name 'pnpm-lock.yaml' -o -name 'tsconfig.json' -o \ - -name 'codex-hook-*' -o -name 'codex-hook.sh' \) -print -quit | grep -q .; then - fail "Codex tracing plugin still contains a legacy tracing runtime" -fi - -# Every marketplace entry's source path must exist in the built tree. -if command -v jq >/dev/null 2>&1; then - while IFS= read -r p; do - [[ -d "$TARGET_DIR/$p" ]] || fail "marketplace source path not found: $p" - done < <(jq -r '.plugins[].source.path' "$MARKETPLACE") -fi - -echo "validate: codex dist OK ($TARGET_DIR)" +python3 "$(dirname "${BASH_SOURCE[0]}")/package.py" validate "${1:?usage: validate.sh }" From db0a3531e247641cc11160bf01351bbe3b9d49af Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 27 Oct 2026 15:55:51 -0400 Subject: [PATCH 2/3] cleanup --- README.md | 8 +++++--- src/plugins/codex/content/plugins/braintrust/mcp.json | 4 +--- src/plugins/codex/content/plugins/braintrust/plugin.json | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 508faee7..d6918d37 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,11 @@ This monorepo contains the plugins and the shared Rust tracing daemon embedded in the Braintrust CLI (`bt`). Plugins forward native events locally; the daemon builds traces, journals events for recovery, and delivers spans to Braintrust. -The Codex marketplace also includes an independent [Braintrust MCP -package](src/plugins/codex/content/plugins/braintrust/README.md) for hosted tools -with OAuth. It requires no `bt` installation and does not enable session tracing. +This repository also packages the [Braintrust +plugin](src/plugins/codex/content/plugins/braintrust/README.md) for submission to +the official ChatGPT and Codex plugin directory. It connects to Braintrust's +hosted MCP server through OAuth and requires no `bt` installation. Installing it +does not enable session tracing. ## Get started diff --git a/src/plugins/codex/content/plugins/braintrust/mcp.json b/src/plugins/codex/content/plugins/braintrust/mcp.json index 0c119693..295749ce 100644 --- a/src/plugins/codex/content/plugins/braintrust/mcp.json +++ b/src/plugins/codex/content/plugins/braintrust/mcp.json @@ -8,9 +8,7 @@ "com.openai": { "auth": { "type": "oauth", - "client": { - "mode": "dcr" - } + "client": { "mode": "dcr" } } } } diff --git a/src/plugins/codex/content/plugins/braintrust/plugin.json b/src/plugins/codex/content/plugins/braintrust/plugin.json index 4adc1eef..a6f1a69d 100644 --- a/src/plugins/codex/content/plugins/braintrust/plugin.json +++ b/src/plugins/codex/content/plugins/braintrust/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "braintrust", "version": "2.1.1", - "description": "Query Braintrust traces, manage prompts and datasets, and run evaluations through the hosted MCP server.", + "description": "Explore Braintrust traces, manage prompts and datasets, and run evaluations.", "author": { "name": "Braintrust", "email": "support@braintrust.dev", @@ -17,7 +17,7 @@ "interface": { "displayName": "Braintrust", "shortDescription": "Explore traces and run evals", - "longDescription": "Connect to Braintrust to query production traces, investigate failures, manage prompts and datasets, and run evaluations. For developers building and evaluating AI applications. Requires a Braintrust account and access to the selected organization's data plane. Tools act with your connected account's permissions and can read or modify Braintrust data. This package uses the NA (US) endpoint; public-directory template endpoint selection requires separate OpenAI setup. MCP sign-in is independent of optional bt session tracing.", + "longDescription": "Query production traces, investigate failures, manage prompts and datasets, and run evaluations in Braintrust. Sign in with a Braintrust account to access your organization's data. Tools use your account's permissions and can read or modify data. This package connects to Braintrust's US data plane.", "developerName": "Braintrust", "category": "Developer Tools", "capabilities": ["Read", "Write"], From 0617661508f6efbe45cd609f278a5c9a4af96881 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 27 Oct 2026 16:09:40 -0400 Subject: [PATCH 3/3] fix(codex): separate submission auth from the marketplace package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > ๐Ÿค– **This text was AI-generated.** Codex installs the Braintrust plugin but drops its MCP server when mcp.json contains server-level auth extensions. Keep the GitHub package compatible with Codex and add OpenAI's documented OAuth declaration only when building the submission ZIP: ```text GitHub package -> server URL -> Codex discovers OAuth submission ZIP -> server URL + OAuth DCR declaration -> OpenAI portal ``` Validate the two formats separately. The ZIP builder leaves the source package untouched and continues to produce a reproducible MCP-only archive. Existing tracing setup and the reviewed release flow stay the same. Add an installed-plugin check that reads the MCP server through Codex's loader, since plugin add/list alone can succeed with a broken server entry. Run it in CI using the existing resolved Codex installation. Authentication and tool calls still require authorized integration checks. Refs: https://github.com/openai/codex/issues/53658 Refs: https://linear.app/braintrustdata/issue/SDK-699/add-official-braintrust-chatgpt-marketplace-plugin --- .github/workflows/ci.yml | 5 + Makefile | 7 +- README.md | 1 + src/plugins/codex/content/AGENTS.md | 3 + .../content/plugins/braintrust/README.md | 7 ++ .../codex/content/plugins/braintrust/mcp.json | 10 +- src/plugins/codex/package.py | 55 ++++++--- src/plugins/codex/tests/mcp-loading.mjs | 114 ++++++++++++++++++ src/plugins/codex/tests/test_packaging.py | 60 +++++++-- 9 files changed, 228 insertions(+), 34 deletions(-) create mode 100644 src/plugins/codex/tests/mcp-loading.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe2a65c5..9c3c80f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -357,6 +357,11 @@ jobs: npm exec --prefix "${{ runner.temp }}/coding-agents" -- claude --version npm exec --prefix "${{ runner.temp }}/coding-agents" -- opencode --version npm exec --prefix "${{ runner.temp }}/coding-agents" -- pi --version + - name: Check installed Codex MCP loading + if: runner.os == 'Linux' + env: + CODEX_BIN: ${{ runner.temp }}/coding-agents/node_modules/.bin/codex + run: make test-codex-mcp-loading - name: Check formatting if: runner.os == 'Linux' run: cargo fmt --manifest-path bt-daemon/Cargo.toml -- --check diff --git a/Makefile b/Makefile index 74bfa2d6..59240d20 100644 --- a/Makefile +++ b/Makefile @@ -62,9 +62,14 @@ clean: @rm -rf "$(DIST)" # MCP-only package for manual OpenAI submission. Never publish automatically. -.PHONY: codex-submission test-codex-packaging +.PHONY: codex-submission test-codex-packaging test-codex-mcp-loading codex-submission: validate-codex python3 src/plugins/codex/package.py zip "$(DIST)/codex" "$(DIST)/submissions" test-codex-packaging: python3 -m unittest discover -s src/plugins/codex/tests -p 'test_*.py' -v + +# Requires Codex on PATH (or the integration harness's CODEX_BIN override). +# Loads the installed MCP entry without OAuth, inference, or tool calls. +test-codex-mcp-loading: validate-codex + node src/plugins/codex/tests/mcp-loading.mjs "$(DIST)/codex" diff --git a/README.md b/README.md index d6918d37..94bbe5ba 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ make build # Build all plugins into dist/ make build-codex # Build one plugin make codex-submission # Validate and build the MCP-only OpenAI ZIP make test-codex-packaging # Test Codex packaging and release contracts +make test-codex-mcp-loading # Check installed MCP loading with a local Codex CLI make test # Build and validate plugin packages and forwarders cargo test --manifest-path bt-daemon/Cargo.toml --all-features --locked ``` diff --git a/src/plugins/codex/content/AGENTS.md b/src/plugins/codex/content/AGENTS.md index 696280fb..22cf47e7 100644 --- a/src/plugins/codex/content/AGENTS.md +++ b/src/plugins/codex/content/AGENTS.md @@ -27,6 +27,9 @@ Key files for the tracing plugin: - **MCP metadata**: edit `plugins/braintrust/plugin.json` and `mcp.json` in the source monorepo. Keep portable identity and OpenAI presentation in one manifest. + Keep server-level auth extensions out of the Codex `mcp.json`: current Codex + rejects them. The monorepo ZIP builder adds OAuth DCR metadata only to the + OpenAI submission archive and validates both formats separately. - **Plugin metadata**: edit `plugins/trace-codex/.codex-plugin/plugin.json`. - **Marketplace**: edit `.agents/plugins/marketplace.json` to change the plugin entry, category, or install policy. diff --git a/src/plugins/codex/content/plugins/braintrust/README.md b/src/plugins/codex/content/plugins/braintrust/README.md index 5f4e964a..1c36eb23 100644 --- a/src/plugins/codex/content/plugins/braintrust/README.md +++ b/src/plugins/codex/content/plugins/braintrust/README.md @@ -11,6 +11,13 @@ endpoints. No API key, client secret, authorization header, local process, `bt` installation, hooks, or desktop execution is required by this package. MCP authorization is independent of optional `bt` session tracing. +Codex discovers OAuth from the server. Its package declares only the HTTP +transport and server URL because current Codex versions reject explicit OAuth +metadata in `mcp.json`. The OpenAI submission ZIP adds the documented OAuth DCR +declaration during packaging; it does not change the GitHub package. See the +[Codex compatibility issue](https://github.com/openai/codex/issues/53658) and +[OpenAI authentication fields](https://developers.openai.com/plugins/build/plugins#configure-mcp-authentication). + ## GitHub marketplace After this version is released to the generated distribution repository: diff --git a/src/plugins/codex/content/plugins/braintrust/mcp.json b/src/plugins/codex/content/plugins/braintrust/mcp.json index 295749ce..d5b9bcbd 100644 --- a/src/plugins/codex/content/plugins/braintrust/mcp.json +++ b/src/plugins/codex/content/plugins/braintrust/mcp.json @@ -3,15 +3,7 @@ "mcpServers": { "braintrust": { "type": "streamable-http", - "url": "https://api.braintrust.dev/mcp", - "extensions": { - "com.openai": { - "auth": { - "type": "oauth", - "client": { "mode": "dcr" } - } - } - } + "url": "https://api.braintrust.dev/mcp" } } } diff --git a/src/plugins/codex/package.py b/src/plugins/codex/package.py index 62895220..cd3d8276 100644 --- a/src/plugins/codex/package.py +++ b/src/plugins/codex/package.py @@ -69,7 +69,38 @@ def inventory(root): return files -def validate_public(root): +def submission_mcp_config(config): + """Add portal auth metadata without changing the Codex-compatible source. + + Codex currently rejects server-level extensions, even though OpenAI's + submission format documents them. Keep this projection in the ZIP builder. + """ + # Round trip to leave the caller's validated distribution config untouched. + config = json.loads(json.dumps(config)) + config["mcpServers"]["braintrust"]["extensions"] = { + "com.openai": {"auth": {"type": "oauth", "client": {"mode": "dcr"}}}, + } + return config + + +def validate_mcp_config(config, *, submission): + # Exact contracts reject headers, credentials, processes, speculative + # template fields, and submission-only metadata in the Codex distribution. + expected = { + "$schema": SCHEMA_BASE + "mcp.schema.json", + "mcpServers": {"braintrust": { + "type": "streamable-http", "url": "https://api.braintrust.dev/mcp", + }}, + } + if submission: + expected = submission_mcp_config(expected) + message = "Submission MCP must use the concrete NA URL, Streamable HTTP, and discovery-based OAuth DCR" + else: + message = "Codex MCP must use the concrete NA URL and Streamable HTTP without server extensions or credentials" + require(config == expected, message) + + +def validate_mcp_package(root, *, submission=False): require(inventory(root) == PUBLIC_FILES, "public package files must match MCP-only allowlist (no hooks, apps, skills, scripts, or marketplace)") require({p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_dir()} == {"assets"}, "unexpected public package directory") manifest = read_json(root / "plugin.json") @@ -108,16 +139,7 @@ def validate_public(root): require(24 <= len(data) <= 5 * 1024 * 1024 and data[:8] == b"\x89PNG\r\n\x1a\n" and data[12:16] == b"IHDR", "invalid PNG asset") width, height = struct.unpack(">II", data[16:24]) require(width == height and 48 <= width <= 4096, "listing icon must be square, 48..4096px") - config = read_json(root / "mcp.json") - # Exact contract forbids headers, tokens, processes, env interpolation, and - # speculative template/default fields at every nesting level. - require(config == { - "$schema": SCHEMA_BASE + "mcp.schema.json", - "mcpServers": {"braintrust": { - "type": "streamable-http", "url": "https://api.braintrust.dev/mcp", - "extensions": {"com.openai": {"auth": {"type": "oauth", "client": {"mode": "dcr"}}}}, - }}, - }, "MCP must use the concrete NA review URL, Streamable HTTP, and discovery-based OAuth DCR; templates belong in portal setup") + validate_mcp_config(read_json(root / "mcp.json"), submission=submission) return manifest @@ -136,7 +158,7 @@ def validate_distribution(root): require(entry["source"] == {"source": "local", "path": f"./plugins/{name}"}, "unexpected marketplace source/path") require(entry["policy"] == {"installation": "AVAILABLE", "authentication": "ON_INSTALL"}, "plugins must be separately opt-in") require(entry["category"] == ("Developer Tools" if name == "braintrust" else "Observability"), "wrong marketplace category") - public = validate_public(root / "plugins/braintrust") + public = validate_mcp_package(root / "plugins/braintrust") tracing = root / "plugins/trace-codex" manifest = read_json(tracing / ".codex-plugin/plugin.json") require(manifest["name"] == "trace-codex" and manifest["hooks"] == "./hooks/hooks.json", "tracing identity/hooks changed") @@ -170,7 +192,7 @@ def validate_zip(path): require(stat.S_IFMT(info.external_attr >> 16) == stat.S_IFREG, "ZIP entry must be a regular file") require(info.file_size <= 5 * 1024 * 1024, "oversized ZIP entry") archive.extractall(temp) # Paths were checked against exact allowlist. - return validate_public(Path(temp)) + return validate_mcp_package(Path(temp), submission=True) def build_zip(root, output): @@ -186,7 +208,10 @@ def build_zip(root, output): info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) info.create_system = 3 info.external_attr = (stat.S_IFREG | 0o644) << 16 - archive.writestr(info, (package / name).read_bytes()) + data = (package / name).read_bytes() + if name == "mcp.json": + data = (json.dumps(submission_mcp_config(read_json(package / name)), indent=2) + "\n").encode() + archive.writestr(info, data) validate_zip(staged) os.replace(staged, path) digest = hashlib.sha256(path.read_bytes()).hexdigest() @@ -205,7 +230,7 @@ def main(): require(args.output is not None, "zip requires output directory") print(build_zip(args.path, args.output)) else: - {"validate": validate_distribution, "validate-public": validate_public, "validate-zip": validate_zip}[args.command](args.path) + {"validate": validate_distribution, "validate-public": lambda path: validate_mcp_package(path, submission=True), "validate-zip": validate_zip}[args.command](args.path) print(f"validate: {args.command} OK ({args.path})") except (ValueError, KeyError, TypeError, OSError, zipfile.BadZipFile) as error: parser.exit(1, f"validate: {error}\n") diff --git a/src/plugins/codex/tests/mcp-loading.mjs b/src/plugins/codex/tests/mcp-loading.mjs new file mode 100644 index 00000000..0f7c5f86 --- /dev/null +++ b/src/plugins/codex/tests/mcp-loading.mjs @@ -0,0 +1,114 @@ +// Exercise Codex's MCP parser, which plugin add/list alone does not validate. +// This reads installed plugin metadata; it never signs in or calls MCP tools. +import assert from "node:assert/strict" +import { cp, mkdtemp, mkdir, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { createInterface } from "node:readline" +import test from "node:test" +import { + execFileBackground, + spawnBackground, +} from "../../../runtime/js-daemon-client/src/background-process.ts" + +const distribution = resolve(process.argv[2] ?? "dist/codex") +const codex = process.env.CODEX_BIN || "codex" + +async function readPlugin(env, cwd, marketplace) { + const child = spawnBackground(codex, ["app-server", "--stdio"], { + env, + cwd, + stdio: ["pipe", "pipe", "pipe"], + }) + const pending = new Map() + let stderr = "" + child.stderr.setEncoding("utf8").on("data", (data) => { stderr += data }) + const lines = createInterface({ input: child.stdout }) + const fail = (error) => { + for (const request of pending.values()) request.reject(error) + pending.clear() + } + child.on("error", fail) + const exited = new Promise((done) => child.once("close", () => { + fail(new Error(`Codex app-server exited: ${stderr}`)) + done() + })) + lines.on("line", (line) => { + try { + const message = JSON.parse(line) + const request = pending.get(message.id) + if (request) { + pending.delete(message.id) + if (message.error) request.reject(new Error(JSON.stringify(message.error))) + else request.resolve(message.result) + } + } catch (error) { fail(error) } + }) + let id = 0 + async function call(method, params) { + const requestId = ++id + let timer + try { + return await new Promise((resolve, reject) => { + timer = setTimeout(() => reject(new Error(`Timeout: ${method}\n${stderr}`)), 10_000) + pending.set(requestId, { resolve, reject }) + child.stdin.write(JSON.stringify({ id: requestId, method, params }) + "\n") + }) + } finally { + clearTimeout(timer) + pending.delete(requestId) + } + } + try { + await call("initialize", { + clientInfo: { name: "braintrust_mcp_loading_test", version: "1.0.0" }, + capabilities: { experimentalApi: true }, + }) + child.stdin.write(JSON.stringify({ method: "initialized", params: {} }) + "\n") + const result = await call("plugin/read", { + marketplacePath: join(marketplace, ".agents/plugins/marketplace.json"), + pluginName: "braintrust", + }) + return result.plugin + } finally { + const killTimer = setTimeout(() => child.kill("SIGKILL"), 1000) + killTimer.unref() + child.kill() + await exited + clearTimeout(killTimer) + lines.close() + } +} + +test("installed Braintrust MCP server loads in Codex", { timeout: 40_000 }, async () => { + const root = await mkdtemp(join(tmpdir(), "braintrust-mcp-loading-")) + try { + const marketplace = join(root, "marketplace") + const home = join(root, "codex") + await mkdir(home) + await cp(distribution, marketplace, { recursive: true }) + const env = { ...process.env, CODEX_HOME: home } + // Separate profile and cwd prevent loading user credentials or repo hooks. + const options = { env, cwd: root, encoding: "utf8", timeout: 10_000 } + await execFileBackground(codex, ["plugin", "marketplace", "add", marketplace], options) + const install = await execFileBackground(codex, [ + "plugin", "add", "braintrust@braintrust-codex-plugins", "--json", + ], options) + const cached = JSON.parse(install.stdout).installedPath + const config = JSON.parse(await readFile(join(cached, "mcp.json"), "utf8")) + assert.equal(config.mcpServers.braintrust.type, "streamable-http") + const plugin = await readPlugin(env, root, marketplace) + assert.equal(plugin.summary.installed, true) + assert.deepEqual(plugin.mcpServers, ["braintrust"], "MCP entry was dropped by Codex's parser") + assert.deepEqual(plugin.hooks, []) + assert.deepEqual(plugin.apps, []) + const listed = await execFileBackground(codex, [ + "plugin", "list", "--marketplace", "braintrust-codex-plugins", "--json", + ], options) + assert.deepEqual(JSON.parse(listed.stdout).installed.map((p) => p.pluginId), [ + "braintrust@braintrust-codex-plugins", + ]) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/src/plugins/codex/tests/test_packaging.py b/src/plugins/codex/tests/test_packaging.py index e6ce703f..f7783efa 100644 --- a/src/plugins/codex/tests/test_packaging.py +++ b/src/plugins/codex/tests/test_packaging.py @@ -78,7 +78,7 @@ def test_symlink_rejected(self): icon.unlink() icon.symlink_to(CODEX / "content/plugins/braintrust/assets/icon.png") with self.assertRaisesRegex(ValueError, "symlink"): - package.validate_public(self.public) + package.validate_mcp_package(self.public) def test_zip_injection_rejected(self): original = package.build_zip(self.dist, self.root / "valid") @@ -114,7 +114,7 @@ def test_zip_symlink_rejected(self): with self.assertRaisesRegex(ValueError, "regular file"): package.validate_zip(bad) - def test_mcp_auth_transport_and_template_misconfiguration_rejected(self): + def test_codex_transport_and_template_misconfiguration_rejected(self): path = self.public / "mcp.json" original = path.read_text() mutations = [ @@ -124,16 +124,58 @@ def test_mcp_auth_transport_and_template_misconfiguration_rejected(self): lambda s: s.update(type="stdio", command="bt"), lambda s: s.update(headers={"Authorization": "Bearer fake-test-secret"}), lambda s: s.update(default="api.braintrust.dev"), + ] + for index, mutation in enumerate(mutations): + with self.subTest(index=index): + path.write_text(original) + self.edit(path, lambda d: mutation(d["mcpServers"]["braintrust"])) + with self.assertRaisesRegex(ValueError, "Codex MCP"): + package.validate_mcp_package(self.public) + + def test_zip_adds_submission_auth_without_mutating_distribution(self): + path = self.public / "mcp.json" + original = path.read_bytes() + archive = package.build_zip(self.dist, self.root / "output") + with zipfile.ZipFile(archive) as zipped: + config = json.loads(zipped.read("mcp.json")) + self.assertEqual(config["mcpServers"]["braintrust"], { + "type": "streamable-http", + "url": "https://api.braintrust.dev/mcp", + "extensions": {"com.openai": {"auth": {"type": "oauth", "client": {"mode": "dcr"}}}}, + }) + self.assertEqual(path.read_bytes(), original) + self.assertNotIn("extensions", json.loads(original)["mcpServers"]["braintrust"]) + package.validate_distribution(self.dist) + + def test_codex_rejects_submission_auth_metadata(self): + path = self.public / "mcp.json" + path.write_text(json.dumps(package.submission_mcp_config(json.loads(path.read_text())))) + with self.assertRaisesRegex(ValueError, "without server extensions"): + package.validate_distribution(self.dist) + + def test_submission_rejects_missing_or_invalid_auth(self): + archive = package.build_zip(self.dist, self.root / "valid") + mutations = [ + lambda s: s.pop("extensions"), lambda s: s["extensions"]["com.openai"]["auth"].update(clientSecret="fake-test-secret"), lambda s: s["extensions"]["com.openai"]["auth"].update(type="none"), lambda s: s["extensions"]["com.openai"]["auth"]["client"].update(mode="provided", clientId="test"), + lambda s: s.update(headers={"Authorization": "Bearer fake-test-secret"}), + lambda s: s.update(url="https://{data_plane_host}/mcp"), ] for index, mutation in enumerate(mutations): with self.subTest(index=index): - path.write_text(original) - self.edit(path, lambda d: mutation(d["mcpServers"]["braintrust"])) + bad = self.root / "bad.zip" + with zipfile.ZipFile(archive) as source, zipfile.ZipFile(bad, "w") as target: + for info in source.infolist(): + data = source.read(info.filename) + if info.filename == "mcp.json": + config = json.loads(data) + mutation(config["mcpServers"]["braintrust"]) + data = json.dumps(config).encode() + target.writestr(info, data) with self.assertRaisesRegex(ValueError, "OAuth DCR"): - package.validate_public(self.public) + package.validate_zip(bad) def test_manifest_hook_app_and_template_fields_rejected(self): path = self.public / "plugin.json" @@ -143,21 +185,21 @@ def test_manifest_hook_app_and_template_fields_rejected(self): path.write_text(original) self.edit(path, lambda d: d["extensions"]["com.openai"].update({field: "bad"})) with self.assertRaisesRegex(ValueError, "fields"): - package.validate_public(self.public) + package.validate_mcp_package(self.public) def test_asset_reference_and_missing_asset_rejected(self): self.edit(self.public / "plugin.json", lambda d: d["extensions"]["com.openai"]["interface"].update(logo="../trace-codex/icon.png")) with self.assertRaisesRegex(ValueError, "logo path"): - package.validate_public(self.public) + package.validate_mcp_package(self.public) (self.public / "assets/icon.png").unlink() with self.assertRaisesRegex(ValueError, "allowlist"): - package.validate_public(self.public) + package.validate_mcp_package(self.public) def test_duplicate_json_keys_rejected(self): path = self.public / "mcp.json" path.write_text(path.read_text().replace('"type": "streamable-http"', '"type": "stdio", "type": "streamable-http"')) with self.assertRaisesRegex(ValueError, "duplicate JSON"): - package.validate_public(self.public) + package.validate_mcp_package(self.public) def test_marketplace_escape_identity_and_implicit_install_rejected(self): path = self.dist / ".agents/plugins/marketplace.json"