From 3f136c6e82e8f2d4ee1cb7e88ae20026eda0c766 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Wed, 21 Oct 2026 02:47:22 +0800 Subject: [PATCH 01/10] Report Codex hook trust readiness and document managed activation --- bt-daemon/src/codex_hooks.rs | 498 ++++++++++++++++++ bt-daemon/src/command_output.rs | 78 ++- bt-daemon/src/lib.rs | 2 + bt-daemon/src/trace_runtime.rs | 31 +- docs/codex-managed-tracing.md | 131 +++++ .../content/plugins/trace-codex/README.md | 15 +- 6 files changed, 740 insertions(+), 15 deletions(-) create mode 100644 bt-daemon/src/codex_hooks.rs create mode 100644 docs/codex-managed-tracing.md diff --git a/bt-daemon/src/codex_hooks.rs b/bt-daemon/src/codex_hooks.rs new file mode 100644 index 00000000..20796606 --- /dev/null +++ b/bt-daemon/src/codex_hooks.rs @@ -0,0 +1,498 @@ +//! Read-only hook readiness through Codex's app-server protocol. Trust remains +//! owned by Codex; this module never writes its private trust store. + +use anyhow::{anyhow, Context}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::path::Path; +use std::process::Stdio; +use std::time::Duration; +use tokio::io::{ + AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, +}; + +const PLUGIN: &str = "trace-codex@braintrust-codex-plugins"; +const TIMEOUT: Duration = Duration::from_secs(5); +const EVENTS: &[&str] = &[ + "sessionStart", + "userPromptSubmit", + "preToolUse", + "permissionRequest", + "postToolUse", + "preCompact", + "postCompact", + "subagentStart", + "subagentStop", + "stop", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CodexHookState { + Trusted, + ReviewRequired, + Disabled, + PluginMissing, + MissingHooks, + ConfigurationError, + Unknown, +} + +impl std::fmt::Display for CodexHookState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Trusted => "trusted", + Self::ReviewRequired => "review required", + Self::Disabled => "disabled", + Self::PluginMissing => "plugin missing", + Self::MissingHooks => "missing hooks", + Self::ConfigurationError => "configuration error", + Self::Unknown => "unknown", + }) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct CodexHookReadiness { + pub state: CodexHookState, + pub hook_count: usize, + pub trusted_count: usize, + pub disabled_count: usize, + pub review_required_count: usize, + pub missing_events: Vec, + pub detail: String, +} + +impl CodexHookReadiness { + fn unknown(error: impl std::fmt::Display) -> Self { + Self { + state: CodexHookState::Unknown, + hook_count: 0, trusted_count: 0, disabled_count: 0, review_required_count: 0, + missing_events: Vec::new(), + detail: format!("Could not inspect Codex hooks: {error}. Open /hooks in Codex to inspect Braintrust hook trust; older Codex versions may not expose hooks/list."), + } + } +} + +pub(crate) async fn probe() -> CodexHookReadiness { + let cwd = match std::env::current_dir() { + Ok(cwd) => cwd, + Err(error) => return CodexHookReadiness::unknown(error), + }; + probe_at("codex", &cwd, None).await +} + +async fn probe_at(program: &str, cwd: &Path, home: Option<&Path>) -> CodexHookReadiness { + let mut command: tokio::process::Command = + crate::subprocess::background_command(program).into(); + command + .args(["--no-daemon", "app-server", "--stdio"]) + .current_dir(cwd) + .kill_on_drop(true) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + if let Some(home) = home { + command.env("CODEX_HOME", home); + } + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => return CodexHookReadiness::unknown(error), + }; + let result = + tokio::time::timeout(TIMEOUT, async { + let mut stdin = child.stdin.take().context("Codex stdin unavailable")?; + let stdout = child.stdout.take().context("Codex stdout unavailable")?; + // Bound both total response bytes and wall time, including notifications. + let mut stdout = BufReader::new(stdout.take(8 * 1024 * 1024)); + request(&mut stdin, &mut stdout, 1, "initialize", json!({ + "clientInfo":{"name":"braintrust_trace_doctor","version":env!("CARGO_PKG_VERSION")}, + "capabilities":{"experimentalApi":true} + })).await?; + stdin.write_all(b"{\"method\":\"initialized\"}\n").await?; + let config = request( + &mut stdin, + &mut stdout, + 2, + "config/read", + json!({"includeLayers":false}), + ) + .await?; + let hooks = request( + &mut stdin, + &mut stdout, + 3, + "hooks/list", + json!({"cwds":[cwd]}), + ) + .await?; + let plugins = request( + &mut stdin, + &mut stdout, + 4, + "plugin/list", + json!({ + "cwds":[cwd], "forceRefetch":false, "marketplaceKinds":["local"] + }), + ) + .await?; + classify(&hooks, &config, &plugins) + }) + .await; + // Always reap the probe, even when the protocol is unsupported or stalls. + let _ = child.kill().await; + match result { + Ok(Ok(readiness)) => readiness, + Ok(Err(error)) => CodexHookReadiness::unknown(error), + Err(_) => CodexHookReadiness::unknown("probe timed out after 5 seconds"), + } +} + +async fn request( + stdin: &mut (impl AsyncWrite + Unpin), + stdout: &mut (impl AsyncBufRead + Unpin), + id: u64, + method: &str, + params: Value, +) -> anyhow::Result { + let mut encoded = serde_json::to_vec(&json!({"id":id,"method":method,"params":params}))?; + encoded.push(b'\n'); + stdin.write_all(&encoded).await?; + stdin.flush().await?; + loop { + let mut line = String::new(); + anyhow::ensure!( + stdout.read_line(&mut line).await? != 0, + "Codex closed the probe stream" + ); + let response: Value = serde_json::from_str(&line).context("invalid Codex response")?; + if response.get("id") != Some(&json!(id)) { + continue; + } + if let Some(error) = response.get("error") { + return Err(anyhow!( + "{method}: {}", + error + .get("message") + .and_then(Value::as_str) + .unwrap_or("request failed") + )); + } + return response + .get("result") + .cloned() + .context("Codex response has no result"); + } +} + +fn classify(hooks: &Value, config: &Value, plugins: &Value) -> anyhow::Result { + let entries = hooks + .get("data") + .and_then(Value::as_array) + .context("hooks/list has no data array")?; + let mut selected = Vec::new(); + let mut has_errors = false; + for entry in entries { + has_errors |= entry + .get("errors") + .and_then(Value::as_array) + .is_none_or(|errors| !errors.is_empty()); + for hook in entry + .get("hooks") + .and_then(Value::as_array) + .context("hooks/list entry has no hooks array")? + { + let is_plugin = hook.get("pluginId").and_then(Value::as_str) == Some(PLUGIN); + let is_managed = hook.get("isManaged").and_then(Value::as_bool) == Some(true) + && hook + .get("command") + .and_then(Value::as_str) + .is_some_and(|command| command.ends_with(" trace hook --source codex")); + if is_plugin || is_managed { + selected.push(hook); + } + } + } + let mut readiness = CodexHookReadiness { + state: CodexHookState::Unknown, + hook_count: selected.len(), + trusted_count: 0, + disabled_count: 0, + review_required_count: 0, + missing_events: EVENTS + .iter() + .filter(|event| { + !selected + .iter() + .any(|hook| hook.get("eventName").and_then(Value::as_str) == Some(**event)) + }) + .map(|event| (*event).to_owned()) + .collect(), + detail: String::new(), + }; + for hook in &selected { + anyhow::ensure!( + hook.get("enabled").and_then(Value::as_bool).is_some(), + "hook has no enabled state" + ); + if hook.get("enabled").and_then(Value::as_bool) == Some(false) { + readiness.disabled_count += 1; + } + match hook.get("trustStatus").and_then(Value::as_str) { + Some("trusted" | "managed") => readiness.trusted_count += 1, + Some("untrusted" | "modified") => readiness.review_required_count += 1, + _ => {} + } + } + let marketplaces = plugins + .get("marketplaces") + .and_then(Value::as_array) + .context("plugin/list has no marketplaces array")?; + let plugin = marketplaces + .iter() + .filter_map(|marketplace| marketplace.get("plugins").and_then(Value::as_array)) + .flatten() + .find(|plugin| plugin.get("id").and_then(Value::as_str) == Some(PLUGIN)); + let config = config + .get("config") + .filter(|value| value.is_object()) + .context("config/read has no config object")?; + let features = &config["features"]; + let plugin_capture = selected.is_empty() + || selected + .iter() + .any(|hook| hook.get("pluginId").and_then(Value::as_str) == Some(PLUGIN)); + let disabled = features + .get("hooks") + .or_else(|| features.get("codex_hooks")) + .and_then(Value::as_bool) + == Some(false) + || plugin_capture + && plugin.is_some_and(|plugin| { + plugin.get("enabled").and_then(Value::as_bool) == Some(false) + }); + let (state, detail) = if disabled || readiness.disabled_count > 0 { + (CodexHookState::Disabled, "Codex hooks or the Braintrust plugin are disabled. Enable them in Codex and inspect /hooks.") + } else if has_errors { + (CodexHookState::ConfigurationError, "Codex reported hook configuration errors. Inspect /hooks and Codex startup diagnostics.") + } else if readiness.review_required_count > 0 { + (CodexHookState::ReviewRequired, "Open /hooks in Codex and review the current Braintrust hook definitions. Untrusted or modified hooks are skipped before capture.") + } else if selected.is_empty() + && plugin + .is_none_or(|plugin| plugin.get("installed").and_then(Value::as_bool) != Some(true)) + { + (CodexHookState::PluginMissing, "Braintrust Codex hooks are not installed. Run bt trace enable codex, then review /hooks in Codex.") + } else if !readiness.missing_events.is_empty() { + (CodexHookState::MissingHooks, "Some Braintrust lifecycle hooks are missing. Update the plugin or complete the managed hook policy, then inspect /hooks.") + } else if readiness.trusted_count == readiness.hook_count { + (CodexHookState::Trusted, "Braintrust hook definitions are enabled and trusted for this working directory. This does not confirm hook execution or Braintrust delivery.") + } else { + ( + CodexHookState::Unknown, + "Codex reported an unknown hook trust state. Inspect /hooks before relying on capture.", + ) + }; + readiness.state = state; + readiness.detail = detail.into(); + Ok(readiness) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn snapshot(trust: &str) -> Value { + json!({"data":[{"errors":[],"hooks":EVENTS.iter().map(|event|json!({ + "eventName":event,"pluginId":PLUGIN,"enabled":true,"trustStatus":trust + })).collect::>()}]}) + } + fn installed() -> Value { + json!({"marketplaces":[{"plugins":[{"id":PLUGIN,"installed":true,"enabled":true}]}]}) + } + + #[test] + fn required_events_match_the_packaged_plugin() { + let hooks: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json" + ))) + .unwrap(); + let events: std::collections::BTreeSet<_> = hooks["hooks"] + .as_object() + .unwrap() + .keys() + .map(|key| format!("{}{}", key[..1].to_ascii_lowercase(), &key[1..])) + .collect(); + assert_eq!( + events, + EVENTS.iter().map(|event| (*event).to_owned()).collect() + ); + } + + #[test] + fn classifies_trust_modifications_missing_disabled_and_managed_hooks() { + let config = json!({"config":{"features":{"hooks":true}}}); + for (trust, state) in [ + ("trusted", CodexHookState::Trusted), + ("untrusted", CodexHookState::ReviewRequired), + ("modified", CodexHookState::ReviewRequired), + ("future", CodexHookState::Unknown), + ] { + assert_eq!( + classify(&snapshot(trust), &config, &installed()) + .unwrap() + .state, + state + ); + } + let mut hooks = snapshot("trusted"); + hooks["data"][0]["hooks"][0]["enabled"] = json!(false); + assert_eq!( + classify(&hooks, &config, &installed()).unwrap().state, + CodexHookState::Disabled + ); + let missing = json!({"data":[{"errors":[],"hooks":[]}]}); + assert_eq!( + classify(&missing, &config, &installed()).unwrap().state, + CodexHookState::MissingHooks + ); + assert_eq!( + classify(&missing, &config, &json!({"marketplaces":[]})) + .unwrap() + .state, + CodexHookState::PluginMissing + ); + assert_eq!( + classify( + &snapshot("trusted"), + &json!({"config":{"features":{"hooks":false}}}), + &installed() + ) + .unwrap() + .state, + CodexHookState::Disabled + ); + let mut managed = snapshot("managed"); + for hook in managed["data"][0]["hooks"].as_array_mut().unwrap() { + hook["pluginId"] = Value::Null; + hook["isManaged"] = json!(true); + hook["command"] = json!("/enterprise/bt trace hook --source codex"); + } + assert_eq!( + classify(&managed, &config, &json!({"marketplaces":[]})) + .unwrap() + .state, + CodexHookState::Trusted + ); + } + + #[test] + fn unrelated_plugins_and_invalid_responses_never_establish_readiness() { + let mut hooks = snapshot("trusted"); + for hook in hooks["data"][0]["hooks"].as_array_mut().unwrap() { + hook["pluginId"] = json!("other@marketplace"); + } + assert_ne!( + classify(&hooks, &json!({"config":{}}), &installed()) + .unwrap() + .state, + CodexHookState::Trusted + ); + assert!(classify(&json!({}), &json!({}), &installed()).is_err()); + hooks["data"][0]["errors"] = json!([{"message":"bad hooks"}]); + assert_eq!( + classify(&hooks, &json!({"config":{}}), &installed()) + .unwrap() + .state, + CodexHookState::ConfigurationError + ); + } + + #[tokio::test] + async fn rpc_ignores_notifications_and_reports_unsupported_methods() { + let mut input = Vec::new(); + let mut output = std::io::Cursor::new( + b"{\"method\":\"notice\"}\n{\"id\":3,\"error\":{\"message\":\"Method not found\"}}\n", + ); + let error = request(&mut input, &mut output, 3, "hooks/list", json!({})) + .await + .unwrap_err(); + assert!(error.to_string().contains("Method not found")); + assert_eq!( + serde_json::from_slice::(&input).unwrap()["method"], + "hooks/list" + ); + } + + #[tokio::test] + async fn unavailable_codex_and_closed_protocol_report_unknown() { + let tmp = tempfile::tempdir().unwrap(); + let result = probe_at( + tmp.path().join("missing-codex").to_str().unwrap(), + tmp.path(), + None, + ) + .await; + assert_eq!(result.state, CodexHookState::Unknown); + let mut input = Vec::new(); + let mut output = std::io::Cursor::new(b""); + assert!(request(&mut input, &mut output, 1, "initialize", json!({})) + .await + .unwrap_err() + .to_string() + .contains("closed")); + } + + #[tokio::test] + #[ignore = "Requires an installed Codex CLI; installs only into an isolated temporary home"] + async fn installed_codex_distinguishes_missing_and_unreviewed_hooks() { + let tmp = tempfile::tempdir().unwrap(); + let readiness = probe_at("codex", tmp.path(), Some(tmp.path())).await; + assert_eq!( + readiness.state, + CodexHookState::PluginMissing, + "{readiness:?}" + ); + let marketplace = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../src/plugins/codex/content") + .canonicalize() + .unwrap(); + let marketplace = marketplace.to_string_lossy(); + for args in [ + vec![ + "plugin", + "marketplace", + "add", + marketplace.as_ref(), + "--json", + ], + vec!["plugin", "add", PLUGIN, "--json"], + vec!["plugin", "add", PLUGIN, "--json"], + ] { + let mut command: tokio::process::Command = + crate::subprocess::background_command("codex").into(); + let output = command + .args(["--no-daemon"]) + .args(args) + .env("CODEX_HOME", tmp.path()) + .current_dir(tmp.path()) + .kill_on_drop(true) + .output() + .await + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + let readiness = probe_at("codex", tmp.path(), Some(tmp.path())).await; + assert_eq!( + readiness.state, + CodexHookState::ReviewRequired, + "{readiness:?}" + ); + assert_eq!(readiness.review_required_count, EVENTS.len()); + assert!(readiness.missing_events.is_empty()); + } +} diff --git a/bt-daemon/src/command_output.rs b/bt-daemon/src/command_output.rs index 704e7f09..3a192062 100644 --- a/bt-daemon/src/command_output.rs +++ b/bt-daemon/src/command_output.rs @@ -59,6 +59,8 @@ pub struct SetupCommandOutput { pub display_name: String, pub settings_path: PathBuf, pub restart_required: bool, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub next_steps: Vec, } #[derive(Debug, Clone, Serialize)] @@ -66,6 +68,8 @@ pub struct UpdateCommandOutput { pub source: String, pub display_name: String, pub restart_required: bool, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub next_steps: Vec, } #[derive(Debug, Clone, Serialize)] @@ -158,6 +162,8 @@ pub struct DoctorCommandOutput { pub daemon: DaemonDiagnostic, pub warnings: Vec, pub plugin_diagnostics: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub codex_hooks: Option, } #[derive(Debug, Clone, Serialize)] @@ -186,8 +192,10 @@ impl TraceCommandOutput { display_name: impl Into, settings_path: impl Into, ) -> Self { + let source = source.into(); Self::Enable(SetupCommandOutput { - source: source.into(), + next_steps: activation_steps(&source), + source, display_name: display_name.into(), settings_path: settings_path.into(), restart_required: true, @@ -208,6 +216,7 @@ impl TraceCommandOutput { settings_path: impl Into, ) -> Self { Self::Disable(SetupCommandOutput { + next_steps: Vec::new(), source: source.into(), display_name: display_name.into(), settings_path: settings_path.into(), @@ -216,8 +225,10 @@ impl TraceCommandOutput { } pub fn update(source: impl Into, display_name: impl Into) -> Self { + let source = source.into(); Self::Update(UpdateCommandOutput { - source: source.into(), + next_steps: activation_steps(&source), + source, display_name: display_name.into(), restart_required: true, }) @@ -274,6 +285,14 @@ impl TraceCommandOutput { for warning in &doctor.warnings { rendered.push_str(&format!("\nWarning: {warning}")); } + if let Some(hooks) = &doctor.codex_hooks { + rendered.push_str(&format!("\nCodex hook trust: {}\n{}\nHooks: {} loaded, {} trusted, {} require review, {} disabled", + hooks.state, hooks.detail, hooks.hook_count, hooks.trusted_count, + hooks.review_required_count, hooks.disabled_count)); + if !hooks.missing_events.is_empty() { + rendered.push_str(&format!("\nMissing hook events: {}", hooks.missing_events.join(", "))); + } + } for diagnostic in &doctor.plugin_diagnostics { rendered.push_str(&format!( "\nPlugin error: {} ({} occurrence{})\nFirst seen: {}\nLast seen: {}\n{}", @@ -288,9 +307,9 @@ impl TraceCommandOutput { Ok(rendered) } Self::Enable(setup) => Ok(format!( - "The Braintrust tracing plugin is installed for {} and configured in {}.\nRestart the coding agent to load the tracing plugin.", + "The Braintrust tracing plugin is installed for {} and configured in {}.\nRestart the coding agent to load the tracing plugin.{}", setup.display_name, - setup.settings_path.display() + setup.settings_path.display(), render_next_steps(&setup.next_steps) )), Self::Disable(disable) => Ok(format!( "The Braintrust tracing plugin and configuration were removed for {} from {}.\nRestart the coding agent to apply the change.", @@ -298,8 +317,8 @@ impl TraceCommandOutput { disable.settings_path.display() )), Self::Update(update) => Ok(format!( - "The Braintrust tracing plugin was updated for {}.\nRestart the coding agent to load the update.", - update.display_name, + "The Braintrust tracing plugin was updated for {}.\nRestart the coding agent to load the update.{}", + update.display_name, render_next_steps(&update.next_steps), )), Self::Stop(stop) if stop.stopped => Ok("Tracing daemon stopped.".into()), Self::Stop(_) => Ok("No tracing daemon is running.".into()), @@ -353,6 +372,18 @@ fn push_auth_lines(rendered: &mut String, prefix: &str, auth: &AuthDiagnostic) { } } +fn activation_steps(source: &str) -> Vec { + if source == "codex" { + vec!["Codex separately requires trust for plugin hooks. Open /hooks in Codex and review the Braintrust definitions, then run bt trace doctor codex. Installation alone does not confirm capture or delivery.".into()] + } else { + Vec::new() + } +} + +fn render_next_steps(steps: &[String]) -> String { + steps.iter().map(|step| format!("\n{step}")).collect() +} + fn render_timestamp(timestamp_ms: i64) -> String { chrono::DateTime::from_timestamp_millis(timestamp_ms) .map(|timestamp| timestamp.to_rfc3339()) @@ -427,6 +458,27 @@ mod tests { .contains("removed for Google Antigravity")); } + #[test] + fn codex_setup_and_updates_explain_the_separate_hook_review_step() { + for output in [ + TraceCommandOutput::setup("codex", "Codex", PathBuf::from("braintrust.json")), + TraceCommandOutput::update("codex", "Codex"), + ] { + let human = output.render(OutputFormat::Human).unwrap(); + assert!(human.contains("Open /hooks")); + assert!(human.contains("Installation alone does not confirm capture or delivery")); + let json: serde_json::Value = + serde_json::from_str(&output.render(OutputFormat::Json).unwrap()).unwrap(); + assert_eq!(json["next_steps"].as_array().unwrap().len(), 1); + } + let disabled = + TraceCommandOutput::disable("codex", "Codex", PathBuf::from("braintrust.json")); + assert!(!disabled + .render(OutputFormat::Human) + .unwrap() + .contains("Open /hooks")); + } + #[test] fn stop_json_reports_idempotent_and_successful_shutdowns() { let absent: serde_json::Value = serde_json::from_str( @@ -535,6 +587,15 @@ mod tests { error: None, }, warnings: Vec::new(), + codex_hooks: Some(crate::CodexHookReadiness { + state: crate::CodexHookState::ReviewRequired, + hook_count: 10, + trusted_count: 0, + disabled_count: 0, + review_required_count: 10, + missing_events: Vec::new(), + detail: "Open /hooks to review Braintrust definitions".into(), + }), plugin_diagnostics: vec![crate::PluginDiagnostic { source: "codex".into(), plugin_path: PathBuf::from("/tmp/redact.mjs"), @@ -551,6 +612,11 @@ mod tests { assert_eq!(value["auth"]["source"], "saved_profile"); assert_eq!(value["daemon"]["status"], "running"); assert_eq!(value["daemon"]["auth"]["status"], "error"); + assert_eq!(value["codex_hooks"]["state"], "review_required"); + assert!(output + .render(OutputFormat::Human) + .unwrap() + .contains("10 require review")); assert_eq!( value["daemon"]["session_errors"][0], "could not resolve Braintrust auth for codex" diff --git a/bt-daemon/src/lib.rs b/bt-daemon/src/lib.rs index 61a3f58f..9e1e5470 100644 --- a/bt-daemon/src/lib.rs +++ b/bt-daemon/src/lib.rs @@ -15,6 +15,7 @@ pub mod paths; mod agents; mod args; mod client; +mod codex_hooks; mod command_output; mod correlation; mod delivery_ledger; @@ -51,6 +52,7 @@ pub use args::{ pub use client::{ flush_managed_run, flush_session, forward_envelope, run_status, shutdown_daemon, HostInfo, }; +pub use codex_hooks::{CodexHookReadiness, CodexHookState}; pub use command_output::{ AuthDiagnostic, DaemonDiagnostic, DaemonStatus, DoctorCommandOutput, ImportSummary, OutputFormat, SetupCommandOutput, StatusCommandOutput, StopCommandOutput, TraceCommandOutput, diff --git a/bt-daemon/src/trace_runtime.rs b/bt-daemon/src/trace_runtime.rs index 684c97b6..5a5fece5 100644 --- a/bt-daemon/src/trace_runtime.rs +++ b/bt-daemon/src/trace_runtime.rs @@ -495,14 +495,28 @@ fn daemon_warnings( async fn doctor_output(host: &TraceHostContext, args: DoctorArgs) -> DoctorCommandOutput { let source = args.agent.identity().id; - doctor_output_at( - host, - args, - &paths::socket_path(None), - paths::agent_settings_path(source, None), - crate::settings::AgentSettings::load(source), - ) - .await + let settings = crate::settings::AgentSettings::load(source); + let socket = paths::socket_path(None); + let inspect_hooks = matches!(args.agent, DoctorAgent::Codex) && settings.tracing_enabled(); + let hooks = async { + if inspect_hooks { + Some(crate::codex_hooks::probe().await) + } else { + None + } + }; + let (mut output, hooks) = tokio::join!( + doctor_output_at( + host, + args, + &socket, + paths::agent_settings_path(source, None), + settings, + ), + hooks + ); + output.codex_hooks = hooks; + output } async fn doctor_output_at( @@ -632,6 +646,7 @@ async fn doctor_output_at( daemon, warnings, plugin_diagnostics, + codex_hooks: None, } } diff --git a/docs/codex-managed-tracing.md b/docs/codex-managed-tracing.md new file mode 100644 index 00000000..c9835761 --- /dev/null +++ b/docs/codex-managed-tracing.md @@ -0,0 +1,131 @@ +# Codex hook activation and enterprise deployment + +Installing `trace-codex` does not approve its hooks. Codex skips untrusted or +modified plugin hooks before Braintrust receives their events. Restart Codex, +open `/hooks`, review the Braintrust definitions, and run `bt trace doctor codex`. +The doctor probes the installed Codex app server for the current working directory. +It distinguishes trusted, review-required, disabled, missing, erroneous, and +unknown configurations. Trust readiness does not prove capture or delivery. + +## Supported approval mechanisms + +[Official OpenAI hook documentation](https://learn.chatgpt.com/docs/hooks#review-and-trust-hooks) +describes user review through `/hooks` and administrator-managed hooks trusted by +policy. Codex CLI 0.163.0 exposes `hooks/list` with each definition's enabled and +trust state, but its generated app-server protocol has no hook approval method. +Braintrust therefore does not modify Codex's private trust files or approve +unrelated hooks. A release that changes hook definitions can require review again; +route changes and `bt` updates do not change the marketplace definitions. + +The invocation-wide `--dangerously-bypass-hook-trust` flag is not scoped to +Braintrust. `bt trace run codex` continues to reject it. For unattended enterprise +activation, use Codex's supported managed policy rather than that flag. + +## Administrator-managed local Codex hooks + +This recipe targets local Codex execution. Administrators own policy deployment; +`bt trace enable` does not write managed policy. Check the installed Codex version +against [OpenAI's managed hook documentation](https://learn.chatgpt.com/docs/hooks#managed-hooks-from-requirementstoml). + +1. Distribute a current `bt` executable to the managed directory on each platform. +2. Have each user authenticate and configure their route with + `bt trace enable codex --project PROJECT`. Credentials stay in the CLI's + authentication store; the route stays in the user's `braintrust.json`. +3. Remove the user plugin with + `codex plugin remove trace-codex@braintrust-codex-plugins` after configuring + the route. This avoids duplicate plugin and managed capture; this Codex + command leaves Braintrust's route settings intact. +4. Merge the following into the organization's Codex `requirements.toml` using + its normal device-management mechanism. Preserve unrelated requirements. + Replace the executable directories with administrator-controlled locations. + +```toml +[features] +hooks = true + +[hooks] +managed_dir = "/opt/company/braintrust" +windows_managed_dir = 'C:\Braintrust' + +[[hooks.SessionStart]] +[[hooks.SessionStart.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.UserPromptSubmit]] +[[hooks.UserPromptSubmit.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.PreToolUse]] +[[hooks.PreToolUse.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.PermissionRequest]] +[[hooks.PermissionRequest.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.PostToolUse]] +[[hooks.PostToolUse.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.PreCompact]] +[[hooks.PreCompact.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.PostCompact]] +[[hooks.PostCompact.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.SubagentStart]] +[[hooks.SubagentStart.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.SubagentStop]] +[[hooks.SubagentStop.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.Stop]] +[[hooks.Stop.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' + +[[hooks.SessionEnd]] +[[hooks.SessionEnd.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' +``` + +Keep all lifecycle events so sessions, turns, tools, and subagents retain their +normal trace structure. Codex does not distribute the executable itself. The +policy above enables hooks without disabling other hook sources; an organization +may separately choose `allow_managed_hooks_only` if that is its intended policy. + +After deployment, restart Codex and inspect `/hooks`: Braintrust definitions +should be managed and enabled. Run `bt trace doctor codex` from the actual project +directory, then run a short session and verify its matching trace in Braintrust. +A running daemon, trusted definitions, or historical traces alone do not confirm +that the newly deployed path delivered the session. + +For ordinary plugin installs, hooks still need review after definition changes. +Events skipped before approval are not in the daemon's journal; replay cannot +recover those events. Historical transcript import may recover supported data, +with the limitations documented for that import path. diff --git a/src/plugins/codex/content/plugins/trace-codex/README.md b/src/plugins/codex/content/plugins/trace-codex/README.md index efc5553e..62441f7a 100644 --- a/src/plugins/codex/content/plugins/trace-codex/README.md +++ b/src/plugins/codex/content/plugins/trace-codex/README.md @@ -23,7 +23,20 @@ bt trace enable codex --project my-coding-agent Use `--profile` or `--org` when needed. Restart Codex after setup so it loads the plugin. Codex will apply its normal hook-review flow; approve the stable -Braintrust hook definition through `/hooks` when prompted. +Braintrust hook definition through `/hooks` when prompted. Installing the plugin +alone does not approve hooks: untrusted or modified definitions are skipped +before Braintrust can capture events. + +`bt trace doctor codex` inspects hook trust through the installed Codex app server +for the current directory. It reports missing or disabled hooks, pending review, +and trusted definitions separately from daemon authentication and delivery. +Older Codex versions without this inspection API report an unknown state and +still require inspection through `/hooks`. + +For unattended enterprise deployment, see the +[administrator-managed hook recipe](https://github.com/braintrustdata/braintrust-coding-agent-plugins/blob/main/docs/codex-managed-tracing.md). +Managed hooks use Codex policy-based trust. Braintrust does not edit Codex's +private trust store or bypass review for other hooks. To inspect tracing configuration and daemon status: From 661377b1cc4354c1fb1d79a156d25b05209c4a8f Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 17:44:29 +0800 Subject: [PATCH 02/10] Validate managed Codex terminal hooks and reject duplicate capture --- bt-daemon/src/codex_hooks.rs | 171 +++++++++++++++++++++++++--- bt-daemon/tests/codex_translator.rs | 54 +++++++++ docs/codex-managed-tracing.md | 17 ++- 3 files changed, 226 insertions(+), 16 deletions(-) diff --git a/bt-daemon/src/codex_hooks.rs b/bt-daemon/src/codex_hooks.rs index 20796606..b176dd68 100644 --- a/bt-daemon/src/codex_hooks.rs +++ b/bt-daemon/src/codex_hooks.rs @@ -13,7 +13,7 @@ use tokio::io::{ const PLUGIN: &str = "trace-codex@braintrust-codex-plugins"; const TIMEOUT: Duration = Duration::from_secs(5); -const EVENTS: &[&str] = &[ +const PLUGIN_EVENTS: &[&str] = &[ "sessionStart", "userPromptSubmit", "preToolUse", @@ -25,6 +25,9 @@ const EVENTS: &[&str] = &[ "subagentStop", "stop", ]; +// Managed policy is independent of the marketplace artifact and must observe +// terminal lifecycle records even when no subsequent hook fires. +const MANAGED_EXTRA_EVENTS: &[&str] = &["interrupt", "sessionEnd"]; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] @@ -191,6 +194,8 @@ fn classify(hooks: &Value, config: &Value, plugins: &Value) -> anyhow::Result anyhow::Result anyhow::Result anyhow::Result 0 { + let enabled = |hooks: &[&Value]| { + hooks + .iter() + .any(|hook| hook.get("enabled").and_then(Value::as_bool) == Some(true)) + }; + let duplicate_capture = enabled(&plugin_hooks) && enabled(&managed_hooks); + let (state, detail) = if duplicate_capture { + (CodexHookState::ConfigurationError, "Both plugin and managed Braintrust hooks are enabled, which can forward lifecycle events twice. Remove trace-codex@braintrust-codex-plugins from Codex when using managed capture, or remove the managed Braintrust policy when using the plugin.") + } else if disabled || readiness.disabled_count > 0 { (CodexHookState::Disabled, "Codex hooks or the Braintrust plugin are disabled. Enable them in Codex and inspect /hooks.") } else if has_errors { (CodexHookState::ConfigurationError, "Codex reported hook configuration errors. Inspect /hooks and Codex startup diagnostics.") @@ -302,7 +333,7 @@ mod tests { use super::*; fn snapshot(trust: &str) -> Value { - json!({"data":[{"errors":[],"hooks":EVENTS.iter().map(|event|json!({ + json!({"data":[{"errors":[],"hooks":PLUGIN_EVENTS.iter().map(|event|json!({ "eventName":event,"pluginId":PLUGIN,"enabled":true,"trustStatus":trust })).collect::>()}]}) } @@ -310,6 +341,118 @@ mod tests { json!({"marketplaces":[{"plugins":[{"id":PLUGIN,"installed":true,"enabled":true}]}]}) } + fn managed_snapshot() -> Value { + let hooks: Vec<_> = PLUGIN_EVENTS + .iter() + .chain(MANAGED_EXTRA_EVENTS) + .map(|event| { + json!({ + "eventName":event,"pluginId":null,"enabled":true,"trustStatus":"managed", + "isManaged":true,"command":"/enterprise/bt trace hook --source codex" + }) + }) + .collect(); + json!({"data":[{"errors":[],"hooks":hooks}]}) + } + + #[test] + fn managed_capture_requires_interrupt_and_session_end_without_changing_plugin_requirements() { + let config = json!({"config":{}}); + let plugins = json!({"marketplaces":[]}); + for missing in MANAGED_EXTRA_EVENTS { + let mut managed = managed_snapshot(); + managed["data"][0]["hooks"] + .as_array_mut() + .unwrap() + .retain(|hook| hook["eventName"] != *missing); + let result = classify(&managed, &config, &plugins).unwrap(); + assert_eq!(result.state, CodexHookState::MissingHooks); + assert_eq!(result.missing_events, [*missing]); + } + let plugin = classify(&snapshot("trusted"), &config, &installed()).unwrap(); + assert_eq!(plugin.state, CodexHookState::Trusted); + assert!(plugin.missing_events.is_empty()); + let managed = classify(&managed_snapshot(), &config, &plugins).unwrap(); + assert_eq!(managed.state, CodexHookState::Trusted); + assert!(managed.missing_events.is_empty()); + + let policy = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../docs/codex-managed-tracing.md" + )); + for event in PLUGIN_EVENTS.iter().chain(MANAGED_EXTRA_EVENTS) { + let event = format!("{}{}", event[..1].to_ascii_uppercase(), &event[1..]); + assert!( + policy.contains(&format!("[[hooks.{event}.hooks]]")), + "policy omits {event}" + ); + } + } + + #[test] + fn simultaneous_enabled_plugin_and_managed_capture_is_a_configuration_error() { + let config = json!({"config":{}}); + let mut mixed = snapshot("trusted"); + mixed["data"][0]["hooks"].as_array_mut().unwrap().extend( + managed_snapshot()["data"][0]["hooks"] + .as_array() + .unwrap() + .iter() + .cloned(), + ); + let result = classify(&mixed, &config, &installed()).unwrap(); + assert_eq!(result.state, CodexHookState::ConfigurationError); + assert!(result.detail.contains("events twice")); + + // A plugin hook cannot fill a missing terminal event in managed policy. + mixed["data"][0]["hooks"] + .as_array_mut() + .unwrap() + .retain(|hook| hook["eventName"] != "interrupt"); + let mut plugin_interrupt = snapshot("trusted")["data"][0]["hooks"][0].clone(); + plugin_interrupt["eventName"] = json!("interrupt"); + mixed["data"][0]["hooks"] + .as_array_mut() + .unwrap() + .push(plugin_interrupt); + assert_eq!( + classify(&mixed, &config, &installed()) + .unwrap() + .missing_events, + ["interrupt"] + ); + + let mut unrelated = snapshot("trusted"); + let mut managed = managed_snapshot(); + for hook in managed["data"][0]["hooks"].as_array_mut().unwrap() { + hook["command"] = json!("/enterprise/bt trace hook --source claude-code"); + } + unrelated["data"][0]["hooks"] + .as_array_mut() + .unwrap() + .extend( + managed["data"][0]["hooks"] + .as_array() + .unwrap() + .iter() + .cloned(), + ); + assert_eq!( + classify(&unrelated, &config, &installed()).unwrap().state, + CodexHookState::Trusted + ); + + for hook in mixed["data"][0]["hooks"].as_array_mut().unwrap() { + if hook["isManaged"] == true { + hook["enabled"] = json!(false); + } + } + assert_eq!( + classify(&mixed, &config, &installed()).unwrap().state, + CodexHookState::Disabled + ); + } + #[test] fn required_events_match_the_packaged_plugin() { let hooks: Value = serde_json::from_str(include_str!(concat!( @@ -325,7 +468,10 @@ mod tests { .collect(); assert_eq!( events, - EVENTS.iter().map(|event| (*event).to_owned()).collect() + PLUGIN_EVENTS + .iter() + .map(|event| (*event).to_owned()) + .collect() ); } @@ -372,12 +518,7 @@ mod tests { .state, CodexHookState::Disabled ); - let mut managed = snapshot("managed"); - for hook in managed["data"][0]["hooks"].as_array_mut().unwrap() { - hook["pluginId"] = Value::Null; - hook["isManaged"] = json!(true); - hook["command"] = json!("/enterprise/bt trace hook --source codex"); - } + let managed = managed_snapshot(); assert_eq!( classify(&managed, &config, &json!({"marketplaces":[]})) .unwrap() @@ -492,7 +633,7 @@ mod tests { CodexHookState::ReviewRequired, "{readiness:?}" ); - assert_eq!(readiness.review_required_count, EVENTS.len()); + assert_eq!(readiness.review_required_count, PLUGIN_EVENTS.len()); assert!(readiness.missing_events.is_empty()); } } diff --git a/bt-daemon/tests/codex_translator.rs b/bt-daemon/tests/codex_translator.rs index 30d7751a..84937d8f 100644 --- a/bt-daemon/tests/codex_translator.rs +++ b/bt-daemon/tests/codex_translator.rs @@ -284,6 +284,60 @@ fn codex_happy_path_builds_session_turn_llm_tool_tree() { ); } +#[test] +fn interrupt_hook_closes_aborted_work_without_a_later_hook_or_flush() { + let tmp = tempfile::tempdir().unwrap(); + let transcript = tmp.path().join("rollout.jsonl"); + let path = transcript.to_str().unwrap(); + for record in [ + json!({"timestamp":"2026-01-01T00:00:01Z","type":"session_meta","payload":{"id":"s","cwd":"/x/app"}}), + json!({"timestamp":"2026-01-01T00:00:02Z","type":"event_msg","payload":{"type":"task_started","turn_id":"t1"}}), + json!({"timestamp":"2026-01-01T00:00:03Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Working"}]}}), + json!({"timestamp":"2026-01-01T00:00:04Z","type":"response_item","payload":{"type":"function_call","call_id":"c1","name":"shell","arguments":"{}","metadata":{"turn_id":"t1"}}}), + ] { + append(&transcript, record); + } + let mut translator = Registry::default_agents().create("codex", "s"); + let ctx = SessionCtx { + session_id: "s".into(), + config: None, + }; + let mut ops = translator + .handle(&envelope("s", "SessionStart", path, json!({})), &ctx) + .unwrap(); + let before = reduce(ops.clone()); + assert_eq!(find(&before, SpanType::Task, "turn: t1").end_ms, None); + assert_eq!(find(&before, SpanType::Tool, "shell").end_ms, None); + append( + &transcript, + json!({"timestamp":"2026-01-01T00:00:05Z","type":"event_msg","payload":{"type":"turn_aborted","turn_id":"t1","reason":"interrupted"}}), + ); + ops.extend( + translator + .handle( + &envelope("s", "Interrupt", path, json!({"turn_id":"t1"})), + &ctx, + ) + .unwrap(), + ); + let after = reduce(ops); + for row in [ + find(&after, SpanType::Task, "turn: t1"), + find(&after, SpanType::Tool, "shell"), + ] { + assert_eq!(row.end_ms, Some(1_767_225_605_000)); + assert_eq!(row.metadata.as_ref().unwrap()["outcome"], "interrupted"); + assert!(row.error.is_none()); + } + let turn = find(&after, SpanType::Task, "turn: t1"); + let llm = after + .values() + .find(|row| row.span_type == SpanType::Llm && row.parent_span_ids.contains(&turn.span_id)) + .unwrap(); + assert!(llm.end_ms.is_some()); + assert_eq!(llm.metadata.as_ref().unwrap()["outcome"], "interrupted"); +} + #[test] fn codex_turn_aborted_closes_open_work_as_interrupted() { let tmp = tempfile::tempdir().unwrap(); diff --git a/docs/codex-managed-tracing.md b/docs/codex-managed-tracing.md index c9835761..ba5995dd 100644 --- a/docs/codex-managed-tracing.md +++ b/docs/codex-managed-tracing.md @@ -112,11 +112,26 @@ command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' type = "command" command = "/opt/company/braintrust/bt trace hook --source codex" command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' +timeout = 3 + +[[hooks.Interrupt]] +[[hooks.Interrupt.hooks]] +type = "command" +command = "/opt/company/braintrust/bt trace hook --source codex" +command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' +timeout = 3 ``` Keep all lifecycle events so sessions, turns, tools, and subagents retain their normal trace structure. Codex does not distribute the executable itself. The -policy above enables hooks without disabling other hook sources; an organization +managed policy must include `Interrupt` to observe cancellation records even +when the session stays idle, and `SessionEnd` to observe session termination. +Doctor checks these managed-only requirements separately from the packaged +plugin's event set. It reports a configuration error if plugin and managed +Braintrust capture are both enabled, including after a later `bt trace enable +codex` reinstalls the plugin. Keep only one capture source active. + +The policy above enables hooks without disabling other hook sources; an organization may separately choose `allow_managed_hooks_only` if that is its intended policy. After deployment, restart Codex and inspect `/hooks`: Braintrust definitions From 96a41aa23ae3d1104190ad74eb425ec26d91b483 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 18:48:14 +0800 Subject: [PATCH 03/10] Automatically trust installed Braintrust Codex hooks through config API --- bt-daemon/src/codex_hooks.rs | 222 ++++++++++++++---- bt-daemon/src/command_output.rs | 34 +-- bt-daemon/src/trace_runtime.rs | 18 +- docs/codex-managed-tracing.md | 56 +++-- .../content/plugins/trace-codex/README.md | 27 +-- 5 files changed, 246 insertions(+), 111 deletions(-) diff --git a/bt-daemon/src/codex_hooks.rs b/bt-daemon/src/codex_hooks.rs index b176dd68..659e84c7 100644 --- a/bt-daemon/src/codex_hooks.rs +++ b/bt-daemon/src/codex_hooks.rs @@ -1,5 +1,5 @@ -//! Read-only hook readiness through Codex's app-server protocol. Trust remains -//! owned by Codex; this module never writes its private trust store. +//! Inspect and activate Braintrust hooks through Codex's app-server protocol. +//! Trust writes use the same exact-definition config API as Codex's /hooks UI. use anyhow::{anyhow, Context}; use serde::Serialize; @@ -12,6 +12,10 @@ use tokio::io::{ }; const PLUGIN: &str = "trace-codex@braintrust-codex-plugins"; +const PACKAGED_HOOKS: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json" +)); const TIMEOUT: Duration = Duration::from_secs(5); const PLUGIN_EVENTS: &[&str] = &[ "sessionStart", @@ -85,7 +89,28 @@ pub(crate) async fn probe() -> CodexHookReadiness { probe_at("codex", &cwd, None).await } +pub(crate) async fn activate(allow_disabled: bool) -> anyhow::Result<()> { + let cwd = std::env::current_dir()?; + let readiness = inspect_at("codex", &cwd, None, true).await; + anyhow::ensure!( + readiness.state == CodexHookState::Trusted + || (allow_disabled && readiness.state == CodexHookState::Disabled), + "Codex plugin installed, but hook activation failed: {}", + readiness.detail + ); + Ok(()) +} + async fn probe_at(program: &str, cwd: &Path, home: Option<&Path>) -> CodexHookReadiness { + inspect_at(program, cwd, home, false).await +} + +async fn inspect_at( + program: &str, + cwd: &Path, + home: Option<&Path>, + approve: bool, +) -> CodexHookReadiness { let mut command: tokio::process::Command = crate::subprocess::background_command(program).into(); command @@ -102,46 +127,81 @@ async fn probe_at(program: &str, cwd: &Path, home: Option<&Path>) -> CodexHookRe Ok(child) => child, Err(error) => return CodexHookReadiness::unknown(error), }; - let result = - tokio::time::timeout(TIMEOUT, async { - let mut stdin = child.stdin.take().context("Codex stdin unavailable")?; - let stdout = child.stdout.take().context("Codex stdout unavailable")?; - // Bound both total response bytes and wall time, including notifications. - let mut stdout = BufReader::new(stdout.take(8 * 1024 * 1024)); - request(&mut stdin, &mut stdout, 1, "initialize", json!({ - "clientInfo":{"name":"braintrust_trace_doctor","version":env!("CARGO_PKG_VERSION")}, - "capabilities":{"experimentalApi":true} - })).await?; - stdin.write_all(b"{\"method\":\"initialized\"}\n").await?; - let config = request( - &mut stdin, - &mut stdout, - 2, - "config/read", - json!({"includeLayers":false}), - ) - .await?; - let hooks = request( - &mut stdin, - &mut stdout, - 3, - "hooks/list", - json!({"cwds":[cwd]}), - ) - .await?; - let plugins = request( - &mut stdin, - &mut stdout, - 4, - "plugin/list", - json!({ - "cwds":[cwd], "forceRefetch":false, "marketplaceKinds":["local"] - }), - ) - .await?; - classify(&hooks, &config, &plugins) - }) - .await; + let result = tokio::time::timeout(TIMEOUT, async { + let mut stdin = child.stdin.take().context("Codex stdin unavailable")?; + let stdout = child.stdout.take().context("Codex stdout unavailable")?; + // Bound both total response bytes and wall time, including notifications. + let mut stdout = BufReader::new(stdout.take(8 * 1024 * 1024)); + request( + &mut stdin, + &mut stdout, + 1, + "initialize", + json!({ + "clientInfo":{"name":"braintrust_trace","version":env!("CARGO_PKG_VERSION")}, + "capabilities":{"experimentalApi":true} + }), + ) + .await?; + stdin.write_all(b"{\"method\":\"initialized\"}\n").await?; + let config = request( + &mut stdin, + &mut stdout, + 2, + "config/read", + json!({"includeLayers":false}), + ) + .await?; + let hooks = request( + &mut stdin, + &mut stdout, + 3, + "hooks/list", + json!({"cwds":[cwd]}), + ) + .await?; + let plugins = request( + &mut stdin, + &mut stdout, + 4, + "plugin/list", + json!({ + "cwds":[cwd], "forceRefetch":false, "marketplaceKinds":["local"] + }), + ) + .await?; + let readiness = classify(&hooks, &config, &plugins)?; + if !approve || readiness.state != CodexHookState::ReviewRequired { + return Ok(readiness); + } + anyhow::ensure!( + readiness.missing_events.is_empty(), + "Braintrust lifecycle hooks are missing" + ); + let edits = trust_edits(&hooks)?; + anyhow::ensure!(!edits.is_empty(), "No Braintrust hooks available to trust"); + request( + &mut stdin, + &mut stdout, + 5, + "config/batchWrite", + json!({ + "edits":[{"keyPath":"hooks.state", "value":edits,"mergeStrategy":"upsert"}], + "reloadUserConfig":true + }), + ) + .await?; + let hooks = request( + &mut stdin, + &mut stdout, + 6, + "hooks/list", + json!({"cwds":[cwd]}), + ) + .await?; + classify(&hooks, &config, &plugins) + }) + .await; // Always reap the probe, even when the protocol is unsupported or stalls. let _ = child.kill().await; match result { @@ -151,6 +211,77 @@ async fn probe_at(program: &str, cwd: &Path, home: Option<&Path>) -> CodexHookRe } } +/// Validate the complete installed file, including the other platform's command, +/// before approving exact hashes returned by Codex. Never infer hashes or keys. +fn trust_edits(hooks: &Value) -> anyhow::Result> { + let expected: Value = serde_json::from_str(PACKAGED_HOOKS)?; + let mut checked_paths = std::collections::HashSet::new(); + let mut events = std::collections::HashSet::new(); + let mut edits = serde_json::Map::new(); + for entry in hooks["data"] + .as_array() + .context("hooks/list has no data array")? + { + for hook in entry["hooks"] + .as_array() + .context("hooks/list entry has no hooks array")? + { + if hook["pluginId"].as_str() != Some(PLUGIN) { + continue; + } + let event = hook["eventName"].as_str().context("hook event missing")?; + anyhow::ensure!( + PLUGIN_EVENTS.contains(&event) + && events.insert(event) + && hook["handlerType"] == "command" + && hook["command"] == "bt trace hook --source codex" + && hook["enabled"] == true + && hook["isManaged"] == false + && hook["source"] == "plugin", + "Unexpected or disabled Braintrust hook definition; inspect /hooks" + ); + let path = hook["sourcePath"] + .as_str() + .context("hook source path missing")?; + anyhow::ensure!( + Path::new(path).is_absolute(), + "hook source path is not absolute" + ); + if checked_paths.insert(path) { + let installed: Value = serde_json::from_slice(&std::fs::read(path)?)?; + anyhow::ensure!(installed == expected, + "Installed Braintrust hooks differ from this bt version; update bt and retry setup"); + } + if hook["trustStatus"] == "trusted" { + continue; + } + anyhow::ensure!( + hook["trustStatus"] == "untrusted" || hook["trustStatus"] == "modified", + "Unknown Braintrust hook trust state" + ); + let key = hook["key"] + .as_str() + .filter(|s| !s.is_empty()) + .context("hook key missing")?; + let hash = hook["currentHash"] + .as_str() + .filter(|s| !s.is_empty()) + .context("hook hash missing")?; + anyhow::ensure!( + edits + .insert(key.to_owned(), json!({"trusted_hash":hash})) + .is_none(), + "Duplicate Braintrust hook key" + ); + } + } + anyhow::ensure!( + events.len() == PLUGIN_EVENTS.len(), + "Braintrust lifecycle hooks are missing" + ); + Ok(edits) +} + async fn request( stdin: &mut (impl AsyncWrite + Unpin), stdout: &mut (impl AsyncBufRead + Unpin), @@ -307,12 +438,15 @@ fn classify(hooks: &Value, config: &Value, plugins: &Value) -> anyhow::Result 0 { - (CodexHookState::ReviewRequired, "Open /hooks in Codex and review the current Braintrust hook definitions. Untrusted or modified hooks are skipped before capture.") + (CodexHookState::ReviewRequired, "Run bt trace update codex to activate the current Braintrust hook definitions. Untrusted or modified hooks are skipped before capture.") } else if selected.is_empty() && plugin .is_none_or(|plugin| plugin.get("installed").and_then(Value::as_bool) != Some(true)) { - (CodexHookState::PluginMissing, "Braintrust Codex hooks are not installed. Run bt trace enable codex, then review /hooks in Codex.") + ( + CodexHookState::PluginMissing, + "Braintrust Codex hooks are not installed. Run bt trace enable codex.", + ) } else if !readiness.missing_events.is_empty() { (CodexHookState::MissingHooks, "Some Braintrust lifecycle hooks are missing. Update the plugin or complete the managed hook policy, then inspect /hooks.") } else if readiness.trusted_count == readiness.hook_count { diff --git a/bt-daemon/src/command_output.rs b/bt-daemon/src/command_output.rs index 3a192062..882819cb 100644 --- a/bt-daemon/src/command_output.rs +++ b/bt-daemon/src/command_output.rs @@ -59,8 +59,6 @@ pub struct SetupCommandOutput { pub display_name: String, pub settings_path: PathBuf, pub restart_required: bool, - #[serde(skip_serializing_if = "Vec::is_empty")] - pub next_steps: Vec, } #[derive(Debug, Clone, Serialize)] @@ -68,8 +66,6 @@ pub struct UpdateCommandOutput { pub source: String, pub display_name: String, pub restart_required: bool, - #[serde(skip_serializing_if = "Vec::is_empty")] - pub next_steps: Vec, } #[derive(Debug, Clone, Serialize)] @@ -194,7 +190,6 @@ impl TraceCommandOutput { ) -> Self { let source = source.into(); Self::Enable(SetupCommandOutput { - next_steps: activation_steps(&source), source, display_name: display_name.into(), settings_path: settings_path.into(), @@ -216,7 +211,6 @@ impl TraceCommandOutput { settings_path: impl Into, ) -> Self { Self::Disable(SetupCommandOutput { - next_steps: Vec::new(), source: source.into(), display_name: display_name.into(), settings_path: settings_path.into(), @@ -227,7 +221,6 @@ impl TraceCommandOutput { pub fn update(source: impl Into, display_name: impl Into) -> Self { let source = source.into(); Self::Update(UpdateCommandOutput { - next_steps: activation_steps(&source), source, display_name: display_name.into(), restart_required: true, @@ -307,9 +300,9 @@ impl TraceCommandOutput { Ok(rendered) } Self::Enable(setup) => Ok(format!( - "The Braintrust tracing plugin is installed for {} and configured in {}.\nRestart the coding agent to load the tracing plugin.{}", + "The Braintrust tracing plugin is installed for {} and configured in {}.\nRestart the coding agent to load the tracing plugin.", setup.display_name, - setup.settings_path.display(), render_next_steps(&setup.next_steps) + setup.settings_path.display() )), Self::Disable(disable) => Ok(format!( "The Braintrust tracing plugin and configuration were removed for {} from {}.\nRestart the coding agent to apply the change.", @@ -317,8 +310,8 @@ impl TraceCommandOutput { disable.settings_path.display() )), Self::Update(update) => Ok(format!( - "The Braintrust tracing plugin was updated for {}.\nRestart the coding agent to load the update.{}", - update.display_name, render_next_steps(&update.next_steps), + "The Braintrust tracing plugin was updated for {}.\nRestart the coding agent to load the update.", + update.display_name, )), Self::Stop(stop) if stop.stopped => Ok("Tracing daemon stopped.".into()), Self::Stop(_) => Ok("No tracing daemon is running.".into()), @@ -372,18 +365,6 @@ fn push_auth_lines(rendered: &mut String, prefix: &str, auth: &AuthDiagnostic) { } } -fn activation_steps(source: &str) -> Vec { - if source == "codex" { - vec!["Codex separately requires trust for plugin hooks. Open /hooks in Codex and review the Braintrust definitions, then run bt trace doctor codex. Installation alone does not confirm capture or delivery.".into()] - } else { - Vec::new() - } -} - -fn render_next_steps(steps: &[String]) -> String { - steps.iter().map(|step| format!("\n{step}")).collect() -} - fn render_timestamp(timestamp_ms: i64) -> String { chrono::DateTime::from_timestamp_millis(timestamp_ms) .map(|timestamp| timestamp.to_rfc3339()) @@ -459,17 +440,16 @@ mod tests { } #[test] - fn codex_setup_and_updates_explain_the_separate_hook_review_step() { + fn codex_setup_and_updates_do_not_request_manual_trust() { for output in [ TraceCommandOutput::setup("codex", "Codex", PathBuf::from("braintrust.json")), TraceCommandOutput::update("codex", "Codex"), ] { let human = output.render(OutputFormat::Human).unwrap(); - assert!(human.contains("Open /hooks")); - assert!(human.contains("Installation alone does not confirm capture or delivery")); + assert!(!human.contains("Open /hooks")); let json: serde_json::Value = serde_json::from_str(&output.render(OutputFormat::Json).unwrap()).unwrap(); - assert_eq!(json["next_steps"].as_array().unwrap().len(), 1); + assert!(json.get("next_steps").is_none()); } let disabled = TraceCommandOutput::disable("codex", "Codex", PathBuf::from("braintrust.json")); diff --git a/bt-daemon/src/trace_runtime.rs b/bt-daemon/src/trace_runtime.rs index 5a5fece5..8f55dabe 100644 --- a/bt-daemon/src/trace_runtime.rs +++ b/bt-daemon/src/trace_runtime.rs @@ -6,7 +6,7 @@ //! the coding-agent integrations. use crate::route::{apply_additional_metadata, apply_tags, resolve_span_plugin_paths}; -use crate::trace_command::{DoctorAgent, DoctorArgs, TraceCommand}; +use crate::trace_command::{DoctorAgent, DoctorArgs, SetupAgent, TraceCommand}; use crate::wire::{AuthSelection, AuthSource, SessionConfig, SessionRoute, TraceDestination}; use crate::{ braintrust_serve_options, paths, run_disable, run_enable, run_import, run_serve, run_status, @@ -668,13 +668,23 @@ pub async fn run_trace(args: TraceArgs, host: TraceHostContext) -> anyhow::Resul if !enable_args.plugin.is_empty() { route.span_plugins = resolve_span_plugin_paths(&enable_args.plugin)?; } - print_output(run_enable(enable_args, route)?, host.output_format) + let codex = matches!(enable_args.agent, SetupAgent::Codex); + let output = run_enable(enable_args, route)?; + if codex { + crate::codex_hooks::activate(false).await?; + } + print_output(output, host.output_format) } TraceCommand::Disable(disable_args) => { print_output(run_disable(disable_args.agent)?, host.output_format) } TraceCommand::Update(update_args) => { - print_output(crate::run_update(update_args.agent)?, host.output_format) + let codex = matches!(update_args.agent, SetupAgent::Codex); + let output = crate::run_update(update_args.agent)?; + if codex { + crate::codex_hooks::activate(true).await?; + } + print_output(output, host.output_format) } TraceCommand::Daemon(serve_args) => { init_daemon_logging(host.verbose); @@ -779,7 +789,7 @@ async fn run_hook_command( #[cfg(test)] mod tests { use super::*; - use crate::trace_command::{SetupAgent, SetupArgs, StopArgs}; + use crate::trace_command::{SetupArgs, StopArgs}; use crate::wire::{BackendAuth, TraceDestination}; use crate::{ImportArgs, ImportSource, RunArgs, RunSource, StatusArgs, TraceCommand}; use std::path::PathBuf; diff --git a/docs/codex-managed-tracing.md b/docs/codex-managed-tracing.md index ba5995dd..84016531 100644 --- a/docs/codex-managed-tracing.md +++ b/docs/codex-managed-tracing.md @@ -1,25 +1,38 @@ # Codex hook activation and enterprise deployment -Installing `trace-codex` does not approve its hooks. Codex skips untrusted or -modified plugin hooks before Braintrust receives their events. Restart Codex, -open `/hooks`, review the Braintrust definitions, and run `bt trace doctor codex`. -The doctor probes the installed Codex app server for the current working directory. -It distinguishes trusted, review-required, disabled, missing, erroneous, and -unknown configurations. Trust readiness does not prove capture or delivery. - -## Supported approval mechanisms - -[Official OpenAI hook documentation](https://learn.chatgpt.com/docs/hooks#review-and-trust-hooks) -describes user review through `/hooks` and administrator-managed hooks trusted by -policy. Codex CLI 0.163.0 exposes `hooks/list` with each definition's enabled and -trust state, but its generated app-server protocol has no hook approval method. -Braintrust therefore does not modify Codex's private trust files or approve -unrelated hooks. A release that changes hook definitions can require review again; -route changes and `bt` updates do not change the marketplace definitions. - -The invocation-wide `--dangerously-bypass-hook-trust` flag is not scoped to -Braintrust. `bt trace run codex` continues to reject it. For unattended enterprise -activation, use Codex's supported managed policy rather than that flag. +`bt trace enable codex` and `bt trace update codex` install the plugin and trust +its current Braintrust hook definitions automatically. Restart Codex after setup +so existing sessions load the plugin and trust settings. + +## Scoped approval through Codex + +Codex CLI 0.163.0 exposes the same configuration API used by its `/hooks` UI: +`hooks/list` supplies each definition's key and current hash, and +`config/batchWrite` upserts `hooks.state` entries containing `trusted_hash`. +Braintrust uses that API, reloads configuration, and checks that the hooks are +trusted. It does not calculate Codex hashes or write private trust files. + +Only hooks belonging to `trace-codex@braintrust-codex-plugins` are eligible. Before +writing trust, Braintrust verifies the complete installed hook file against the +definitions bundled with this version of `bt`, including Windows commands, and +checks that Codex reports the expected commands and lifecycle events. Unexpected +definitions fail activation; update `bt` and retry. Unrelated hook trust and +existing configuration are preserved. Already trusted definitions need no write. +Update preserves disabled hooks rather than enabling them. + +An unsupported API, failed write, missing definition, or configuration error makes +setup report an activation failure instead of claiming success. Older Codex +versions can still use manual approval through `/hooks`. After a marketplace +update changes definitions, run `bt trace update codex` to trust the current +Braintrust definitions. The invocation-wide `--dangerously-bypass-hook-trust` +flag is not used; `bt trace run codex` continues to reject it. + +`bt trace doctor codex` remains read-only. It probes the installed Codex app +server for the current directory and distinguishes trusted, review-required, +disabled, missing, erroneous, and unknown configurations. Trust readiness does +not prove capture or delivery. See the +[official OpenAI hook documentation](https://learn.chatgpt.com/docs/hooks#review-and-trust-hooks) +for Codex's trust model. ## Administrator-managed local Codex hooks @@ -140,7 +153,8 @@ directory, then run a short session and verify its matching trace in Braintrust. A running daemon, trusted definitions, or historical traces alone do not confirm that the newly deployed path delivered the session. -For ordinary plugin installs, hooks still need review after definition changes. +For plugin installs performed outside `bt`, approve hooks through `/hooks` or +run `bt trace update codex` after definition changes. Events skipped before approval are not in the daemon's journal; replay cannot recover those events. Historical transcript import may recover supported data, with the limitations documented for that import path. diff --git a/src/plugins/codex/content/plugins/trace-codex/README.md b/src/plugins/codex/content/plugins/trace-codex/README.md index 62441f7a..b032e9b8 100644 --- a/src/plugins/codex/content/plugins/trace-codex/README.md +++ b/src/plugins/codex/content/plugins/trace-codex/README.md @@ -21,22 +21,19 @@ Install and configure the published plugin with the Braintrust CLI: bt trace enable codex --project my-coding-agent ``` -Use `--profile` or `--org` when needed. Restart Codex after setup so it loads -the plugin. Codex will apply its normal hook-review flow; approve the stable -Braintrust hook definition through `/hooks` when prompted. Installing the plugin -alone does not approve hooks: untrusted or modified definitions are skipped -before Braintrust can capture events. - -`bt trace doctor codex` inspects hook trust through the installed Codex app server -for the current directory. It reports missing or disabled hooks, pending review, -and trusted definitions separately from daemon authentication and delivery. -Older Codex versions without this inspection API report an unknown state and -still require inspection through `/hooks`. - -For unattended enterprise deployment, see the +Use `--profile` or `--org` when needed. Setup automatically trusts the installed +Braintrust hook definitions through Codex's configuration API. Restart Codex after +setup so it loads the plugin and trust settings. `bt trace update codex` refreshes +the plugin and trusts its current definitions, preserving unrelated hooks and +explicitly disabled hook settings. + +If activation fails, setup reports the cause. Update Codex and `bt`, then retry; +older Codex versions can use manual approval through `/hooks`. +`bt trace doctor codex` provides a read-only view of hook readiness separately +from daemon authentication and delivery. + +For enterprise policy deployment, see the [administrator-managed hook recipe](https://github.com/braintrustdata/braintrust-coding-agent-plugins/blob/main/docs/codex-managed-tracing.md). -Managed hooks use Codex policy-based trust. Braintrust does not edit Codex's -private trust store or bypass review for other hooks. To inspect tracing configuration and daemon status: From 554cea655801ea950a3746a629f2c049d476681f Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 20:37:41 +0800 Subject: [PATCH 04/10] Align Codex managed runs with packaged hook trust --- bt-daemon/src/codex_hooks.rs | 141 +++++++++++++++++- bt-daemon/src/managed_run/codex.rs | 81 +++++++--- bt-daemon/src/managed_run/mod.rs | 45 +++++- .../content/plugins/trace-codex/README.md | 4 +- .../plugins/trace-codex/hooks/hooks.json | 24 +++ 5 files changed, 265 insertions(+), 30 deletions(-) diff --git a/bt-daemon/src/codex_hooks.rs b/bt-daemon/src/codex_hooks.rs index 659e84c7..c2492490 100644 --- a/bt-daemon/src/codex_hooks.rs +++ b/bt-daemon/src/codex_hooks.rs @@ -28,10 +28,10 @@ const PLUGIN_EVENTS: &[&str] = &[ "subagentStart", "subagentStop", "stop", + "interrupt", + "sessionEnd", ]; -// Managed policy is independent of the marketplace artifact and must observe -// terminal lifecycle records even when no subsequent hook fires. -const MANAGED_EXTRA_EVENTS: &[&str] = &["interrupt", "sessionEnd"]; +const MANAGED_EXTRA_EVENTS: &[&str] = &[]; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] @@ -211,6 +211,141 @@ async fn inspect_at( } } +/// Build a command-line-scoped hooks.state value for Codex managed runs. The +/// keys and hashes come from Codex after it has loaded our exact hook overrides. +pub(crate) async fn run_trust_override( + program: &std::ffi::OsStr, + cwd: &Path, + hook_args: &[std::ffi::OsString], + expected_command: &str, +) -> anyhow::Result { + let mut command: tokio::process::Command = + crate::subprocess::background_command(program).into(); + command + .arg("--no-daemon") + .args(hook_args) + .args(["app-server", "--stdio"]) + .current_dir(cwd) + .kill_on_drop(true) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + let mut child = command + .spawn() + .context("start Codex to prepare managed hook trust")?; + let result = + tokio::time::timeout(TIMEOUT, async { + let mut stdin = child.stdin.take().context("Codex stdin unavailable")?; + let stdout = child.stdout.take().context("Codex stdout unavailable")?; + let mut stdout = BufReader::new(stdout.take(8 * 1024 * 1024)); + request(&mut stdin, &mut stdout, 1, "initialize", json!({ + "clientInfo":{"name":"braintrust_trace_run","version":env!("CARGO_PKG_VERSION")}, + "capabilities":{"experimentalApi":true} + })).await?; + stdin.write_all(b"{\"method\":\"initialized\"}\n").await?; + let hooks = request( + &mut stdin, + &mut stdout, + 2, + "hooks/list", + json!({"cwds":[cwd]}), + ) + .await?; + managed_run_trust_state(&hooks, expected_command) + }) + .await; + let _ = child.kill().await; + match result { + Ok(result) => result, + Err(_) => Err(anyhow!( + "Codex managed hook trust inspection timed out after 5 seconds" + )), + } +} + +fn managed_run_trust_state( + hooks: &Value, + expected_command: &str, +) -> anyhow::Result { + let expected: Value = serde_json::from_str(PACKAGED_HOOKS)?; + let expected_events: std::collections::HashSet = expected["hooks"] + .as_object() + .context("packaged hooks missing")? + .keys() + .map(|event| format!("{}{}", event[..1].to_ascii_lowercase(), &event[1..])) + .collect(); + let entries = hooks["data"] + .as_array() + .context("hooks/list has no data array")?; + let mut seen = std::collections::HashSet::new(); + let mut trust = std::collections::BTreeMap::new(); + for entry in entries { + anyhow::ensure!( + entry["errors"].as_array().is_some_and(Vec::is_empty), + "Codex reported hook errors during managed-run setup" + ); + for hook in entry["hooks"] + .as_array() + .context("hooks/list entry has no hooks array")? + { + if hook["source"] != "sessionFlags" { + continue; + } + let Some(event) = hook["eventName"].as_str() else { + continue; + }; + if !expected_events.contains(event) { + continue; + } + anyhow::ensure!( + seen.insert(event.to_owned()) + && hook["handlerType"] == "command" + && hook["command"] == expected_command + && hook["enabled"] == true + && hook["isManaged"] == false, + "Codex changed a Braintrust hook while preparing this run" + ); + anyhow::ensure!( + matches!( + hook["trustStatus"].as_str(), + Some("trusted" | "untrusted" | "modified") + ), + "Codex reported an unsupported trust state for a managed Braintrust hook" + ); + let key = hook["key"] + .as_str() + .filter(|value| !value.is_empty()) + .context("Codex omitted a managed Braintrust hook key")?; + let hash = hook["currentHash"] + .as_str() + .filter(|value| !value.is_empty()) + .context("Codex omitted a managed Braintrust hook hash")?; + anyhow::ensure!( + trust.insert(key.to_owned(), hash.to_owned()).is_none(), + "duplicate managed Braintrust hook key" + ); + } + } + anyhow::ensure!( + seen == expected_events, + "Codex did not load the complete Braintrust hook set for this run" + ); + let fields = trust + .iter() + .map(|(key, hash)| { + Ok(format!( + "{}={{trusted_hash={}}}", + serde_json::to_string(key)?, + serde_json::to_string(hash)? + )) + }) + .collect::>>()? + .join(","); + Ok(std::ffi::OsString::from(format!( + "hooks.state={{{fields}}}" + ))) +} + /// Validate the complete installed file, including the other platform's command, /// before approving exact hashes returned by Codex. Never infer hashes or keys. fn trust_edits(hooks: &Value) -> anyhow::Result> { diff --git a/bt-daemon/src/managed_run/codex.rs b/bt-daemon/src/managed_run/codex.rs index aeadca29..8c0dab1a 100644 --- a/bt-daemon/src/managed_run/codex.rs +++ b/bt-daemon/src/managed_run/codex.rs @@ -6,36 +6,80 @@ use super::{managed_hook_shell_command, Injection, ManagedRun, Terminal}; use crate::agents::Codex; use crate::args::RunHookCommand; -pub(super) const CODEX_RUN_HOOK_EVENTS: &[&str] = &[ - "SessionStart", - "UserPromptSubmit", - "PreToolUse", - "PermissionRequest", - "PostToolUse", - "PreCompact", - "PostCompact", - "SubagentStart", - "SubagentStop", - "Stop", - "SessionEnd", -]; +pub(super) const PACKAGED_HOOKS: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json" +)); + +pub(super) fn managed_commands(hook_command: &RunHookCommand) -> anyhow::Result<(String, String)> { + Ok(( + managed_hook_shell_command(hook_command, "codex", false)?, + managed_hook_shell_command(hook_command, "codex", true)?, + )) +} fn codex_managed_run_args( unix_command: &str, windows_command: &str, ) -> anyhow::Result> { - let unix_command = serde_json::to_string(unix_command)?; - let windows_command = serde_json::to_string(windows_command)?; + let manifest: serde_json::Value = serde_json::from_str(PACKAGED_HOOKS)?; let mut args = vec![OsString::from("--enable"), OsString::from("hooks")]; - for event in CODEX_RUN_HOOK_EVENTS { + for (event, groups) in manifest["hooks"] + .as_object() + .ok_or_else(|| anyhow::anyhow!("Codex hook manifest has no hooks map"))? + { + let mut groups = groups.clone(); + for group in groups + .as_array_mut() + .ok_or_else(|| anyhow::anyhow!("Codex hook groups must be an array"))? + { + for hook in group["hooks"] + .as_array_mut() + .ok_or_else(|| anyhow::anyhow!("Codex hook group has no hooks array"))? + { + anyhow::ensure!(hook["type"] == "command", "Unsupported Codex hook type"); + hook["command"] = serde_json::Value::String(unix_command.to_owned()); + hook["commandWindows"] = serde_json::Value::String(windows_command.to_owned()); + } + } args.push(OsString::from("-c")); args.push(OsString::from(format!( - "hooks.{event}=[{{hooks=[{{type=\"command\",command={unix_command},commandWindows={windows_command}}}]}}]" + "hooks.{event}={}", + toml_literal(&groups)? ))); } Ok(args) } +fn toml_literal(value: &serde_json::Value) -> anyhow::Result { + Ok(match value { + serde_json::Value::Null => anyhow::bail!("null is unsupported in Codex hook config"), + serde_json::Value::Bool(value) => value.to_string(), + serde_json::Value::Number(value) => value.to_string(), + serde_json::Value::String(value) => serde_json::to_string(value)?, + serde_json::Value::Array(values) => format!( + "[{}]", + values + .iter() + .map(toml_literal) + .collect::>>()? + .join(",") + ), + serde_json::Value::Object(values) => format!( + "{{{}}}", + values + .iter() + .map(|(key, value)| Ok(format!( + "{}={}", + serde_json::to_string(key)?, + toml_literal(value)? + ))) + .collect::>>()? + .join(",") + ), + }) +} + impl ManagedRun for Codex { fn executable(&self) -> (&'static str, &'static str) { ("CODEX_BIN", "codex") @@ -59,8 +103,7 @@ impl ManagedRun for Codex { hook_command: &RunHookCommand, _managed_run_id: &str, ) -> anyhow::Result { - let unix_command = managed_hook_shell_command(hook_command, "codex", false)?; - let windows_command = managed_hook_shell_command(hook_command, "codex", true)?; + let (unix_command, windows_command) = managed_commands(hook_command)?; Ok(Injection { args: codex_managed_run_args(&unix_command, &windows_command)?, ..Injection::default() diff --git a/bt-daemon/src/managed_run/mod.rs b/bt-daemon/src/managed_run/mod.rs index 1f3716f4..f51b9703 100644 --- a/bt-daemon/src/managed_run/mod.rs +++ b/bt-daemon/src/managed_run/mod.rs @@ -12,7 +12,7 @@ use std::io::IsTerminal; use std::path::PathBuf; use crate::agents::{registrar, Agent}; -use crate::args::{RunArgs, RunHookCommand}; +use crate::args::{RunArgs, RunHookCommand, RunSource}; use crate::client::{flush_managed_run_in, shutdown_daemon}; use crate::hook::MANAGED_RUN_ID_ENV; use crate::route::resolve_span_plugin_paths; @@ -88,7 +88,28 @@ pub async fn run_traced( let executable = std::env::var_os(executable_env).unwrap_or_else(|| OsString::from(default_executable)); let managed_run_id = uuid::Uuid::new_v4().to_string(); - let injection = agent.inject(&hook_command, &managed_run_id)?; + let mut injection = agent.inject(&hook_command, &managed_run_id)?; + if args.source == RunSource::Codex { + let executable_for_trust = + std::env::var_os("CODEX_BIN").unwrap_or_else(|| OsString::from("codex")); + let cwd = std::env::current_dir()?; + let (unix_command, windows_command) = + crate::managed_run::codex::managed_commands(&hook_command)?; + let expected_command = if cfg!(windows) { + windows_command + } else { + unix_command + }; + let trust_override = crate::codex_hooks::run_trust_override( + &executable_for_trust, + &cwd, + &injection.args, + &expected_command, + ) + .await?; + injection.args.push(OsString::from("-c")); + injection.args.push(trust_override); + } // A caller-supplied socket is already an explicit daemon boundary (for // example an integration harness or a deliberately isolated runtime). // Only create our own boundary when environment auth would otherwise use @@ -231,7 +252,6 @@ pub(super) fn quote_windows_command_arg(arg: &str) -> String { #[cfg(test)] mod tests { - use super::codex::CODEX_RUN_HOOK_EVENTS; use super::*; use crate::agents::{Codex, Pi}; use crate::args::RunSource; @@ -328,10 +348,21 @@ mod tests { assert!(!args .iter() .any(|arg| arg == "--dangerously-bypass-hook-trust")); - assert_eq!( - args.iter().filter(|arg| *arg == "-c").count(), - CODEX_RUN_HOOK_EVENTS.len() - ); + let hook_overrides: Vec<_> = args + .iter() + .filter_map(|arg| arg.to_str()) + .filter(|arg| arg.starts_with("hooks.")) + .collect(); + let manifest: serde_json::Value = + serde_json::from_str(super::codex::PACKAGED_HOOKS).unwrap(); + let expected_events = manifest["hooks"].as_object().unwrap(); + assert_eq!(hook_overrides.len(), expected_events.len()); + assert!(hook_overrides + .iter() + .any(|arg| arg.starts_with("hooks.Interrupt="))); + assert!(hook_overrides + .iter() + .any(|arg| arg.starts_with("hooks.SessionEnd="))); let config = args .iter() .find_map(|arg| { diff --git a/src/plugins/codex/content/plugins/trace-codex/README.md b/src/plugins/codex/content/plugins/trace-codex/README.md index b032e9b8..285c067f 100644 --- a/src/plugins/codex/content/plugins/trace-codex/README.md +++ b/src/plugins/codex/content/plugins/trace-codex/README.md @@ -22,7 +22,9 @@ bt trace enable codex --project my-coding-agent ``` Use `--profile` or `--org` when needed. Setup automatically trusts the installed -Braintrust hook definitions through Codex's configuration API. Restart Codex after +Braintrust hook definitions through Codex's configuration API. The same packaged +event definitions are used by `bt trace run codex`; run trust is scoped to that +Codex invocation. Restart Codex after setup so it loads the plugin and trust settings. `bt trace update codex` refreshes the plugin and trusts its current definitions, preserving unrelated hooks and explicitly disabled hook settings. diff --git a/src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json b/src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json index 768e5ca5..5359fe17 100644 --- a/src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json +++ b/src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json @@ -119,6 +119,30 @@ } ] } + ], + "Interrupt": [ + { + "hooks": [ + { + "type": "command", + "command": "bt trace hook --source codex", + "commandWindows": "bt trace hook --source codex", + "statusMessage": "Braintrust tracing" + } + ] + } + ], + "SessionEnd": [ + { + "hooks": [ + { + "type": "command", + "command": "bt trace hook --source codex", + "commandWindows": "bt trace hook --source codex", + "statusMessage": "Braintrust tracing" + } + ] + } ] } } From a0abbf7baa726ec5456fac24770c5c6375c8ca0f Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 20:38:19 +0800 Subject: [PATCH 05/10] Document invocation-scoped Codex hook trust --- docs/codex-managed-tracing.md | 7 +++++-- src/plugins/codex/content/plugins/trace-codex/README.md | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/codex-managed-tracing.md b/docs/codex-managed-tracing.md index 84016531..243af16c 100644 --- a/docs/codex-managed-tracing.md +++ b/docs/codex-managed-tracing.md @@ -24,8 +24,11 @@ An unsupported API, failed write, missing definition, or configuration error mak setup report an activation failure instead of claiming success. Older Codex versions can still use manual approval through `/hooks`. After a marketplace update changes definitions, run `bt trace update codex` to trust the current -Braintrust definitions. The invocation-wide `--dangerously-bypass-hook-trust` -flag is not used; `bt trace run codex` continues to reject it. +Braintrust definitions. For `bt trace run codex`, the same packaged definitions and exact Codex keys and +hashes are passed as invocation-only command-line configuration. Run does not +write persistent trust or change the user's global plugin settings. The +invocation-wide `--dangerously-bypass-hook-trust` flag is not used and remains +rejected by `bt trace run codex`. `bt trace doctor codex` remains read-only. It probes the installed Codex app server for the current directory and distinguishes trusted, review-required, diff --git a/src/plugins/codex/content/plugins/trace-codex/README.md b/src/plugins/codex/content/plugins/trace-codex/README.md index 285c067f..885037a6 100644 --- a/src/plugins/codex/content/plugins/trace-codex/README.md +++ b/src/plugins/codex/content/plugins/trace-codex/README.md @@ -24,8 +24,8 @@ bt trace enable codex --project my-coding-agent Use `--profile` or `--org` when needed. Setup automatically trusts the installed Braintrust hook definitions through Codex's configuration API. The same packaged event definitions are used by `bt trace run codex`; run trust is scoped to that -Codex invocation. Restart Codex after -setup so it loads the plugin and trust settings. `bt trace update codex` refreshes +Codex invocation. Restart Codex after setup so it loads the plugin and trust +settings. `bt trace update codex` refreshes the plugin and trusts its current definitions, preserving unrelated hooks and explicitly disabled hook settings. From e4c2591c3f34833cfe6ba19d2d7f591411f002d5 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 21:10:01 +0800 Subject: [PATCH 06/10] Make Codex setup enforce trusted plugin state --- bt-daemon/src/codex_hooks.rs | 677 +++++--------------------------- bt-daemon/src/command_output.rs | 24 -- bt-daemon/src/lib.rs | 1 - bt-daemon/src/setup/codex.rs | 23 +- bt-daemon/src/trace_runtime.rs | 33 +- docs/codex-managed-tracing.md | 27 +- 6 files changed, 131 insertions(+), 654 deletions(-) diff --git a/bt-daemon/src/codex_hooks.rs b/bt-daemon/src/codex_hooks.rs index c2492490..b3a7fd41 100644 --- a/bt-daemon/src/codex_hooks.rs +++ b/bt-daemon/src/codex_hooks.rs @@ -2,7 +2,6 @@ //! Trust writes use the same exact-definition config API as Codex's /hooks UI. use anyhow::{anyhow, Context}; -use serde::Serialize; use serde_json::{json, Value}; use std::path::Path; use std::process::Stdio; @@ -31,183 +30,109 @@ const PLUGIN_EVENTS: &[&str] = &[ "interrupt", "sessionEnd", ]; -const MANAGED_EXTRA_EVENTS: &[&str] = &[]; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum CodexHookState { - Trusted, - ReviewRequired, - Disabled, - PluginMissing, - MissingHooks, - ConfigurationError, - Unknown, -} - -impl std::fmt::Display for CodexHookState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(match self { - Self::Trusted => "trusted", - Self::ReviewRequired => "review required", - Self::Disabled => "disabled", - Self::PluginMissing => "plugin missing", - Self::MissingHooks => "missing hooks", - Self::ConfigurationError => "configuration error", - Self::Unknown => "unknown", - }) - } -} - -#[derive(Debug, Clone, Serialize)] -pub struct CodexHookReadiness { - pub state: CodexHookState, - pub hook_count: usize, - pub trusted_count: usize, - pub disabled_count: usize, - pub review_required_count: usize, - pub missing_events: Vec, - pub detail: String, -} - -impl CodexHookReadiness { - fn unknown(error: impl std::fmt::Display) -> Self { - Self { - state: CodexHookState::Unknown, - hook_count: 0, trusted_count: 0, disabled_count: 0, review_required_count: 0, - missing_events: Vec::new(), - detail: format!("Could not inspect Codex hooks: {error}. Open /hooks in Codex to inspect Braintrust hook trust; older Codex versions may not expose hooks/list."), - } - } -} - -pub(crate) async fn probe() -> CodexHookReadiness { - let cwd = match std::env::current_dir() { - Ok(cwd) => cwd, - Err(error) => return CodexHookReadiness::unknown(error), - }; - probe_at("codex", &cwd, None).await -} - -pub(crate) async fn activate(allow_disabled: bool) -> anyhow::Result<()> { +pub(crate) async fn activate() -> anyhow::Result<()> { let cwd = std::env::current_dir()?; - let readiness = inspect_at("codex", &cwd, None, true).await; - anyhow::ensure!( - readiness.state == CodexHookState::Trusted - || (allow_disabled && readiness.state == CodexHookState::Disabled), - "Codex plugin installed, but hook activation failed: {}", - readiness.detail - ); - Ok(()) -} - -async fn probe_at(program: &str, cwd: &Path, home: Option<&Path>) -> CodexHookReadiness { - inspect_at(program, cwd, home, false).await -} - -async fn inspect_at( - program: &str, - cwd: &Path, - home: Option<&Path>, - approve: bool, -) -> CodexHookReadiness { let mut command: tokio::process::Command = - crate::subprocess::background_command(program).into(); + crate::subprocess::background_command("codex").into(); command .args(["--no-daemon", "app-server", "--stdio"]) - .current_dir(cwd) + .current_dir(&cwd) .kill_on_drop(true) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()); - if let Some(home) = home { - command.env("CODEX_HOME", home); - } - let mut child = match command.spawn() { - Ok(child) => child, - Err(error) => return CodexHookReadiness::unknown(error), - }; - let result = tokio::time::timeout(TIMEOUT, async { - let mut stdin = child.stdin.take().context("Codex stdin unavailable")?; - let stdout = child.stdout.take().context("Codex stdout unavailable")?; - // Bound both total response bytes and wall time, including notifications. - let mut stdout = BufReader::new(stdout.take(8 * 1024 * 1024)); - request( - &mut stdin, - &mut stdout, - 1, - "initialize", - json!({ - "clientInfo":{"name":"braintrust_trace","version":env!("CARGO_PKG_VERSION")}, - "capabilities":{"experimentalApi":true} - }), - ) - .await?; - stdin.write_all(b"{\"method\":\"initialized\"}\n").await?; - let config = request( - &mut stdin, - &mut stdout, - 2, - "config/read", - json!({"includeLayers":false}), - ) - .await?; - let hooks = request( - &mut stdin, - &mut stdout, - 3, - "hooks/list", - json!({"cwds":[cwd]}), - ) - .await?; - let plugins = request( - &mut stdin, - &mut stdout, - 4, - "plugin/list", - json!({ - "cwds":[cwd], "forceRefetch":false, "marketplaceKinds":["local"] - }), - ) - .await?; - let readiness = classify(&hooks, &config, &plugins)?; - if !approve || readiness.state != CodexHookState::ReviewRequired { - return Ok(readiness); - } - anyhow::ensure!( - readiness.missing_events.is_empty(), - "Braintrust lifecycle hooks are missing" - ); - let edits = trust_edits(&hooks)?; - anyhow::ensure!(!edits.is_empty(), "No Braintrust hooks available to trust"); - request( - &mut stdin, - &mut stdout, - 5, - "config/batchWrite", - json!({ - "edits":[{"keyPath":"hooks.state", "value":edits,"mergeStrategy":"upsert"}], - "reloadUserConfig":true - }), - ) - .await?; - let hooks = request( - &mut stdin, - &mut stdout, - 6, - "hooks/list", - json!({"cwds":[cwd]}), - ) - .await?; - classify(&hooks, &config, &plugins) - }) - .await; - // Always reap the probe, even when the protocol is unsupported or stalls. + let mut child = command + .spawn() + .context("start Codex to activate tracing hooks")?; + let result = + tokio::time::timeout(TIMEOUT, async { + let mut stdin = child.stdin.take().context("Codex stdin unavailable")?; + let stdout = child.stdout.take().context("Codex stdout unavailable")?; + let mut stdout = BufReader::new(stdout.take(8 * 1024 * 1024)); + request(&mut stdin, &mut stdout, 1, "initialize", json!({ + "clientInfo":{"name":"braintrust_trace_setup","version":env!("CARGO_PKG_VERSION")}, + "capabilities":{"experimentalApi":true} + })).await?; + stdin.write_all(b"{\"method\":\"initialized\"}\n").await?; + let config = request( + &mut stdin, + &mut stdout, + 2, + "config/read", + json!({"includeLayers":false}), + ) + .await?; + let features = &config["config"]["features"]; + anyhow::ensure!( + features["hooks"] != false && features["codex_hooks"] != false, + "Codex hooks are disabled in config; enable hooks and retry setup" + ); + let plugins = request( + &mut stdin, + &mut stdout, + 3, + "plugin/list", + json!({ + "cwds":[cwd], "forceRefetch":false, "marketplaceKinds":["local"] + }), + ) + .await?; + let installed = plugins["marketplaces"] + .as_array() + .into_iter() + .flatten() + .filter_map(|marketplace| marketplace["plugins"].as_array()) + .flatten() + .any(|plugin| { + plugin["id"] == PLUGIN + && plugin["installed"] == true + && plugin["enabled"] == true + }); + anyhow::ensure!( + installed, + "Codex tracing plugin is not installed and enabled; rerun `bt trace enable codex`" + ); + let hooks = request( + &mut stdin, + &mut stdout, + 4, + "hooks/list", + json!({"cwds":[cwd]}), + ) + .await?; + let edits = trust_edits(&hooks)?; + if !edits.is_empty() { + request( + &mut stdin, + &mut stdout, + 5, + "config/batchWrite", + json!({ + "edits":[{"keyPath":"hooks.state", "value":edits,"mergeStrategy":"upsert"}], + "reloadUserConfig":true + }), + ) + .await?; + } + let hooks = request( + &mut stdin, + &mut stdout, + 6, + "hooks/list", + json!({"cwds":[cwd]}), + ) + .await?; + anyhow::ensure!( + trust_edits(&hooks)?.is_empty(), + "Codex did not retain Braintrust hook trust after setup" + ); + Ok(()) + }) + .await; let _ = child.kill().await; match result { - Ok(Ok(readiness)) => readiness, - Ok(Err(error)) => CodexHookReadiness::unknown(error), - Err(_) => CodexHookReadiness::unknown("probe timed out after 5 seconds"), + Ok(result) => result.context("Codex plugin installed, but hook activation failed"), + Err(_) => Err(anyhow!("Codex hook activation timed out after 5 seconds")), } } @@ -454,281 +379,13 @@ async fn request( } } -fn classify(hooks: &Value, config: &Value, plugins: &Value) -> anyhow::Result { - let entries = hooks - .get("data") - .and_then(Value::as_array) - .context("hooks/list has no data array")?; - let mut selected = Vec::new(); - let mut plugin_hooks = Vec::new(); - let mut managed_hooks = Vec::new(); - let mut has_errors = false; - for entry in entries { - has_errors |= entry - .get("errors") - .and_then(Value::as_array) - .is_none_or(|errors| !errors.is_empty()); - for hook in entry - .get("hooks") - .and_then(Value::as_array) - .context("hooks/list entry has no hooks array")? - { - let is_plugin = hook.get("pluginId").and_then(Value::as_str) == Some(PLUGIN); - let is_managed = hook.get("isManaged").and_then(Value::as_bool) == Some(true) - && hook - .get("command") - .and_then(Value::as_str) - .is_some_and(|command| command.ends_with(" trace hook --source codex")); - if is_plugin { - plugin_hooks.push(hook); - } else if is_managed { - managed_hooks.push(hook); - } - if is_plugin || is_managed { - selected.push(hook); - } - } - } - let mut readiness = CodexHookReadiness { - state: CodexHookState::Unknown, - hook_count: selected.len(), - trusted_count: 0, - disabled_count: 0, - review_required_count: 0, - missing_events: PLUGIN_EVENTS - .iter() - .filter(|event| { - // Check each source separately: a hook from one source cannot - // complete a partial definition in another source. - let has_event = |hooks: &[&Value]| { - hooks - .iter() - .any(|hook| hook.get("eventName").and_then(Value::as_str) == Some(**event)) - }; - selected.is_empty() - || (!plugin_hooks.is_empty() && !has_event(&plugin_hooks)) - || (!managed_hooks.is_empty() && !has_event(&managed_hooks)) - }) - .chain(MANAGED_EXTRA_EVENTS.iter().filter(|event| { - !managed_hooks.is_empty() - && !managed_hooks - .iter() - .any(|hook| hook.get("eventName").and_then(Value::as_str) == Some(**event)) - })) - .map(|event| (*event).to_owned()) - .collect(), - detail: String::new(), - }; - for hook in &selected { - anyhow::ensure!( - hook.get("enabled").and_then(Value::as_bool).is_some(), - "hook has no enabled state" - ); - if hook.get("enabled").and_then(Value::as_bool) == Some(false) { - readiness.disabled_count += 1; - } - match hook.get("trustStatus").and_then(Value::as_str) { - Some("trusted" | "managed") => readiness.trusted_count += 1, - Some("untrusted" | "modified") => readiness.review_required_count += 1, - _ => {} - } - } - let marketplaces = plugins - .get("marketplaces") - .and_then(Value::as_array) - .context("plugin/list has no marketplaces array")?; - let plugin = marketplaces - .iter() - .filter_map(|marketplace| marketplace.get("plugins").and_then(Value::as_array)) - .flatten() - .find(|plugin| plugin.get("id").and_then(Value::as_str) == Some(PLUGIN)); - let config = config - .get("config") - .filter(|value| value.is_object()) - .context("config/read has no config object")?; - let features = &config["features"]; - let plugin_capture = selected.is_empty() - || selected - .iter() - .any(|hook| hook.get("pluginId").and_then(Value::as_str) == Some(PLUGIN)); - let disabled = features - .get("hooks") - .or_else(|| features.get("codex_hooks")) - .and_then(Value::as_bool) - == Some(false) - || plugin_capture - && plugin.is_some_and(|plugin| { - plugin.get("enabled").and_then(Value::as_bool) == Some(false) - }); - let enabled = |hooks: &[&Value]| { - hooks - .iter() - .any(|hook| hook.get("enabled").and_then(Value::as_bool) == Some(true)) - }; - let duplicate_capture = enabled(&plugin_hooks) && enabled(&managed_hooks); - let (state, detail) = if duplicate_capture { - (CodexHookState::ConfigurationError, "Both plugin and managed Braintrust hooks are enabled, which can forward lifecycle events twice. Remove trace-codex@braintrust-codex-plugins from Codex when using managed capture, or remove the managed Braintrust policy when using the plugin.") - } else if disabled || readiness.disabled_count > 0 { - (CodexHookState::Disabled, "Codex hooks or the Braintrust plugin are disabled. Enable them in Codex and inspect /hooks.") - } else if has_errors { - (CodexHookState::ConfigurationError, "Codex reported hook configuration errors. Inspect /hooks and Codex startup diagnostics.") - } else if readiness.review_required_count > 0 { - (CodexHookState::ReviewRequired, "Run bt trace update codex to activate the current Braintrust hook definitions. Untrusted or modified hooks are skipped before capture.") - } else if selected.is_empty() - && plugin - .is_none_or(|plugin| plugin.get("installed").and_then(Value::as_bool) != Some(true)) - { - ( - CodexHookState::PluginMissing, - "Braintrust Codex hooks are not installed. Run bt trace enable codex.", - ) - } else if !readiness.missing_events.is_empty() { - (CodexHookState::MissingHooks, "Some Braintrust lifecycle hooks are missing. Update the plugin or complete the managed hook policy, then inspect /hooks.") - } else if readiness.trusted_count == readiness.hook_count { - (CodexHookState::Trusted, "Braintrust hook definitions are enabled and trusted for this working directory. This does not confirm hook execution or Braintrust delivery.") - } else { - ( - CodexHookState::Unknown, - "Codex reported an unknown hook trust state. Inspect /hooks before relying on capture.", - ) - }; - readiness.state = state; - readiness.detail = detail.into(); - Ok(readiness) -} - #[cfg(test)] mod tests { use super::*; - fn snapshot(trust: &str) -> Value { - json!({"data":[{"errors":[],"hooks":PLUGIN_EVENTS.iter().map(|event|json!({ - "eventName":event,"pluginId":PLUGIN,"enabled":true,"trustStatus":trust - })).collect::>()}]}) - } - fn installed() -> Value { - json!({"marketplaces":[{"plugins":[{"id":PLUGIN,"installed":true,"enabled":true}]}]}) - } - - fn managed_snapshot() -> Value { - let hooks: Vec<_> = PLUGIN_EVENTS - .iter() - .chain(MANAGED_EXTRA_EVENTS) - .map(|event| { - json!({ - "eventName":event,"pluginId":null,"enabled":true,"trustStatus":"managed", - "isManaged":true,"command":"/enterprise/bt trace hook --source codex" - }) - }) - .collect(); - json!({"data":[{"errors":[],"hooks":hooks}]}) - } - - #[test] - fn managed_capture_requires_interrupt_and_session_end_without_changing_plugin_requirements() { - let config = json!({"config":{}}); - let plugins = json!({"marketplaces":[]}); - for missing in MANAGED_EXTRA_EVENTS { - let mut managed = managed_snapshot(); - managed["data"][0]["hooks"] - .as_array_mut() - .unwrap() - .retain(|hook| hook["eventName"] != *missing); - let result = classify(&managed, &config, &plugins).unwrap(); - assert_eq!(result.state, CodexHookState::MissingHooks); - assert_eq!(result.missing_events, [*missing]); - } - let plugin = classify(&snapshot("trusted"), &config, &installed()).unwrap(); - assert_eq!(plugin.state, CodexHookState::Trusted); - assert!(plugin.missing_events.is_empty()); - let managed = classify(&managed_snapshot(), &config, &plugins).unwrap(); - assert_eq!(managed.state, CodexHookState::Trusted); - assert!(managed.missing_events.is_empty()); - - let policy = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../docs/codex-managed-tracing.md" - )); - for event in PLUGIN_EVENTS.iter().chain(MANAGED_EXTRA_EVENTS) { - let event = format!("{}{}", event[..1].to_ascii_uppercase(), &event[1..]); - assert!( - policy.contains(&format!("[[hooks.{event}.hooks]]")), - "policy omits {event}" - ); - } - } - - #[test] - fn simultaneous_enabled_plugin_and_managed_capture_is_a_configuration_error() { - let config = json!({"config":{}}); - let mut mixed = snapshot("trusted"); - mixed["data"][0]["hooks"].as_array_mut().unwrap().extend( - managed_snapshot()["data"][0]["hooks"] - .as_array() - .unwrap() - .iter() - .cloned(), - ); - let result = classify(&mixed, &config, &installed()).unwrap(); - assert_eq!(result.state, CodexHookState::ConfigurationError); - assert!(result.detail.contains("events twice")); - - // A plugin hook cannot fill a missing terminal event in managed policy. - mixed["data"][0]["hooks"] - .as_array_mut() - .unwrap() - .retain(|hook| hook["eventName"] != "interrupt"); - let mut plugin_interrupt = snapshot("trusted")["data"][0]["hooks"][0].clone(); - plugin_interrupt["eventName"] = json!("interrupt"); - mixed["data"][0]["hooks"] - .as_array_mut() - .unwrap() - .push(plugin_interrupt); - assert_eq!( - classify(&mixed, &config, &installed()) - .unwrap() - .missing_events, - ["interrupt"] - ); - - let mut unrelated = snapshot("trusted"); - let mut managed = managed_snapshot(); - for hook in managed["data"][0]["hooks"].as_array_mut().unwrap() { - hook["command"] = json!("/enterprise/bt trace hook --source claude-code"); - } - unrelated["data"][0]["hooks"] - .as_array_mut() - .unwrap() - .extend( - managed["data"][0]["hooks"] - .as_array() - .unwrap() - .iter() - .cloned(), - ); - assert_eq!( - classify(&unrelated, &config, &installed()).unwrap().state, - CodexHookState::Trusted - ); - - for hook in mixed["data"][0]["hooks"].as_array_mut().unwrap() { - if hook["isManaged"] == true { - hook["enabled"] = json!(false); - } - } - assert_eq!( - classify(&mixed, &config, &installed()).unwrap().state, - CodexHookState::Disabled - ); - } - #[test] fn required_events_match_the_packaged_plugin() { - let hooks: Value = serde_json::from_str(include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../src/plugins/codex/content/plugins/trace-codex/hooks/hooks.json" - ))) - .unwrap(); + let hooks: Value = serde_json::from_str(PACKAGED_HOOKS).unwrap(); let events: std::collections::BTreeSet<_> = hooks["hooks"] .as_object() .unwrap() @@ -744,80 +401,6 @@ mod tests { ); } - #[test] - fn classifies_trust_modifications_missing_disabled_and_managed_hooks() { - let config = json!({"config":{"features":{"hooks":true}}}); - for (trust, state) in [ - ("trusted", CodexHookState::Trusted), - ("untrusted", CodexHookState::ReviewRequired), - ("modified", CodexHookState::ReviewRequired), - ("future", CodexHookState::Unknown), - ] { - assert_eq!( - classify(&snapshot(trust), &config, &installed()) - .unwrap() - .state, - state - ); - } - let mut hooks = snapshot("trusted"); - hooks["data"][0]["hooks"][0]["enabled"] = json!(false); - assert_eq!( - classify(&hooks, &config, &installed()).unwrap().state, - CodexHookState::Disabled - ); - let missing = json!({"data":[{"errors":[],"hooks":[]}]}); - assert_eq!( - classify(&missing, &config, &installed()).unwrap().state, - CodexHookState::MissingHooks - ); - assert_eq!( - classify(&missing, &config, &json!({"marketplaces":[]})) - .unwrap() - .state, - CodexHookState::PluginMissing - ); - assert_eq!( - classify( - &snapshot("trusted"), - &json!({"config":{"features":{"hooks":false}}}), - &installed() - ) - .unwrap() - .state, - CodexHookState::Disabled - ); - let managed = managed_snapshot(); - assert_eq!( - classify(&managed, &config, &json!({"marketplaces":[]})) - .unwrap() - .state, - CodexHookState::Trusted - ); - } - - #[test] - fn unrelated_plugins_and_invalid_responses_never_establish_readiness() { - let mut hooks = snapshot("trusted"); - for hook in hooks["data"][0]["hooks"].as_array_mut().unwrap() { - hook["pluginId"] = json!("other@marketplace"); - } - assert_ne!( - classify(&hooks, &json!({"config":{}}), &installed()) - .unwrap() - .state, - CodexHookState::Trusted - ); - assert!(classify(&json!({}), &json!({}), &installed()).is_err()); - hooks["data"][0]["errors"] = json!([{"message":"bad hooks"}]); - assert_eq!( - classify(&hooks, &json!({"config":{}}), &installed()) - .unwrap() - .state, - CodexHookState::ConfigurationError - ); - } - #[tokio::test] async fn rpc_ignores_notifications_and_reports_unsupported_methods() { let mut input = Vec::new(); @@ -833,76 +416,4 @@ mod tests { "hooks/list" ); } - - #[tokio::test] - async fn unavailable_codex_and_closed_protocol_report_unknown() { - let tmp = tempfile::tempdir().unwrap(); - let result = probe_at( - tmp.path().join("missing-codex").to_str().unwrap(), - tmp.path(), - None, - ) - .await; - assert_eq!(result.state, CodexHookState::Unknown); - let mut input = Vec::new(); - let mut output = std::io::Cursor::new(b""); - assert!(request(&mut input, &mut output, 1, "initialize", json!({})) - .await - .unwrap_err() - .to_string() - .contains("closed")); - } - - #[tokio::test] - #[ignore = "Requires an installed Codex CLI; installs only into an isolated temporary home"] - async fn installed_codex_distinguishes_missing_and_unreviewed_hooks() { - let tmp = tempfile::tempdir().unwrap(); - let readiness = probe_at("codex", tmp.path(), Some(tmp.path())).await; - assert_eq!( - readiness.state, - CodexHookState::PluginMissing, - "{readiness:?}" - ); - let marketplace = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../src/plugins/codex/content") - .canonicalize() - .unwrap(); - let marketplace = marketplace.to_string_lossy(); - for args in [ - vec![ - "plugin", - "marketplace", - "add", - marketplace.as_ref(), - "--json", - ], - vec!["plugin", "add", PLUGIN, "--json"], - vec!["plugin", "add", PLUGIN, "--json"], - ] { - let mut command: tokio::process::Command = - crate::subprocess::background_command("codex").into(); - let output = command - .args(["--no-daemon"]) - .args(args) - .env("CODEX_HOME", tmp.path()) - .current_dir(tmp.path()) - .kill_on_drop(true) - .output() - .await - .unwrap(); - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - } - let readiness = probe_at("codex", tmp.path(), Some(tmp.path())).await; - assert_eq!( - readiness.state, - CodexHookState::ReviewRequired, - "{readiness:?}" - ); - assert_eq!(readiness.review_required_count, PLUGIN_EVENTS.len()); - assert!(readiness.missing_events.is_empty()); - } } diff --git a/bt-daemon/src/command_output.rs b/bt-daemon/src/command_output.rs index 882819cb..00b57da1 100644 --- a/bt-daemon/src/command_output.rs +++ b/bt-daemon/src/command_output.rs @@ -158,8 +158,6 @@ pub struct DoctorCommandOutput { pub daemon: DaemonDiagnostic, pub warnings: Vec, pub plugin_diagnostics: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub codex_hooks: Option, } #[derive(Debug, Clone, Serialize)] @@ -278,14 +276,6 @@ impl TraceCommandOutput { for warning in &doctor.warnings { rendered.push_str(&format!("\nWarning: {warning}")); } - if let Some(hooks) = &doctor.codex_hooks { - rendered.push_str(&format!("\nCodex hook trust: {}\n{}\nHooks: {} loaded, {} trusted, {} require review, {} disabled", - hooks.state, hooks.detail, hooks.hook_count, hooks.trusted_count, - hooks.review_required_count, hooks.disabled_count)); - if !hooks.missing_events.is_empty() { - rendered.push_str(&format!("\nMissing hook events: {}", hooks.missing_events.join(", "))); - } - } for diagnostic in &doctor.plugin_diagnostics { rendered.push_str(&format!( "\nPlugin error: {} ({} occurrence{})\nFirst seen: {}\nLast seen: {}\n{}", @@ -567,15 +557,6 @@ mod tests { error: None, }, warnings: Vec::new(), - codex_hooks: Some(crate::CodexHookReadiness { - state: crate::CodexHookState::ReviewRequired, - hook_count: 10, - trusted_count: 0, - disabled_count: 0, - review_required_count: 10, - missing_events: Vec::new(), - detail: "Open /hooks to review Braintrust definitions".into(), - }), plugin_diagnostics: vec![crate::PluginDiagnostic { source: "codex".into(), plugin_path: PathBuf::from("/tmp/redact.mjs"), @@ -592,11 +573,6 @@ mod tests { assert_eq!(value["auth"]["source"], "saved_profile"); assert_eq!(value["daemon"]["status"], "running"); assert_eq!(value["daemon"]["auth"]["status"], "error"); - assert_eq!(value["codex_hooks"]["state"], "review_required"); - assert!(output - .render(OutputFormat::Human) - .unwrap() - .contains("10 require review")); assert_eq!( value["daemon"]["session_errors"][0], "could not resolve Braintrust auth for codex" diff --git a/bt-daemon/src/lib.rs b/bt-daemon/src/lib.rs index 9e1e5470..a532d1b1 100644 --- a/bt-daemon/src/lib.rs +++ b/bt-daemon/src/lib.rs @@ -52,7 +52,6 @@ pub use args::{ pub use client::{ flush_managed_run, flush_session, forward_envelope, run_status, shutdown_daemon, HostInfo, }; -pub use codex_hooks::{CodexHookReadiness, CodexHookState}; pub use command_output::{ AuthDiagnostic, DaemonDiagnostic, DaemonStatus, DoctorCommandOutput, ImportSummary, OutputFormat, SetupCommandOutput, StatusCommandOutput, StopCommandOutput, TraceCommandOutput, diff --git a/bt-daemon/src/setup/codex.rs b/bt-daemon/src/setup/codex.rs index 1dcfe1df..f963b6f1 100644 --- a/bt-daemon/src/setup/codex.rs +++ b/bt-daemon/src/setup/codex.rs @@ -5,7 +5,6 @@ use super::common::{ }; use crate::agents::Codex; use crate::setup::Setup; -use anyhow::bail; use serde_json::Value; const MARKETPLACE_NAME: &str = "braintrust-codex-plugins"; @@ -63,7 +62,9 @@ impl Setup for Codex { fn update(&self, runner: &mut dyn CommandRunner) -> anyhow::Result<()> { if installed_plugin(&list_plugins(runner)?).is_none() { - bail!("Codex tracing plugin is not installed; run `bt trace enable codex`"); + // Setup owns the desired state: repair a removed plugin the same + // way enable does, then let the caller reapply and verify trust. + return self.enable(runner); } MARKETPLACE.refresh_published(runner, "Codex", "codex")?; runner.run("codex", &["plugin", "add", PLUGIN]) @@ -138,14 +139,18 @@ mod tests { } #[test] - fn update_refuses_to_install_a_missing_plugin() { - let mut runner = FakeRunner::new([serde_json::json!({"installed": []})]); + fn update_repairs_a_missing_plugin() { + let mut runner = FakeRunner::new([ + serde_json::json!({"installed": []}), + serde_json::json!({"marketplaces": []}), + ]); - assert!(Codex.update(&mut runner).is_err()); - assert!(!runner - .calls - .iter() - .any(|call| call.starts_with("codex plugin add "))); + Codex.update(&mut runner).unwrap(); + + assert!( + runner.called("codex plugin marketplace add braintrustdata/braintrust-codex-plugin") + ); + assert!(runner.called("codex plugin add trace-codex@braintrust-codex-plugins")); } #[test] diff --git a/bt-daemon/src/trace_runtime.rs b/bt-daemon/src/trace_runtime.rs index 8f55dabe..a3cc1c63 100644 --- a/bt-daemon/src/trace_runtime.rs +++ b/bt-daemon/src/trace_runtime.rs @@ -497,26 +497,14 @@ async fn doctor_output(host: &TraceHostContext, args: DoctorArgs) -> DoctorComma let source = args.agent.identity().id; let settings = crate::settings::AgentSettings::load(source); let socket = paths::socket_path(None); - let inspect_hooks = matches!(args.agent, DoctorAgent::Codex) && settings.tracing_enabled(); - let hooks = async { - if inspect_hooks { - Some(crate::codex_hooks::probe().await) - } else { - None - } - }; - let (mut output, hooks) = tokio::join!( - doctor_output_at( - host, - args, - &socket, - paths::agent_settings_path(source, None), - settings, - ), - hooks - ); - output.codex_hooks = hooks; - output + doctor_output_at( + host, + args, + &socket, + paths::agent_settings_path(source, None), + settings, + ) + .await } async fn doctor_output_at( @@ -646,7 +634,6 @@ async fn doctor_output_at( daemon, warnings, plugin_diagnostics, - codex_hooks: None, } } @@ -671,7 +658,7 @@ pub async fn run_trace(args: TraceArgs, host: TraceHostContext) -> anyhow::Resul let codex = matches!(enable_args.agent, SetupAgent::Codex); let output = run_enable(enable_args, route)?; if codex { - crate::codex_hooks::activate(false).await?; + crate::codex_hooks::activate().await?; } print_output(output, host.output_format) } @@ -682,7 +669,7 @@ pub async fn run_trace(args: TraceArgs, host: TraceHostContext) -> anyhow::Resul let codex = matches!(update_args.agent, SetupAgent::Codex); let output = crate::run_update(update_args.agent)?; if codex { - crate::codex_hooks::activate(true).await?; + crate::codex_hooks::activate().await?; } print_output(output, host.output_format) } diff --git a/docs/codex-managed-tracing.md b/docs/codex-managed-tracing.md index 243af16c..4463795a 100644 --- a/docs/codex-managed-tracing.md +++ b/docs/codex-managed-tracing.md @@ -18,7 +18,10 @@ definitions bundled with this version of `bt`, including Windows commands, and checks that Codex reports the expected commands and lifecycle events. Unexpected definitions fail activation; update `bt` and retry. Unrelated hook trust and existing configuration are preserved. Already trusted definitions need no write. -Update preserves disabled hooks rather than enabling them. +Setup repairs a removed plugin by installing it again. If a user or Codex +changes hook definitions or trust, setup reapplies trust to the exact packaged +definitions and verifies the result. Codex-level hook disablement prevents +activation and is reported as a setup error. An unsupported API, failed write, missing definition, or configuration error makes setup report an activation failure instead of claiming success. Older Codex @@ -30,12 +33,11 @@ write persistent trust or change the user's global plugin settings. The invocation-wide `--dangerously-bypass-hook-trust` flag is not used and remains rejected by `bt trace run codex`. -`bt trace doctor codex` remains read-only. It probes the installed Codex app -server for the current directory and distinguishes trusted, review-required, -disabled, missing, erroneous, and unknown configurations. Trust readiness does -not prove capture or delivery. See the -[official OpenAI hook documentation](https://learn.chatgpt.com/docs/hooks#review-and-trust-hooks) -for Codex's trust model. +`bt trace doctor codex` reports tracing route, authentication, daemon, and +plugin failures. Setup owns plugin installation and hook trust, so doctor does +not duplicate that status. See the [official OpenAI hook +documentation](https://learn.chatgpt.com/docs/hooks#review-and-trust-hooks) for +Codex's trust model. ## Administrator-managed local Codex hooks @@ -142,17 +144,14 @@ Keep all lifecycle events so sessions, turns, tools, and subagents retain their normal trace structure. Codex does not distribute the executable itself. The managed policy must include `Interrupt` to observe cancellation records even when the session stays idle, and `SessionEnd` to observe session termination. -Doctor checks these managed-only requirements separately from the packaged -plugin's event set. It reports a configuration error if plugin and managed -Braintrust capture are both enabled, including after a later `bt trace enable -codex` reinstalls the plugin. Keep only one capture source active. +Keep only one capture source active; enabling the user plugin while this +managed policy is active can forward lifecycle events twice. The policy above enables hooks without disabling other hook sources; an organization may separately choose `allow_managed_hooks_only` if that is its intended policy. -After deployment, restart Codex and inspect `/hooks`: Braintrust definitions -should be managed and enabled. Run `bt trace doctor codex` from the actual project -directory, then run a short session and verify its matching trace in Braintrust. +After deployment, restart Codex and run a short session, then verify its +matching trace in Braintrust. A running daemon, trusted definitions, or historical traces alone do not confirm that the newly deployed path delivered the session. From 17c4a32dd14b4c207581b9d47873ad66822a3e59 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 21:30:31 +0800 Subject: [PATCH 07/10] Correct Codex plugin setup documentation --- src/plugins/codex/content/plugins/trace-codex/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/plugins/codex/content/plugins/trace-codex/README.md b/src/plugins/codex/content/plugins/trace-codex/README.md index 885037a6..b2985b7b 100644 --- a/src/plugins/codex/content/plugins/trace-codex/README.md +++ b/src/plugins/codex/content/plugins/trace-codex/README.md @@ -27,12 +27,13 @@ event definitions are used by `bt trace run codex`; run trust is scoped to that Codex invocation. Restart Codex after setup so it loads the plugin and trust settings. `bt trace update codex` refreshes the plugin and trusts its current definitions, preserving unrelated hooks and -explicitly disabled hook settings. +unrelated trust settings. Setup reports an error if Codex hooks or a Braintrust +hook definition are disabled. If activation fails, setup reports the cause. Update Codex and `bt`, then retry; older Codex versions can use manual approval through `/hooks`. -`bt trace doctor codex` provides a read-only view of hook readiness separately -from daemon authentication and delivery. +`bt trace doctor codex` reports route, authentication, daemon, and plugin +diagnostics; setup owns plugin installation and hook trust. For enterprise policy deployment, see the [administrator-managed hook recipe](https://github.com/braintrustdata/braintrust-coding-agent-plugins/blob/main/docs/codex-managed-tracing.md). From 9fc3a2f94548c8cf173c8e4153a61cac6aa17643 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 21:42:16 +0800 Subject: [PATCH 08/10] Remove unrequested Codex enterprise hook guide --- docs/codex-managed-tracing.md | 162 ------------------ .../content/plugins/trace-codex/README.md | 3 - 2 files changed, 165 deletions(-) delete mode 100644 docs/codex-managed-tracing.md diff --git a/docs/codex-managed-tracing.md b/docs/codex-managed-tracing.md deleted file mode 100644 index 4463795a..00000000 --- a/docs/codex-managed-tracing.md +++ /dev/null @@ -1,162 +0,0 @@ -# Codex hook activation and enterprise deployment - -`bt trace enable codex` and `bt trace update codex` install the plugin and trust -its current Braintrust hook definitions automatically. Restart Codex after setup -so existing sessions load the plugin and trust settings. - -## Scoped approval through Codex - -Codex CLI 0.163.0 exposes the same configuration API used by its `/hooks` UI: -`hooks/list` supplies each definition's key and current hash, and -`config/batchWrite` upserts `hooks.state` entries containing `trusted_hash`. -Braintrust uses that API, reloads configuration, and checks that the hooks are -trusted. It does not calculate Codex hashes or write private trust files. - -Only hooks belonging to `trace-codex@braintrust-codex-plugins` are eligible. Before -writing trust, Braintrust verifies the complete installed hook file against the -definitions bundled with this version of `bt`, including Windows commands, and -checks that Codex reports the expected commands and lifecycle events. Unexpected -definitions fail activation; update `bt` and retry. Unrelated hook trust and -existing configuration are preserved. Already trusted definitions need no write. -Setup repairs a removed plugin by installing it again. If a user or Codex -changes hook definitions or trust, setup reapplies trust to the exact packaged -definitions and verifies the result. Codex-level hook disablement prevents -activation and is reported as a setup error. - -An unsupported API, failed write, missing definition, or configuration error makes -setup report an activation failure instead of claiming success. Older Codex -versions can still use manual approval through `/hooks`. After a marketplace -update changes definitions, run `bt trace update codex` to trust the current -Braintrust definitions. For `bt trace run codex`, the same packaged definitions and exact Codex keys and -hashes are passed as invocation-only command-line configuration. Run does not -write persistent trust or change the user's global plugin settings. The -invocation-wide `--dangerously-bypass-hook-trust` flag is not used and remains -rejected by `bt trace run codex`. - -`bt trace doctor codex` reports tracing route, authentication, daemon, and -plugin failures. Setup owns plugin installation and hook trust, so doctor does -not duplicate that status. See the [official OpenAI hook -documentation](https://learn.chatgpt.com/docs/hooks#review-and-trust-hooks) for -Codex's trust model. - -## Administrator-managed local Codex hooks - -This recipe targets local Codex execution. Administrators own policy deployment; -`bt trace enable` does not write managed policy. Check the installed Codex version -against [OpenAI's managed hook documentation](https://learn.chatgpt.com/docs/hooks#managed-hooks-from-requirementstoml). - -1. Distribute a current `bt` executable to the managed directory on each platform. -2. Have each user authenticate and configure their route with - `bt trace enable codex --project PROJECT`. Credentials stay in the CLI's - authentication store; the route stays in the user's `braintrust.json`. -3. Remove the user plugin with - `codex plugin remove trace-codex@braintrust-codex-plugins` after configuring - the route. This avoids duplicate plugin and managed capture; this Codex - command leaves Braintrust's route settings intact. -4. Merge the following into the organization's Codex `requirements.toml` using - its normal device-management mechanism. Preserve unrelated requirements. - Replace the executable directories with administrator-controlled locations. - -```toml -[features] -hooks = true - -[hooks] -managed_dir = "/opt/company/braintrust" -windows_managed_dir = 'C:\Braintrust' - -[[hooks.SessionStart]] -[[hooks.SessionStart.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.UserPromptSubmit]] -[[hooks.UserPromptSubmit.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.PreToolUse]] -[[hooks.PreToolUse.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.PermissionRequest]] -[[hooks.PermissionRequest.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.PostToolUse]] -[[hooks.PostToolUse.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.PreCompact]] -[[hooks.PreCompact.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.PostCompact]] -[[hooks.PostCompact.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.SubagentStart]] -[[hooks.SubagentStart.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.SubagentStop]] -[[hooks.SubagentStop.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.Stop]] -[[hooks.Stop.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' - -[[hooks.SessionEnd]] -[[hooks.SessionEnd.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' -timeout = 3 - -[[hooks.Interrupt]] -[[hooks.Interrupt.hooks]] -type = "command" -command = "/opt/company/braintrust/bt trace hook --source codex" -command_windows = 'C:\Braintrust\bt.exe trace hook --source codex' -timeout = 3 -``` - -Keep all lifecycle events so sessions, turns, tools, and subagents retain their -normal trace structure. Codex does not distribute the executable itself. The -managed policy must include `Interrupt` to observe cancellation records even -when the session stays idle, and `SessionEnd` to observe session termination. -Keep only one capture source active; enabling the user plugin while this -managed policy is active can forward lifecycle events twice. - -The policy above enables hooks without disabling other hook sources; an organization -may separately choose `allow_managed_hooks_only` if that is its intended policy. - -After deployment, restart Codex and run a short session, then verify its -matching trace in Braintrust. -A running daemon, trusted definitions, or historical traces alone do not confirm -that the newly deployed path delivered the session. - -For plugin installs performed outside `bt`, approve hooks through `/hooks` or -run `bt trace update codex` after definition changes. -Events skipped before approval are not in the daemon's journal; replay cannot -recover those events. Historical transcript import may recover supported data, -with the limitations documented for that import path. diff --git a/src/plugins/codex/content/plugins/trace-codex/README.md b/src/plugins/codex/content/plugins/trace-codex/README.md index b2985b7b..6c1faaf5 100644 --- a/src/plugins/codex/content/plugins/trace-codex/README.md +++ b/src/plugins/codex/content/plugins/trace-codex/README.md @@ -35,9 +35,6 @@ older Codex versions can use manual approval through `/hooks`. `bt trace doctor codex` reports route, authentication, daemon, and plugin diagnostics; setup owns plugin installation and hook trust. -For enterprise policy deployment, see the -[administrator-managed hook recipe](https://github.com/braintrustdata/braintrust-coding-agent-plugins/blob/main/docs/codex-managed-tracing.md). - To inspect tracing configuration and daemon status: ```bash From e87f7369e0e0e2e2d6fe201333370010c2ee9377 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 21:57:03 +0800 Subject: [PATCH 09/10] Simplify Codex tracing plugin README --- .../content/plugins/trace-codex/README.md | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/plugins/codex/content/plugins/trace-codex/README.md b/src/plugins/codex/content/plugins/trace-codex/README.md index 6c1faaf5..389c3333 100644 --- a/src/plugins/codex/content/plugins/trace-codex/README.md +++ b/src/plugins/codex/content/plugins/trace-codex/README.md @@ -21,19 +21,13 @@ Install and configure the published plugin with the Braintrust CLI: bt trace enable codex --project my-coding-agent ``` -Use `--profile` or `--org` when needed. Setup automatically trusts the installed -Braintrust hook definitions through Codex's configuration API. The same packaged -event definitions are used by `bt trace run codex`; run trust is scoped to that -Codex invocation. Restart Codex after setup so it loads the plugin and trust -settings. `bt trace update codex` refreshes -the plugin and trusts its current definitions, preserving unrelated hooks and -unrelated trust settings. Setup reports an error if Codex hooks or a Braintrust -hook definition are disabled. - -If activation fails, setup reports the cause. Update Codex and `bt`, then retry; -older Codex versions can use manual approval through `/hooks`. -`bt trace doctor codex` reports route, authentication, daemon, and plugin -diagnostics; setup owns plugin installation and hook trust. +Use `--profile` or `--org` when needed. Setup activates Braintrust tracing in +Codex. Restart Codex after setup so it loads the plugin. `bt trace update codex` +refreshes the plugin and reapplies its setup. + +If setup reports an error, update Codex and `bt`, then retry. Use +`bt trace doctor codex` to inspect route, authentication, daemon, and plugin +diagnostics. To inspect tracing configuration and daemon status: From 16aea4bdc08c52805d8e7fa82c0a2cdf731f4795 Mon Sep 17 00:00:00 2001 From: Stephen Belanger Date: Fri, 23 Oct 2026 23:08:58 +0800 Subject: [PATCH 10/10] Fix Codex managed run app-server test fixture --- bt-daemon/tests/pipeline.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/bt-daemon/tests/pipeline.rs b/bt-daemon/tests/pipeline.rs index 8a38f31d..314cfd56 100644 --- a/bt-daemon/tests/pipeline.rs +++ b/bt-daemon/tests/pipeline.rs @@ -2714,6 +2714,28 @@ async fn managed_run_flushes_and_preserves_agent_exit_status() { std::fs::write( &agent, r#"#!/bin/sh +for argument in "$@"; do + if [ "$argument" = app-server ]; then + while IFS= read -r request; do + case "$request" in + *'"method":"initialize"'*) + printf '{"id":1,"result":{}}\n' + ;; + *'"method":"hooks/list"'*) + hooks= + index=0 + for event in sessionStart userPromptSubmit preToolUse permissionRequest postToolUse preCompact postCompact subagentStart subagentStop stop interrupt sessionEnd; do + index=$((index + 1)) + [ -z "$hooks" ] || hooks="$hooks," + hooks="$hooks{\"source\":\"sessionFlags\",\"eventName\":\"$event\",\"handlerType\":\"command\",\"command\":\"'unused-hook-command' '--source' 'codex' '--managed-run-hook'\",\"enabled\":true,\"isManaged\":false,\"trustStatus\":\"untrusted\",\"key\":\"hook-$index\",\"currentHash\":\"hash-$index\"}" + done + printf '{"id":2,"result":{"data":[{"errors":[],"hooks":[%s]}]}}\n' "$hooks" + ;; + esac + done + exit 0 + fi +done previous= last= for argument in "$@"; do