From 0e81c45caeb0f8dbe217983dcad25ba3ed47e109 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 13:42:52 +0100 Subject: [PATCH 01/10] perf: reuse external conformance stack Signed-off-by: lucarlig --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/runtime/conformance/mod.rs | 65 +++++++++++++++++++++++++++------- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68d7476..3df804e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,7 @@ dependencies = [ [[package]] name = "cf-integration" -version = "0.2.0" +version = "0.2.1" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 475f48d..3604396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cf-integration" -version = "0.2.0" +version = "0.2.1" edition = "2024" rust-version = "1.97" license = "Apache-2.0" diff --git a/src/runtime/conformance/mod.rs b/src/runtime/conformance/mod.rs index 11d5b66..035436b 100644 --- a/src/runtime/conformance/mod.rs +++ b/src/runtime/conformance/mod.rs @@ -625,6 +625,11 @@ impl RuntimeContext { } expected_server_scenarios(DEFAULT_CONFORMANCE_SUITE, spec_version) .map_err(AppFailure::from)?; + let run_external_client = spec_version == DEFAULT_MCP_SPEC_VERSION + && lanes.contains(&SemanticLane::ExternalDataPlane); + if run_external_client { + expected_client_scenarios(spec_version).map_err(AppFailure::from)?; + } paths.clear_conformance()?; let topologies = conformance_topologies(lanes); @@ -633,6 +638,7 @@ impl RuntimeContext { } let mut failures = Vec::new(); let mut interrupted = false; + let mut external_stack_retained = false; tokio::pin!(interrupt); let (cancellation_sender, cancellation_receiver) = tokio::sync::watch::channel(false); @@ -726,6 +732,7 @@ impl RuntimeContext { let stack_progress = Activity::spinner(format!("Prepare {}", topology.topology_label())); let mut topology_failure = self.stack_up_for_conformance(topology, true).await.err(); + let stack_started = topology_failure.is_none(); stack_progress.finish(topology_failure.is_none()); let mut fixture_state = None; let mut fixture_metadata = None; @@ -896,11 +903,15 @@ impl RuntimeContext { .err(); } - topology_failure = finish_with_cleanup( - topology_failure, - self.cleanup(topology_selection(topology), CleanupKind::Down), - ) - .err(); + if can_reuse_external_stack(topology, stack_started, interrupted, run_external_client) { + external_stack_retained = true; + } else { + topology_failure = finish_with_cleanup( + topology_failure, + self.cleanup(topology_selection(topology), CleanupKind::Down), + ) + .err(); + } if let Some(error) = topology_failure { failures.push(ConformanceOperationalFailure::server( Some(target), @@ -914,15 +925,13 @@ impl RuntimeContext { } } - if !interrupted - && spec_version == DEFAULT_MCP_SPEC_VERSION - && lanes.contains(&SemanticLane::ExternalDataPlane) - { + if !interrupted && run_external_client { let client = self.run_external_client_conformance( spec_version, server_era, paths, cancellation_receiver.clone(), + external_stack_retained, ); tokio::pin!(client); tokio::select! { @@ -1039,11 +1048,16 @@ impl RuntimeContext { server_era: ConformanceServerEra, paths: &ConformancePaths, cancellation: tokio::sync::watch::Receiver, + reuse_stack: bool, ) -> AppResult<()> { - expected_client_scenarios(spec_version).map_err(AppFailure::from)?; - let stack_progress = Activity::spinner("Prepare external dataplane client conformance"); + let progress = if reuse_stack { + "Reuse external dataplane for client conformance" + } else { + "Prepare external dataplane client conformance" + }; + let stack_progress = Activity::spinner(progress); let stack_result = self - .stack_up_for_conformance(StackMode::Dataplane, true) + .stack_up_for_conformance(StackMode::Dataplane, !reuse_stack) .await; stack_progress.finish(stack_result.is_ok()); let mut failure = stack_result.err(); @@ -1638,6 +1652,15 @@ fn conformance_topologies(lanes: &[SemanticLane]) -> Vec { topologies } +fn can_reuse_external_stack( + topology: StackMode, + stack_started: bool, + interrupted: bool, + run_external_client: bool, +) -> bool { + topology == StackMode::Dataplane && stack_started && !interrupted && run_external_client +} + fn parse_conformance_fixture_endpoint(output: &[u8]) -> anyhow::Result { let output = std::str::from_utf8(output).context("Compose fixture port output is not UTF-8")?; let address = output @@ -1875,6 +1898,24 @@ mod tests { ); } + #[test] + fn external_stack_is_reused_only_for_a_started_uninterrupted_client_run() { + let cases = [ + (StackMode::Dataplane, true, false, true, true), + (StackMode::Controlplane, true, false, true, false), + (StackMode::Dataplane, false, false, true, false), + (StackMode::Dataplane, true, true, true, false), + (StackMode::Dataplane, true, false, false, false), + ]; + + for (topology, stack_started, interrupted, run_client, expected) in cases { + assert_eq!( + can_reuse_external_stack(topology, stack_started, interrupted, run_client), + expected, + ); + } + } + #[test] fn direct_fixture_endpoint_accepts_only_loopback_bindings() { assert_eq!( From 27986b57690b5be2f2c479f9a0bf52b6400e3fa8 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Tue, 1 Sep 2026 14:10:04 +0100 Subject: [PATCH 02/10] fix: flush results before failure exit Signed-off-by: lucarlig --- src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 92d7b6b..4762520 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,7 @@ #[cfg(test)] extern crate self as cf_integration; -use std::process::ExitCode; +use std::{io::Write, process::ExitCode}; use clap::Parser; @@ -107,6 +107,9 @@ pub async fn run() -> ExitCode { } fn report_failure(error: AppFailure) -> ExitCode { + // Keep completed result output ahead of wrapper diagnostics such as Make's + // nonzero-exit message when stdout and stderr are captured separately. + let _ = std::io::stdout().flush(); if !error.is_reported() { eprintln!("{}", OutputStyle::stderr().failure(&error.to_string())); } From 36324af807bd785fb23e1790e8a1a8f7132ed8a5 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Wed, 2 Sep 2026 16:39:01 +0100 Subject: [PATCH 03/10] feat: use semantic MCP protocol modes Signed-off-by: lucarlig --- README.md | 11 ++++-- src/app.rs | 6 ++- src/app_tests.rs | 59 ++++++++++++++++------------- src/cli.rs | 60 +++++++++++++++--------------- src/cli_public_tests.rs | 38 +++++-------------- src/mcp/protocol.rs | 27 ++++++++++++++ src/runtime/conformance/reports.rs | 10 ++--- src/runtime/inspect.rs | 2 +- src/runtime/live/mod.rs | 2 +- src/runtime/performance/mod.rs | 2 +- src/runtime/probe.rs | 2 +- src/runtime/stack/mod.rs | 56 ++++++++++++++++++++++++++++ 12 files changed, 176 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 961676c..af3f259 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ all cleanup failures. Probe, load, and Inspector use physical lanes: ```bash -cf-integration probe --lane dataplane --protocol-version 2026-07-28 +cf-integration probe --lane dataplane --protocol-version modern cf-integration load --lane dataplane --smoke cf-integration debug inspect --lane dataplane --method tools/list ``` @@ -126,7 +126,7 @@ Live and conformance share semantic lanes: `fixture-direct`, ```bash cf-integration live --lane external-data-plane --group mcp cf-integration live --lane fixture-direct --group protocol \ - --protocol-version 2025-06-18 + --protocol-version legacy cf-integration conformance run cf-integration conformance run \ @@ -144,6 +144,11 @@ The direct fixture spelling is only `fixture-direct`. Probe, load, live, and Inspector use `--protocol-version`; conformance uses the explicit `--client-era` and `--server-era` matrix axes. +Operational protocol selection is semantic: `modern` maps to the latest +per-request revision and `legacy` maps to the latest initialization-based +revision. Exact date revisions remain internal wire values and conformance +matrix dimensions. + ## MCP and conformance behavior One MCP client owns endpoint construction, authorization, sessions, stateful @@ -242,7 +247,7 @@ CF_FAST_TIME_EXPECTED_IMAGE=ghcr.io/ibm/cfex-mcp-fast-time-server:latest CF_FAST_TIME_SERVER_ID=9779b6698cbd4b4995ee04a4fab38737 MCP_CLI_BASE_URL=http://127.0.0.1:8080 -MCP_PROTOCOL_VERSION=2026-07-28 +MCP_PROTOCOL_VERSION=modern MCP_SERVER_ID=9779b6698cbd4b4995ee04a4fab38737 PLATFORM_ADMIN_EMAIL=admin@example.com diff --git a/src/app.rs b/src/app.rs index a9907aa..53e6024 100644 --- a/src/app.rs +++ b/src/app.rs @@ -5,7 +5,6 @@ use std::ffi::{OsStr, OsString}; use std::path::{Component, PathBuf}; use std::str::FromStr; -use crate::conformance::DEFAULT_MCP_SPEC_VERSION; use crate::conformance::profile::{ DUAL_CLIENT_PROTOCOL_VERSIONS, LEGACY_CLIENT_PROTOCOL_VERSIONS, MODERN_CLIENT_PROTOCOL_VERSIONS, }; @@ -157,7 +156,10 @@ impl StackAction { }, }; if matches!(self, Self::Up { .. }) { - format!("Topology: {topology}\nProtocol version: {DEFAULT_MCP_SPEC_VERSION}") + format!( + "Topology: {topology}\nProtocol version: {}", + ProtocolVersion::default() + ) } else { format!("Topology: {topology}") } diff --git a/src/app_tests.rs b/src/app_tests.rs index e391fdc..792ee38 100644 --- a/src/app_tests.rs +++ b/src/app_tests.rs @@ -107,7 +107,7 @@ fn every_subcommand_reports_its_resolved_topology_at_startup() { let cases: &[(&[&str], &str)] = &[ ( &["cf-integration", "stack", "up"], - "Topology: external dataplane\nProtocol version: 2026-07-28", + "Topology: external dataplane\nProtocol version: modern", ), ( &["cf-integration", "stack", "down"], @@ -127,15 +127,15 @@ fn every_subcommand_reports_its_resolved_topology_at_startup() { ), ( &["cf-integration", "probe"], - "Topology: external dataplane\nProtocol version: 2026-07-28", + "Topology: external dataplane\nProtocol version: modern", ), ( &["cf-integration", "load"], - "Topology: external dataplane\nProtocol version: 2026-07-28", + "Topology: external dataplane\nProtocol version: modern", ), ( &["cf-integration", "live"], - "Topology: external dataplane\nProtocol version: 2026-07-28", + "Topology: external dataplane\nProtocol version: modern", ), ( &["cf-integration", "conformance", "run"], @@ -147,7 +147,7 @@ fn every_subcommand_reports_its_resolved_topology_at_startup() { ), ( &["cf-integration", "debug", "inspect"], - "Topology: external dataplane\nProtocol version: 2026-07-28", + "Topology: external dataplane\nProtocol version: modern", ), ( &["cf-integration", "debug", "token", "--kind", "admin"], @@ -242,15 +242,13 @@ fn topology_precedence_is_cli_then_environment_then_dataplane() { "--lane", "dataplane", "--protocol-version", - "2025-06-18", + "legacy", ], &[("CF_MCP_STACK_MODE", "invalid")], ), Action::Probe { topology: StackMode::Dataplane, - protocol_version: "2025-06-18" - .parse::() - .expect("valid protocol version"), + protocol_version: ProtocolVersion::Legacy, } ); } @@ -265,6 +263,23 @@ fn invalid_environment_topology_is_rejected_when_used() { assert!(error.to_string().contains("invalid CF_MCP_STACK_MODE")); } +#[test] +fn date_based_protocol_environment_is_rejected() { + let cli = Cli::try_parse_from(["cf-integration", "probe"]).expect("CLI should parse"); + let environment = [( + OsString::from("MCP_PROTOCOL_VERSION"), + OsString::from("2026-07-28"), + )] + .into_iter() + .collect(); + let error = resolve_action(cli, &environment).expect_err("wire revisions must remain internal"); + + assert_eq!( + error.to_string(), + "invalid MCP_PROTOCOL_VERSION: must be modern or legacy" + ); +} + #[test] fn stack_actions_resolve_freshness_and_volume_cleanup() { assert_eq!( @@ -303,7 +318,7 @@ fn load_preserves_explicit_locust_settings() { "--lane", "controlplane", "--protocol-version", - "2025-06-18", + "legacy", "--smoke", "--users", "2", @@ -316,9 +331,7 @@ fn load_preserves_explicit_locust_settings() { ), Action::Load(ResolvedLoadArgs { topology: StackMode::Controlplane, - protocol_version: "2025-06-18" - .parse::() - .expect("valid protocol version"), + protocol_version: ProtocolVersion::Legacy, request: LoadRequest { smoke: true, users: Some(2), @@ -336,15 +349,13 @@ fn live_resolves_lane_group_and_protocol_version() { &["cf-integration", "live", "--group", "mcp"], &[ ("CF_MCP_STACK_MODE", "controlplane"), - ("MCP_PROTOCOL_VERSION", "2025-06-18"), + ("MCP_PROTOCOL_VERSION", "legacy"), ], ), Action::Live { lane: SemanticLane::BuiltInDataPlane, group: LiveGroup::Mcp, - protocol_version: "2025-06-18" - .parse::() - .expect("valid protocol version"), + protocol_version: ProtocolVersion::Legacy, } ); } @@ -361,19 +372,17 @@ fn live_fixture_lane_bypasses_topology_and_cli_version_wins() { "--group", "protocol", "--protocol-version", - "2025-03-26", + "modern", ], &[ ("CF_MCP_STACK_MODE", "invalid"), - ("MCP_PROTOCOL_VERSION", "2025-06-18"), + ("MCP_PROTOCOL_VERSION", "legacy"), ], ), Action::Live { lane: SemanticLane::FixtureDirect, group: LiveGroup::Protocol, - protocol_version: "2025-03-26" - .parse::() - .expect("valid protocol version"), + protocol_version: ProtocolVersion::Modern, } ); } @@ -544,7 +553,7 @@ fn debug_token_and_inspector_remain_explicit_non_gate_operations() { "--lane", "controlplane", "--protocol-version", - "2025-06-18", + "legacy", "--method", "prompts/list", ], @@ -552,9 +561,7 @@ fn debug_token_and_inspector_remain_explicit_non_gate_operations() { ), Action::Debug(DebugAction::Inspect { topology: StackMode::Controlplane, - protocol_version: "2025-06-18" - .parse::() - .expect("valid protocol version"), + protocol_version: ProtocolVersion::Legacy, method: "prompts/list".to_owned(), server_id: None, }) diff --git a/src/cli.rs b/src/cli.rs index 9566556..d00d6d1 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -5,12 +5,12 @@ use std::fmt; use std::path::PathBuf; use std::str::FromStr; -use crate::mcp::protocol::PROTOCOL_VERSION; +use crate::mcp::protocol::{LEGACY_PROTOCOL_VERSION, PROTOCOL_VERSION}; use clap::{ArgAction, Args, Parser, Subcommand, ValueEnum}; const RUN_TIME_ERROR: &str = "must be a positive Locust duration using h, m, and s at most once in that order"; -const PROTOCOL_VERSION_ERROR: &str = "must use the MCP YYYY-MM-DD version format"; +const PROTOCOL_VERSION_ERROR: &str = "must be modern or legacy"; fn parse_positive_usize(value: &str) -> Result { let parsed = value @@ -221,8 +221,8 @@ pub(crate) struct RoutedWorkflowTargetArgs { #[arg(long, value_enum)] pub(crate) lane: Option, - /// MCP version; defaults to MCP_PROTOCOL_VERSION, then 2026-07-28. - #[arg(long)] + /// MCP mode; defaults to MCP_PROTOCOL_VERSION, then modern. + #[arg(long, value_enum)] pub(crate) protocol_version: Option, } @@ -233,8 +233,8 @@ pub(crate) struct WorkflowTargetArgs { #[arg(long, value_enum)] pub(crate) lane: Option, - /// MCP version; defaults to MCP_PROTOCOL_VERSION, then 2026-07-28. - #[arg(long)] + /// MCP mode; defaults to MCP_PROTOCOL_VERSION, then modern. + #[arg(long, value_enum)] pub(crate) protocol_version: Option, } @@ -339,27 +339,33 @@ pub(crate) enum LiveGroup { All, } -/// A syntactically valid date-based MCP protocol version shared by workflows. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct ProtocolVersion(String); +/// Semantic MCP protocol mode shared by operational workflows. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub(crate) enum ProtocolVersion { + /// Use the latest per-request, stateless MCP revision. + #[default] + Modern, + /// Use the latest initialization-based MCP revision. + Legacy, +} impl ProtocolVersion { - /// Returns the exact selected MCP protocol version. + /// Returns the exact MCP wire revision selected by this mode. #[must_use] - pub(crate) fn as_str(&self) -> &str { - &self.0 - } -} - -impl Default for ProtocolVersion { - fn default() -> Self { - Self(PROTOCOL_VERSION.to_owned()) + pub(crate) const fn wire_version(self) -> &'static str { + match self { + Self::Modern => PROTOCOL_VERSION, + Self::Legacy => LEGACY_PROTOCOL_VERSION, + } } } impl fmt::Display for ProtocolVersion { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(&self.0) + formatter.write_str(match self { + Self::Modern => "modern", + Self::Legacy => "legacy", + }) } } @@ -367,18 +373,10 @@ impl FromStr for ProtocolVersion { type Err = String; fn from_str(value: &str) -> Result { - let bytes = value.as_bytes(); - let valid = bytes.len() == 10 - && bytes[4] == b'-' - && bytes[7] == b'-' - && bytes - .iter() - .enumerate() - .all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()); - if valid { - Ok(Self(value.to_owned())) - } else { - Err(String::from(PROTOCOL_VERSION_ERROR)) + match value { + "modern" => Ok(Self::Modern), + "legacy" => Ok(Self::Legacy), + _ => Err(String::from(PROTOCOL_VERSION_ERROR)), } } } diff --git a/src/cli_public_tests.rs b/src/cli_public_tests.rs index 8ec14c4..382e559 100644 --- a/src/cli_public_tests.rs +++ b/src/cli_public_tests.rs @@ -238,7 +238,7 @@ fn live_defaults_to_all_and_accepts_the_main_harness_groups() { } #[test] -fn live_accepts_fixture_lane_and_explicit_protocol_version() { +fn live_accepts_fixture_lane_and_explicit_protocol_mode() { let Command::Live(args) = parse(&[ "cf-integration", "live", @@ -247,7 +247,7 @@ fn live_accepts_fixture_lane_and_explicit_protocol_version() { "--group", "protocol", "--protocol-version", - "2025-06-18", + "legacy", ]) .command else { @@ -256,16 +256,12 @@ fn live_accepts_fixture_lane_and_explicit_protocol_version() { assert_eq!(args.target.lane, Some(CliLane::FixtureDirect)); assert_eq!(args.group, LiveGroup::Protocol); - assert_eq!( - args.target.protocol_version, - Some( - "2025-06-18" - .parse::() - .expect("valid protocol version") - ) - ); + assert_eq!(args.target.protocol_version, Some(ProtocolVersion::Legacy)); + assert_eq!(ProtocolVersion::Legacy.wire_version(), "2025-11-25"); + assert_eq!(ProtocolVersion::Modern.wire_version(), "2026-07-28"); rejected(&["cf-integration", "live", "--protocol-version", "latest"]); + rejected(&["cf-integration", "live", "--protocol-version", "2026-07-28"]); rejected(&["cf-integration", "live", "--lane", "fixture"]); } @@ -294,29 +290,15 @@ fn live_rejects_removed_topology_alias() { fn operational_workflows_share_canonical_lane_and_protocol_version_flags() { fn assert_routed_target(target: &RoutedWorkflowTargetArgs) { assert_eq!(target.lane, Some(CliTopology::Controlplane)); - assert_eq!( - target.protocol_version, - Some( - "2025-06-18" - .parse::() - .expect("valid protocol version") - ) - ); + assert_eq!(target.protocol_version, Some(ProtocolVersion::Legacy)); } fn assert_fixture_target(target: &WorkflowTargetArgs) { assert_eq!(target.lane, Some(CliLane::BuiltInDataPlane)); - assert_eq!( - target.protocol_version, - Some( - "2025-06-18" - .parse::() - .expect("valid protocol version") - ) - ); + assert_eq!(target.protocol_version, Some(ProtocolVersion::Legacy)); } - let common = ["--lane", "controlplane", "--protocol-version", "2025-06-18"]; + let common = ["--lane", "controlplane", "--protocol-version", "legacy"]; let Command::Probe(probe) = parse( &["cf-integration", "probe"] .into_iter() @@ -347,7 +329,7 @@ fn operational_workflows_share_canonical_lane_and_protocol_version_flags() { "--lane", "built-in-data-plane", "--protocol-version", - "2025-06-18", + "legacy", ]) .command else { diff --git a/src/mcp/protocol.rs b/src/mcp/protocol.rs index d72ebca..c26efe2 100644 --- a/src/mcp/protocol.rs +++ b/src/mcp/protocol.rs @@ -6,6 +6,8 @@ use uuid::Uuid; /// Latest MCP protocol version used when a workflow does not select one explicitly. pub(crate) const PROTOCOL_VERSION: &str = "2026-07-28"; +/// Latest initialization-based MCP protocol version used by legacy workflows. +pub(crate) const LEGACY_PROTOCOL_VERSION: &str = "2025-11-25"; /// Stateless MCP protocol version used by the modern dataplane lane. pub(crate) const STATELESS_PROTOCOL_VERSION: &str = "2026-07-28"; /// Accepted MCP streamable-HTTP response media types. @@ -37,6 +39,19 @@ pub(crate) fn is_stateless_protocol(protocol_version: &str) -> bool { protocol_version >= STATELESS_PROTOCOL_VERSION } +/// Returns whether a value has the date-based syntax used by MCP revisions. +#[must_use] +pub(crate) fn is_protocol_revision(value: &str) -> bool { + let bytes = value.as_bytes(); + bytes.len() == 10 + && bytes[4] == b'-' + && bytes[7] == b'-' + && bytes + .iter() + .enumerate() + .all(|(index, byte)| matches!(index, 4 | 7) || byte.is_ascii_digit()) +} + /// Builds the mandatory per-request metadata for stateless MCP requests. #[must_use] pub(crate) fn request_metadata(protocol_version: &str) -> Value { @@ -196,3 +211,15 @@ pub(crate) fn tool_call_args(tool_name: &str) -> Option { _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn protocol_revision_requires_the_date_based_wire_syntax() { + assert!(is_protocol_revision("2026-07-28")); + assert!(!is_protocol_revision("modern")); + assert!(!is_protocol_revision("2026-7-28")); + } +} diff --git a/src/runtime/conformance/reports.rs b/src/runtime/conformance/reports.rs index a73bae1..f43f99b 100644 --- a/src/runtime/conformance/reports.rs +++ b/src/runtime/conformance/reports.rs @@ -367,11 +367,11 @@ fn discover_conformance_runs( .file_name() .into_string() .map_err(|_| AppFailure::from(anyhow!("client-version directory is not UTF-8")))?; - ProtocolVersion::from_str(&client_version).map_err(|error| { - AppFailure::from(anyhow!( - "invalid conformance client-version directory {client_version:?}: {error}" - )) - })?; + if !crate::mcp::protocol::is_protocol_revision(&client_version) { + return Err(AppFailure::from(anyhow!( + "invalid conformance client-version directory {client_version:?}: must use the MCP YYYY-MM-DD version format" + ))); + } for era_entry in strict_directories(&version_entry.path(), "server-era")? { let label = era_entry .file_name() diff --git a/src/runtime/inspect.rs b/src/runtime/inspect.rs index 481ee27..bdfb431 100644 --- a/src/runtime/inspect.rs +++ b/src/runtime/inspect.rs @@ -43,7 +43,7 @@ impl RuntimeContext { let proxy = AuthProxy::start_with_protocol_version( endpoint, &token, - Some(protocol_version.as_str()), + Some(protocol_version.wire_version()), ) .await .context("failed to start the Inspector authentication proxy") diff --git a/src/runtime/live/mod.rs b/src/runtime/live/mod.rs index d8d33f6..ab5109c 100644 --- a/src/runtime/live/mod.rs +++ b/src/runtime/live/mod.rs @@ -131,7 +131,7 @@ impl RuntimeContext { .join("scripts") .join("live_protocol"), inherited_python_path, - protocol_version.as_str(), + protocol_version.wire_version(), ) } } diff --git a/src/runtime/performance/mod.rs b/src/runtime/performance/mod.rs index c9f5310..693792f 100644 --- a/src/runtime/performance/mod.rs +++ b/src/runtime/performance/mod.rs @@ -16,7 +16,7 @@ impl RuntimeContext { &settings, &token, (args.topology == StackMode::Dataplane).then_some(operation_server_id.as_str()), - args.protocol_version.as_str(), + args.protocol_version.wire_version(), ) .map_err(AppFailure::from)?; let command_spec = diff --git a/src/runtime/probe.rs b/src/runtime/probe.rs index eee3d34..760f4f5 100644 --- a/src/runtime/probe.rs +++ b/src/runtime/probe.rs @@ -22,7 +22,7 @@ impl RuntimeContext { request_timeout: Duration::from_secs( self.environment_u64("CF_PROBE_REQUEST_TIMEOUT", 30)?, ), - protocol_version: protocol_version.to_string(), + protocol_version: protocol_version.wire_version().to_owned(), output_style: OutputStyle::stdout(), }; let transport = GatewayClient::builder( diff --git a/src/runtime/stack/mod.rs b/src/runtime/stack/mod.rs index 514e7ac..f3236ae 100644 --- a/src/runtime/stack/mod.rs +++ b/src/runtime/stack/mod.rs @@ -4,6 +4,8 @@ mod sources; use super::*; +const COMPOSE_PROTOCOL_VERSION_ENV: &str = "MCP_PROTOCOL_VERSION"; + impl RuntimeContext { pub(super) async fn execute_stack(&self, action: StackAction) -> AppResult<()> { match action { @@ -262,6 +264,12 @@ impl RuntimeContext { command = command.env(key.clone(), value.value.clone()); } } + if let Some(protocol_version) = compose_protocol_version( + &command_environment, + self.environment_text(COMPOSE_PROTOCOL_VERSION_ENV), + )? { + command = command.env(COMPOSE_PROTOCOL_VERSION_ENV, protocol_version); + } let (controlplane_pull_policy, dataplane_pull_policy) = compose_pull_policies( mode, false, @@ -1031,6 +1039,24 @@ fn require_preloaded_image(label: &str, image: &OsStr, local_exists: bool) -> Ap ))) } +fn compose_protocol_version( + command_environment: &BTreeMap, + configured: Option<&str>, +) -> AppResult> { + if command_environment.contains_key(OsStr::new(COMPOSE_PROTOCOL_VERSION_ENV)) { + return Ok(None); + } + let mode = configured + .filter(|value| !value.is_empty()) + .map(str::parse::) + .transpose() + .map_err(|error| { + AppFailure::from(anyhow!("invalid {COMPOSE_PROTOCOL_VERSION_ENV}: {error}")) + })? + .unwrap_or_default(); + Ok(Some(mode.wire_version())) +} + fn compose_pull_policies( mode: StackMode, build: bool, @@ -1228,6 +1254,36 @@ mod tests { ); } + #[test] + fn compose_translates_semantic_protocol_modes_to_wire_revisions() { + let command_environment = BTreeMap::new(); + + assert_eq!( + compose_protocol_version(&command_environment, None) + .expect("default protocol mode should resolve"), + Some("2026-07-28") + ); + assert_eq!( + compose_protocol_version(&command_environment, Some("legacy")) + .expect("legacy protocol mode should resolve"), + Some("2025-11-25") + ); + } + + #[test] + fn compose_preserves_an_explicit_internal_wire_revision() { + let command_environment = BTreeMap::from([( + OsString::from(COMPOSE_PROTOCOL_VERSION_ENV), + OsString::from("2025-11-25"), + )]); + + assert_eq!( + compose_protocol_version(&command_environment, Some("modern")) + .expect("explicit command environment should be preserved"), + None + ); + } + #[test] fn explicit_conformance_era_is_not_replaced_by_the_stack_default() { let command = with_default_conformance_server_era( From e5ab569ac1f850de3fcdf10d434608ba16aa2fe9 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Wed, 2 Sep 2026 16:57:15 +0100 Subject: [PATCH 04/10] refactor: standardize CLI lane selection Signed-off-by: lucarlig --- .env.example | 6 +- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- README.md | 39 ++++---- docker/docker-compose.cf-dataplane.yaml | 2 +- src/app.rs | 120 ++++++++++++----------- src/app_tests.rs | 88 +++++++---------- src/cli.rs | 90 +++++++++--------- src/cli_public_tests.rs | 121 +++++++++++++++++------- src/conformance/results.rs | 4 +- src/conformance/results_tests.rs | 7 +- src/infrastructure/mode.rs | 16 ++-- src/runtime/conformance/mod.rs | 21 ++-- src/runtime/mod.rs | 4 +- src/runtime/performance/mod.rs | 2 +- src/runtime/stack/mod.rs | 20 ++-- src/runtime/stack/sources.rs | 6 +- 17 files changed, 286 insertions(+), 264 deletions(-) diff --git a/.env.example b/.env.example index a210c76..a9469ec 100644 --- a/.env.example +++ b/.env.example @@ -3,10 +3,10 @@ # Copy this file to .env for local runs. .env is ignored by git. # Shell variables override values from .env: # CF_CONTROLPLANE_REF=user/luca/dataplane-integration-fixes \ -# cargo run --locked -- stack up --topology dataplane +# cargo run --locked -- stack up --lane external -# Default single-stack mode. Possible: controlplane, dataplane. -CF_MCP_STACK_MODE=dataplane +# Default execution lane. Possible: builtin, external. +CF_MCP_LANE=external # Optional developer checkout containing source overlays. Without this, the # current directory is the workspace and the binary can materialize embedded diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b99aeca..ffffbbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: exit 1 fi test ! -e "$sandbox/state" - if (cd "$sandbox" && CF_INTEGRATION_DIR="$sandbox/state" "$binary" stack config --topology dataplane); then + if (cd "$sandbox" && CF_INTEGRATION_DIR="$sandbox/state" "$binary" stack config --lane external); then echo "stack config unexpectedly succeeded outside a checkout" >&2 exit 1 fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e256e0a..6afc3bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,7 +50,7 @@ jobs: exit 1 fi test ! -e "$sandbox/state" - if (cd "$sandbox" && CF_INTEGRATION_DIR="$sandbox/state" "$binary" stack config --topology dataplane); then + if (cd "$sandbox" && CF_INTEGRATION_DIR="$sandbox/state" "$binary" stack config --lane external); then echo "stack config unexpectedly succeeded outside a checkout" >&2 exit 1 fi diff --git a/README.md b/README.md index af3f259..fffb0d9 100644 --- a/README.md +++ b/README.md @@ -97,34 +97,34 @@ failure. Test results use aligned nextest-style labels: green `PASS`, yellow `CARGO_TERM_COLOR` control ANSI output. Command data such as tokens, Compose configuration, and report paths remains on standard output for scripting. -Stack commands use physical `--topology controlplane|dataplane`: +Every stack and workflow selector uses semantic `--lane` values. Stack +commands accept `builtin`, `external`, or `all` where both lanes are valid: ```bash -cf-integration stack up --topology dataplane -cf-integration stack up --topology dataplane --fresh -cf-integration stack status --topology dataplane -cf-integration stack config --topology dataplane -cf-integration stack down --topology all -cf-integration stack down --topology all --volumes +cf-integration stack up --lane external +cf-integration stack up --lane external --fresh +cf-integration stack status --lane external +cf-integration stack config --lane external +cf-integration stack down --lane all +cf-integration stack down --lane all --volumes ``` `stack down --volumes` is the explicit destructive reset. Managed workflows preserve the primary failure, attempt every token and stack cleanup, and report all cleanup failures. -Probe, load, and Inspector use physical lanes: +Probe, load, and Inspector use the same routed lanes: ```bash -cf-integration probe --lane dataplane --protocol-version modern -cf-integration load --lane dataplane --smoke -cf-integration debug inspect --lane dataplane --method tools/list +cf-integration probe --lane external --protocol-version modern +cf-integration load --lane builtin --smoke +cf-integration debug inspect --lane external --method tools/list ``` -Live and conformance share semantic lanes: `fixture-direct`, -`built-in-data-plane`, and `external-data-plane`. +Live and conformance additionally support the direct `fixture-direct` lane. ```bash -cf-integration live --lane external-data-plane --group mcp +cf-integration live --lane external --group mcp cf-integration live --lane fixture-direct --group protocol \ --protocol-version legacy @@ -139,10 +139,9 @@ cf-integration conformance report cf-integration --version ``` -Workflows accept only `--lane`; `--topology` is reserved for stack commands. -The direct fixture spelling is only `fixture-direct`. Probe, load, live, and -Inspector use `--protocol-version`; conformance uses the explicit -`--client-era` and `--server-era` matrix axes. +No command accepts `--topology`. The direct fixture spelling is only +`fixture-direct`. Probe, load, live, and Inspector use `--protocol-version`; +conformance uses the explicit `--client-era` and `--server-era` matrix axes. Operational protocol selection is semantic: `modern` maps to the latest per-request revision and `legacy` maps to the latest initialization-based @@ -170,7 +169,7 @@ built-in and external dataplane routes. For protocol `2026-07-28`, the client suite also makes the external dataplane send requests to the official scenario servers. The four downstream scenarios are `tools_call`, `request-metadata`, `http-standard-headers`, and `http-custom-headers`; they run automatically -whenever `external-data-plane` is selected. The workflow records raw official +whenever the `external` lane is selected. The workflow records raw official results without suppression, writes deterministic comparisons, and continues through every expanded client-revision/server-era combination before returning one aggregated result. `dual` is supported only when selected explicitly. @@ -231,7 +230,7 @@ Copy `.env.example` to `.env`. Process values override the file. ```bash CF_INTEGRATION_ROOT=/path/to/contextforge-dev-tools CF_INTEGRATION_DIR=.integration -CF_MCP_STACK_MODE=dataplane +CF_MCP_LANE=external CF_CONTROLPLANE_REPO=https://github.com/IBM/mcp-context-forge.git CF_CONTROLPLANE_REF=main diff --git a/docker/docker-compose.cf-dataplane.yaml b/docker/docker-compose.cf-dataplane.yaml index 43e37f8..208f1f4 100644 --- a/docker/docker-compose.cf-dataplane.yaml +++ b/docker/docker-compose.cf-dataplane.yaml @@ -4,7 +4,7 @@ # export CF_INTEGRATION_ROOT="$PWD" # export CF_DATAPLANE_IMAGE="ghcr.io/contextforge-org/contextforge-data-plane:latest" # export CF_DATAPLANE_PLATFORM="linux/amd64" -# # Or let `cf-integration stack up --topology dataplane` resolve `auto`. +# # Or let `cf-integration stack up --lane external` resolve `auto`. # docker compose \ # -f /path/to/cf-controlplane/docker-compose.yml \ # -f "$CF_INTEGRATION_ROOT/docker/docker-compose.cf-dataplane.yaml" \ diff --git a/src/app.rs b/src/app.rs index 53e6024..4f4ba22 100644 --- a/src/app.rs +++ b/src/app.rs @@ -15,10 +15,10 @@ use crate::performance::LoadRequest; use anyhow::{Result, bail}; use crate::cli::{ - CiCommand, Cli, CliLane, CliTopology, Command, ConformanceCommand, DebugCommand, LiveGroup, - ProtocolVersion, StackCommand, TokenKind, TopologySelection, + CiCommand, Cli, CliLane, CliRoutedLane, Command, ConformanceCommand, DebugCommand, + LaneSelection, LiveGroup, ProtocolVersion, StackCommand, TokenKind, }; -const STACK_MODE_ENV: &str = "CF_MCP_STACK_MODE"; +const LANE_ENV: &str = "CF_MCP_LANE"; const PROTOCOL_VERSION_ENV: &str = "MCP_PROTOCOL_VERSION"; /// Fully resolved application operation. @@ -76,14 +76,14 @@ impl Action { topology, protocol_version, .. - }) => topology_and_protocol(*topology, protocol_version), - Self::Load(args) => topology_and_protocol(args.topology, &args.protocol_version), + }) => lane_and_protocol(*topology, protocol_version), + Self::Load(args) => lane_and_protocol(args.topology, &args.protocol_version), Self::Live { lane, protocol_version, .. } => format!( - "Topology: {}\nProtocol version: {protocol_version}", + "Lane: {}\nProtocol version: {protocol_version}", lane.label() ), Self::Conformance(ConformanceAction::Run { @@ -92,16 +92,16 @@ impl Action { server_eras, .. }) => format!( - "Topology: {}\nClient era: {}\nServer era: {}", + "Lane: {}\nClient era: {}\nServer era: {}", join_lane_labels(lanes), join_client_eras(client_eras), join_server_eras(server_eras), ), Self::Conformance(ConformanceAction::Report { .. }) => String::from( - "Topology: recorded conformance results\nClient era: recorded conformance results\nServer era: recorded conformance results", + "Lane: recorded conformance results\nClient era: recorded conformance results\nServer era: recorded conformance results", ), Self::Debug(DebugAction::Token { .. }) => { - String::from("Topology: not applicable (token only)") + String::from("Lane: not applicable (token only)") } Self::Ci(CiAction::PrepareImage { .. }) => { String::from("CI operation: prepare prebuilt image") @@ -138,38 +138,36 @@ impl Action { impl StackAction { fn startup_summary(&self) -> String { - let topology = match self { + let lane = match self { Self::Up { topology, .. } | Self::Status(topology) | Self::Logs { topology, .. } - | Self::Config(topology) => topology.topology_label().to_owned(), - Self::Down { topology, .. } => match topology { - TopologySelection::Controlplane => { - StackMode::Controlplane.topology_label().to_owned() - } - TopologySelection::Dataplane => StackMode::Dataplane.topology_label().to_owned(), - TopologySelection::All => format!( + | Self::Config(topology) => topology.lane_label().to_owned(), + Self::Down { lane, .. } => match lane { + LaneSelection::Builtin => StackMode::Controlplane.lane_label().to_owned(), + LaneSelection::External => StackMode::Dataplane.lane_label().to_owned(), + LaneSelection::All => format!( "{}, {}", - StackMode::Controlplane.topology_label(), - StackMode::Dataplane.topology_label() + StackMode::Controlplane.lane_label(), + StackMode::Dataplane.lane_label() ), }, }; if matches!(self, Self::Up { .. }) { format!( - "Topology: {topology}\nProtocol version: {}", + "Lane: {lane}\nProtocol version: {}", ProtocolVersion::default() ) } else { - format!("Topology: {topology}") + format!("Lane: {lane}") } } } -fn topology_and_protocol(topology: StackMode, protocol_version: &ProtocolVersion) -> String { +fn lane_and_protocol(topology: StackMode, protocol_version: &ProtocolVersion) -> String { format!( - "Topology: {}\nProtocol version: {protocol_version}", - topology.topology_label() + "Lane: {}\nProtocol version: {protocol_version}", + topology.lane_label() ) } @@ -212,7 +210,7 @@ pub(crate) enum StackAction { fresh: bool, }, Down { - topology: TopologySelection, + lane: LaneSelection, volumes: bool, }, Status(StackMode), @@ -286,13 +284,13 @@ pub(crate) enum CiAction { /// /// # Errors /// -/// Returns an error when a command needs `CF_MCP_STACK_MODE` and its value is -/// neither `controlplane` nor `dataplane`. +/// Returns an error when a command needs `CF_MCP_LANE` and its value is neither +/// `builtin` nor `external`. pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result { match cli.command { Command::Stack(args) => resolve_stack(args.command, environment).map(Action::Stack), Command::Probe(args) => { - let topology = resolve_topology(args.lane, environment)?; + let topology = resolve_lane(args.lane, environment)?; Ok(Action::Probe { topology, protocol_version: resolve_protocol_version( @@ -303,7 +301,7 @@ pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result { - let topology = resolve_topology(args.target.lane, environment)?; + let topology = resolve_lane(args.target.lane, environment)?; Ok(Action::Load(ResolvedLoadArgs { topology, protocol_version: resolve_protocol_version( @@ -355,7 +353,7 @@ pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result Ok(Action::Debug(match args.command { DebugCommand::Inspect(args) => { - let topology = resolve_topology(args.target.lane, environment)?; + let topology = resolve_lane(args.target.lane, environment)?; DebugAction::Inspect { topology, protocol_version: resolve_protocol_version( @@ -417,9 +415,9 @@ fn environment_utf8(environment: &Environment, key: &str) -> Option { fn resolve_live_lane(lane: Option, environment: &Environment) -> Result { Ok(match lane { Some(CliLane::FixtureDirect) => SemanticLane::FixtureDirect, - Some(CliLane::BuiltInDataPlane) => SemanticLane::BuiltInDataPlane, - Some(CliLane::ExternalDataPlane) => SemanticLane::ExternalDataPlane, - None => match resolve_topology(None, environment)? { + Some(CliLane::Builtin) => SemanticLane::BuiltInDataPlane, + Some(CliLane::External) => SemanticLane::ExternalDataPlane, + None => match resolve_lane(None, environment)? { StackMode::Controlplane => SemanticLane::BuiltInDataPlane, StackMode::Dataplane => SemanticLane::ExternalDataPlane, }, @@ -450,25 +448,23 @@ fn resolve_protocol_version( fn resolve_stack(command: StackCommand, environment: &Environment) -> Result { match command { StackCommand::Up(args) => Ok(StackAction::Up { - topology: resolve_topology(args.topology, environment)?, + topology: resolve_lane(args.lane, environment)?, fresh: args.fresh, }), StackCommand::Down(args) => Ok(StackAction::Down { - topology: args.topology.unwrap_or(TopologySelection::All), + lane: args.lane.unwrap_or(LaneSelection::All), volumes: args.volumes, }), - StackCommand::Status(args) => Ok(StackAction::Status(resolve_topology( - args.topology, - environment, - )?)), + StackCommand::Status(args) => { + Ok(StackAction::Status(resolve_lane(args.lane, environment)?)) + } StackCommand::Logs(args) => Ok(StackAction::Logs { - topology: resolve_topology(args.topology, environment)?, + topology: resolve_lane(args.lane, environment)?, services: args.services, }), - StackCommand::Config(args) => Ok(StackAction::Config(resolve_topology( - args.topology, - environment, - )?)), + StackCommand::Config(args) => { + Ok(StackAction::Config(resolve_lane(args.lane, environment)?)) + } } } @@ -532,40 +528,40 @@ fn resolve_server_eras(eras: Vec) -> Vec, environment: &Environment) -> Result { - if let Some(topology) = explicit { - return Ok(topology.into()); +fn resolve_lane(explicit: Option, environment: &Environment) -> Result { + if let Some(lane) = explicit { + return Ok(lane.into()); } - Ok(environment_topology(environment)?.unwrap_or(StackMode::Dataplane)) + Ok(environment_lane(environment)?.unwrap_or(StackMode::Dataplane)) } -fn environment_topology(environment: &Environment) -> Result> { - let Some(value) = environment.get(OsStr::new(STACK_MODE_ENV)) else { +fn environment_lane(environment: &Environment) -> Result> { + let Some(value) = environment.get(OsStr::new(LANE_ENV)) else { return Ok(None); }; match value.to_str() { - Some("controlplane") => Ok(Some(StackMode::Controlplane)), - Some("dataplane") => Ok(Some(StackMode::Dataplane)), + Some("builtin") => Ok(Some(StackMode::Controlplane)), + Some("external") => Ok(Some(StackMode::Dataplane)), _ => bail!( - "invalid {STACK_MODE_ENV}; expected controlplane or dataplane (got {:?})", + "invalid {LANE_ENV}; expected builtin or external (got {:?})", value ), } } -/// Converts a CLI topology selection into its ordered stack modes. -pub(crate) fn selected_topologies(selection: TopologySelection) -> Vec { +/// Converts a CLI lane selection into its ordered stack modes. +pub(crate) fn selected_topologies(selection: LaneSelection) -> Vec { match selection { - TopologySelection::Controlplane => vec![StackMode::Controlplane], - TopologySelection::Dataplane => vec![StackMode::Dataplane], - TopologySelection::All => vec![StackMode::Controlplane, StackMode::Dataplane], + LaneSelection::Builtin => vec![StackMode::Controlplane], + LaneSelection::External => vec![StackMode::Dataplane], + LaneSelection::All => vec![StackMode::Controlplane, StackMode::Dataplane], } } -/// Converts one concrete stack mode into a CLI topology selection. -pub(crate) const fn topology_selection(topology: StackMode) -> TopologySelection { +/// Converts one concrete stack mode into a CLI lane selection. +pub(crate) const fn topology_selection(topology: StackMode) -> LaneSelection { match topology { - StackMode::Controlplane => TopologySelection::Controlplane, - StackMode::Dataplane => TopologySelection::Dataplane, + StackMode::Controlplane => LaneSelection::Builtin, + StackMode::Dataplane => LaneSelection::External, } } diff --git a/src/app_tests.rs b/src/app_tests.rs index 792ee38..0c093a7 100644 --- a/src/app_tests.rs +++ b/src/app_tests.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use cf_integration::app::{ Action, CiAction, ConformanceAction, DebugAction, ResolvedLoadArgs, StackAction, resolve_action, }; -use cf_integration::cli::{Cli, LiveGroup, ProtocolVersion, TokenKind, TopologySelection}; +use cf_integration::cli::{Cli, LaneSelection, LiveGroup, ProtocolVersion, TokenKind}; use cf_integration::conformance::results::{ConformanceServerEra, SemanticLane}; use cf_integration::infrastructure::StackMode; use cf_integration::infrastructure::config::Environment; @@ -103,55 +103,46 @@ fn ci_image_preparation_rejects_nested_artifact_paths() { } #[test] -fn every_subcommand_reports_its_resolved_topology_at_startup() { +fn every_subcommand_reports_its_resolved_lane_at_startup() { let cases: &[(&[&str], &str)] = &[ ( &["cf-integration", "stack", "up"], - "Topology: external dataplane\nProtocol version: modern", + "Lane: external\nProtocol version: modern", ), ( &["cf-integration", "stack", "down"], - "Topology: built-in dataplane, external dataplane", - ), - ( - &["cf-integration", "stack", "status"], - "Topology: external dataplane", - ), - ( - &["cf-integration", "stack", "logs"], - "Topology: external dataplane", - ), - ( - &["cf-integration", "stack", "config"], - "Topology: external dataplane", + "Lane: builtin, external", ), + (&["cf-integration", "stack", "status"], "Lane: external"), + (&["cf-integration", "stack", "logs"], "Lane: external"), + (&["cf-integration", "stack", "config"], "Lane: external"), ( &["cf-integration", "probe"], - "Topology: external dataplane\nProtocol version: modern", + "Lane: external\nProtocol version: modern", ), ( &["cf-integration", "load"], - "Topology: external dataplane\nProtocol version: modern", + "Lane: external\nProtocol version: modern", ), ( &["cf-integration", "live"], - "Topology: external dataplane\nProtocol version: modern", + "Lane: external\nProtocol version: modern", ), ( &["cf-integration", "conformance", "run"], - "Topology: fixture direct, built-in dataplane, external dataplane\nClient era: modern [2026-07-28]\nServer era: legacy [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25]; modern [2026-07-28]", + "Lane: fixture direct, builtin, external\nClient era: modern [2026-07-28]\nServer era: legacy [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25]; modern [2026-07-28]", ), ( &["cf-integration", "conformance", "report"], - "Topology: recorded conformance results\nClient era: recorded conformance results\nServer era: recorded conformance results", + "Lane: recorded conformance results\nClient era: recorded conformance results\nServer era: recorded conformance results", ), ( &["cf-integration", "debug", "inspect"], - "Topology: external dataplane\nProtocol version: modern", + "Lane: external\nProtocol version: modern", ), ( &["cf-integration", "debug", "token", "--kind", "admin"], - "Topology: not applicable (token only)", + "Lane: not applicable (token only)", ), ]; @@ -168,7 +159,7 @@ fn conformance_startup_reports_every_selected_client_and_server_protocol() { "conformance", "run", "--lane", - "built-in-data-plane", + "builtin", "--client-era", "legacy", "--client-era", @@ -181,7 +172,7 @@ fn conformance_startup_reports_every_selected_client_and_server_protocol() { assert_eq!( resolved.startup_summary(), - "Topology: built-in dataplane\nClient era: legacy [2025-06-18, 2025-11-25]; modern [2026-07-28]\nServer era: dual [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25, 2026-07-28]" + "Lane: builtin\nClient era: legacy [2025-06-18, 2025-11-25]; modern [2026-07-28]\nServer era: dual [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25, 2026-07-28]" ); } @@ -202,7 +193,7 @@ fn conformance_startup_labels_both_legacy_era_selections() { assert_eq!( resolved.startup_summary(), - "Topology: fixture direct, built-in dataplane, external dataplane\nClient era: legacy [2025-06-18, 2025-11-25]\nServer era: legacy [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25]" + "Lane: fixture direct, builtin, external\nClient era: legacy [2025-06-18, 2025-11-25]\nServer era: legacy [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25]" ); } @@ -216,7 +207,7 @@ fn multi_phase_commands_own_detailed_progress_while_simple_commands_use_global_p } #[test] -fn topology_precedence_is_cli_then_environment_then_dataplane() { +fn lane_precedence_is_cli_then_environment_then_external() { assert_eq!( action(&["cf-integration", "probe"], &[]), Action::Probe { @@ -225,10 +216,7 @@ fn topology_precedence_is_cli_then_environment_then_dataplane() { } ); assert_eq!( - action( - &["cf-integration", "probe"], - &[("CF_MCP_STACK_MODE", "controlplane")], - ), + action(&["cf-integration", "probe"], &[("CF_MCP_LANE", "builtin")],), Action::Probe { topology: StackMode::Controlplane, protocol_version: ProtocolVersion::default(), @@ -240,11 +228,11 @@ fn topology_precedence_is_cli_then_environment_then_dataplane() { "cf-integration", "probe", "--lane", - "dataplane", + "external", "--protocol-version", "legacy", ], - &[("CF_MCP_STACK_MODE", "invalid")], + &[("CF_MCP_LANE", "invalid")], ), Action::Probe { topology: StackMode::Dataplane, @@ -254,13 +242,13 @@ fn topology_precedence_is_cli_then_environment_then_dataplane() { } #[test] -fn invalid_environment_topology_is_rejected_when_used() { +fn invalid_environment_lane_is_rejected_when_used() { let cli = Cli::try_parse_from(["cf-integration", "probe"]).expect("CLI should parse"); - let environment = [(OsString::from("CF_MCP_STACK_MODE"), OsString::from("bad"))] + let environment = [(OsString::from("CF_MCP_LANE"), OsString::from("bad"))] .into_iter() .collect(); - let error = resolve_action(cli, &environment).expect_err("invalid topology must fail"); - assert!(error.to_string().contains("invalid CF_MCP_STACK_MODE")); + let error = resolve_action(cli, &environment).expect_err("invalid lane must fail"); + assert!(error.to_string().contains("invalid CF_MCP_LANE")); } #[test] @@ -288,8 +276,8 @@ fn stack_actions_resolve_freshness_and_volume_cleanup() { "cf-integration", "stack", "up", - "--topology", - "controlplane", + "--lane", + "builtin", "--fresh", ], &[], @@ -302,7 +290,7 @@ fn stack_actions_resolve_freshness_and_volume_cleanup() { assert_eq!( action(&["cf-integration", "stack", "down", "--volumes"], &[],), Action::Stack(StackAction::Down { - topology: TopologySelection::All, + lane: LaneSelection::All, volumes: true, }) ); @@ -316,7 +304,7 @@ fn load_preserves_explicit_locust_settings() { "cf-integration", "load", "--lane", - "controlplane", + "builtin", "--protocol-version", "legacy", "--smoke", @@ -348,7 +336,7 @@ fn live_resolves_lane_group_and_protocol_version() { action( &["cf-integration", "live", "--group", "mcp"], &[ - ("CF_MCP_STACK_MODE", "controlplane"), + ("CF_MCP_LANE", "builtin"), ("MCP_PROTOCOL_VERSION", "legacy"), ], ), @@ -375,7 +363,7 @@ fn live_fixture_lane_bypasses_topology_and_cli_version_wins() { "modern", ], &[ - ("CF_MCP_STACK_MODE", "invalid"), + ("CF_MCP_LANE", "invalid"), ("MCP_PROTOCOL_VERSION", "legacy"), ], ), @@ -437,11 +425,11 @@ fn conformance_lanes_are_deduplicated_and_normalized() { "conformance", "run", "--lane", - "external-data-plane", + "external", "--lane", "fixture-direct", "--lane", - "external-data-plane", + "external", "--client-era", "legacy", "--client-era", @@ -509,13 +497,7 @@ fn only_report_and_token_actions_skip_runtime_assets() { &[], ); let stack = action( - &[ - "cf-integration", - "stack", - "status", - "--topology", - "dataplane", - ], + &["cf-integration", "stack", "status", "--lane", "external"], &[], ); @@ -551,7 +533,7 @@ fn debug_token_and_inspector_remain_explicit_non_gate_operations() { "debug", "inspect", "--lane", - "controlplane", + "builtin", "--protocol-version", "legacy", "--method", diff --git a/src/cli.rs b/src/cli.rs index d00d6d1..1c0bf4a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -75,7 +75,7 @@ fn parse_run_time(value: &str) -> Result { Ok(value.to_owned()) } -/// Orchestrates control-plane and dataplane integration workflows. +/// Orchestrates built-in and external dataplane integration workflows. #[derive(Debug, Clone, PartialEq, Parser)] #[command(name = "cf-integration", version, arg_required_else_help = true)] pub(crate) struct Cli { @@ -170,24 +170,24 @@ pub(crate) struct StackArgs { /// Operation on one or more Compose stacks. #[derive(Debug, Clone, PartialEq, Eq, Subcommand)] pub(crate) enum StackCommand { - /// Start one stack topology. + /// Start one execution lane. Up(StackUpArgs), - /// Stop one or both stack topologies. + /// Stop one or both execution lanes. Down(StackDownArgs), - /// Show services for one stack topology. - Status(TopologyArgs), - /// Follow logs for one stack topology. + /// Show services for one execution lane. + Status(StackLaneArgs), + /// Follow logs for one execution lane. Logs(StackLogsArgs), - /// Render the merged configuration for one stack topology. - Config(TopologyArgs), + /// Render the merged configuration for one execution lane. + Config(StackLaneArgs), } /// Options for starting one stack. #[derive(Debug, Clone, PartialEq, Eq, Args)] pub(crate) struct StackUpArgs { - /// Stack topology; defaults to CF_MCP_STACK_MODE, then dataplane. + /// Execution lane; defaults to CF_MCP_LANE, then external. #[arg(long, value_enum)] - pub(crate) topology: Option, + pub(crate) lane: Option, /// Remove existing stack volumes before starting. #[arg(long)] @@ -197,29 +197,29 @@ pub(crate) struct StackUpArgs { /// Options for stopping stacks. #[derive(Debug, Clone, PartialEq, Eq, Args)] pub(crate) struct StackDownArgs { - /// Stack topology; defaults to all. + /// Execution lane; defaults to all. #[arg(long, value_enum)] - pub(crate) topology: Option, + pub(crate) lane: Option, /// Remove persistent volumes as well as containers and networks. #[arg(long)] pub(crate) volumes: bool, } -/// A command targeting one stack topology. +/// A command targeting one stack lane. #[derive(Debug, Clone, PartialEq, Eq, Args)] -pub(crate) struct TopologyArgs { - /// Stack topology; defaults to CF_MCP_STACK_MODE, then dataplane. +pub(crate) struct StackLaneArgs { + /// Execution lane; defaults to CF_MCP_LANE, then external. #[arg(long, value_enum)] - pub(crate) topology: Option, + pub(crate) lane: Option, } /// Target selection for routed MCP workflows. #[derive(Debug, Clone, PartialEq, Eq, Args)] pub(crate) struct RoutedWorkflowTargetArgs { - /// Execution lane; defaults to CF_MCP_STACK_MODE, then dataplane. + /// Execution lane; defaults to CF_MCP_LANE, then external. #[arg(long, value_enum)] - pub(crate) lane: Option, + pub(crate) lane: Option, /// MCP mode; defaults to MCP_PROTOCOL_VERSION, then modern. #[arg(long, value_enum)] @@ -229,7 +229,7 @@ pub(crate) struct RoutedWorkflowTargetArgs { /// Target selection for MCP workflows that support a direct fixture lane. #[derive(Debug, Clone, PartialEq, Eq, Args)] pub(crate) struct WorkflowTargetArgs { - /// Execution lane; defaults to CF_MCP_STACK_MODE, then dataplane. + /// Execution lane; defaults to CF_MCP_LANE, then external. #[arg(long, value_enum)] pub(crate) lane: Option, @@ -241,41 +241,41 @@ pub(crate) struct WorkflowTargetArgs { /// Options for following stack logs. #[derive(Debug, Clone, PartialEq, Eq, Args)] pub(crate) struct StackLogsArgs { - /// Stack topology; defaults to CF_MCP_STACK_MODE, then dataplane. + /// Execution lane; defaults to CF_MCP_LANE, then external. #[arg(long, value_enum)] - pub(crate) topology: Option, + pub(crate) lane: Option, /// Services whose logs to follow; all services when omitted. #[arg(value_name = "SERVICE")] pub(crate) services: Vec, } -/// A live stack topology. +/// A routed MCP execution lane. #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] -pub(crate) enum CliTopology { - /// Python control plane only. - Controlplane, - /// Python control plane routed through the Rust dataplane. - Dataplane, -} - -impl From for crate::infrastructure::StackMode { - fn from(topology: CliTopology) -> Self { - match topology { - CliTopology::Controlplane => Self::Controlplane, - CliTopology::Dataplane => Self::Dataplane, +pub(crate) enum CliRoutedLane { + /// Route through the Python built-in dataplane. + Builtin, + /// Route through the external Rust dataplane. + External, +} + +impl From for crate::infrastructure::StackMode { + fn from(lane: CliRoutedLane) -> Self { + match lane { + CliRoutedLane::Builtin => Self::Controlplane, + CliRoutedLane::External => Self::Dataplane, } } } -/// One or both stack topologies. +/// One or both routed MCP execution lanes. #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] -pub(crate) enum TopologySelection { - /// Python control plane only. - Controlplane, - /// Python control plane routed through the Rust dataplane. - Dataplane, - /// Run controlplane and dataplane sequentially. +pub(crate) enum LaneSelection { + /// Route through the Python built-in dataplane. + Builtin, + /// Route through the external Rust dataplane. + External, + /// Run the built-in and external lanes sequentially. All, } @@ -321,9 +321,9 @@ pub(crate) enum CliLane { /// Run directly against the workflow's reference fixture. FixtureDirect, /// Run the routed endpoint through the Python built-in dataplane. - BuiltInDataPlane, + Builtin, /// Run the routed endpoint through the external Rust data plane. - ExternalDataPlane, + External, } /// Upstream live-test group. @@ -434,8 +434,8 @@ impl From for crate::conformance::results::SemanticLane { fn from(lane: CliLane) -> Self { match lane { CliLane::FixtureDirect => Self::FixtureDirect, - CliLane::BuiltInDataPlane => Self::BuiltInDataPlane, - CliLane::ExternalDataPlane => Self::ExternalDataPlane, + CliLane::Builtin => Self::BuiltInDataPlane, + CliLane::External => Self::ExternalDataPlane, } } } diff --git a/src/cli_public_tests.rs b/src/cli_public_tests.rs index 382e559..c837028 100644 --- a/src/cli_public_tests.rs +++ b/src/cli_public_tests.rs @@ -1,9 +1,9 @@ use std::ffi::OsString; use cf_integration::cli::{ - Cli, CliConformanceEra, CliLane, CliTopology, Command, ConformanceArgs, ConformanceCommand, - DebugArgs, DebugCommand, LiveGroup, LoadArgs, ProtocolVersion, RoutedWorkflowTargetArgs, - StackArgs, StackCommand, TokenKind, TopologySelection, WorkflowTargetArgs, + Cli, CliConformanceEra, CliLane, CliRoutedLane, Command, ConformanceArgs, ConformanceCommand, + DebugArgs, DebugCommand, LaneSelection, LiveGroup, LoadArgs, ProtocolVersion, + RoutedWorkflowTargetArgs, StackArgs, StackCommand, TokenKind, WorkflowTargetArgs, }; use clap::{CommandFactory, Parser, error::ErrorKind}; @@ -100,6 +100,38 @@ fn every_public_command_renders_help() { } } +#[test] +fn every_public_stack_or_workflow_selector_uses_lane_only() { + let paths: &[&[&str]] = &[ + &["stack", "up"], + &["stack", "down"], + &["stack", "status"], + &["stack", "logs"], + &["stack", "config"], + &["probe"], + &["load"], + &["live"], + &["conformance", "run"], + &["debug", "inspect"], + ]; + + for path in paths { + let command = command_at(path); + let argument_ids = command + .get_arguments() + .map(|argument| argument.get_id().as_str()) + .collect::>(); + assert!( + argument_ids.contains(&"lane"), + "missing --lane for {path:?}" + ); + assert!( + !argument_ids.contains(&"topology"), + "obsolete --topology remains on {path:?}" + ); + } +} + #[test] fn obsolete_root_commands_and_combined_workflows_are_rejected() { for command in REMOVED_COMMANDS { @@ -118,15 +150,15 @@ fn stack_up_and_down_make_destructive_behavior_explicit() { "cf-integration", "stack", "up", - "--topology", - "dataplane", + "--lane", + "external", "--fresh", ]) .command else { panic!("expected stack up") }; - assert_eq!(up.topology, Some(CliTopology::Dataplane)); + assert_eq!(up.lane, Some(CliRoutedLane::External)); assert!(up.fresh); let Command::Stack(StackArgs { @@ -135,7 +167,7 @@ fn stack_up_and_down_make_destructive_behavior_explicit() { "cf-integration", "stack", "down", - "--topology", + "--lane", "all", "--volumes", ]) @@ -143,7 +175,7 @@ fn stack_up_and_down_make_destructive_behavior_explicit() { else { panic!("expected stack down") }; - assert_eq!(down.topology, Some(TopologySelection::All)); + assert_eq!(down.lane, Some(LaneSelection::All)); assert!(down.volumes); } @@ -155,8 +187,8 @@ fn stack_logs_preserve_service_arguments() { "cf-integration", "stack", "logs", - "--topology", - "controlplane", + "--lane", + "builtin", "gateway", "worker", ]) @@ -164,7 +196,7 @@ fn stack_logs_preserve_service_arguments() { else { panic!("expected stack logs") }; - assert_eq!(args.topology, Some(CliTopology::Controlplane)); + assert_eq!(args.lane, Some(CliRoutedLane::Builtin)); assert_eq!( args.services, [OsString::from("gateway"), OsString::from("worker")] @@ -224,7 +256,7 @@ fn live_defaults_to_all_and_accepts_the_main_harness_groups() { "cf-integration", "live", "--lane", - "external-data-plane", + "external", "--group", name, ]) @@ -232,7 +264,7 @@ fn live_defaults_to_all_and_accepts_the_main_harness_groups() { else { panic!("expected live workflow") }; - assert_eq!(args.target.lane, Some(CliLane::ExternalDataPlane)); + assert_eq!(args.target.lane, Some(CliLane::External)); assert_eq!(args.group, expected); } } @@ -267,38 +299,58 @@ fn live_accepts_fixture_lane_and_explicit_protocol_mode() { } #[test] -fn probe_rejects_removed_topology_alias() { - rejected(&["cf-integration", "probe", "--topology", "dataplane"]); -} - -#[test] -fn load_rejects_removed_topology_alias() { - rejected(&["cf-integration", "load", "--topology", "dataplane"]); +fn every_public_selector_rejects_the_removed_topology_flag() { + for arguments in [ + vec!["cf-integration", "stack", "up", "--topology", "dataplane"], + vec!["cf-integration", "probe", "--topology", "dataplane"], + vec!["cf-integration", "load", "--topology", "dataplane"], + vec!["cf-integration", "live", "--topology", "dataplane"], + vec![ + "cf-integration", + "debug", + "inspect", + "--topology", + "dataplane", + ], + ] { + rejected(&arguments); + } } #[test] -fn live_rejects_removed_topology_alias() { - rejected(&[ - "cf-integration", - "live", - "--topology", - "external-data-plane", - ]); +fn public_lane_values_reject_physical_and_obsolete_spellings() { + for arguments in [ + vec!["cf-integration", "stack", "up", "--lane", "controlplane"], + vec!["cf-integration", "stack", "up", "--lane", "dataplane"], + vec!["cf-integration", "load", "--lane", "controlplane"], + vec!["cf-integration", "load", "--lane", "dataplane"], + vec!["cf-integration", "live", "--lane", "built-in-data-plane"], + vec!["cf-integration", "live", "--lane", "external-data-plane"], + vec![ + "cf-integration", + "conformance", + "run", + "--lane", + "external-data-plane", + ], + ] { + rejected(&arguments); + } } #[test] fn operational_workflows_share_canonical_lane_and_protocol_version_flags() { fn assert_routed_target(target: &RoutedWorkflowTargetArgs) { - assert_eq!(target.lane, Some(CliTopology::Controlplane)); + assert_eq!(target.lane, Some(CliRoutedLane::Builtin)); assert_eq!(target.protocol_version, Some(ProtocolVersion::Legacy)); } fn assert_fixture_target(target: &WorkflowTargetArgs) { - assert_eq!(target.lane, Some(CliLane::BuiltInDataPlane)); + assert_eq!(target.lane, Some(CliLane::Builtin)); assert_eq!(target.protocol_version, Some(ProtocolVersion::Legacy)); } - let common = ["--lane", "controlplane", "--protocol-version", "legacy"]; + let common = ["--lane", "builtin", "--protocol-version", "legacy"]; let Command::Probe(probe) = parse( &["cf-integration", "probe"] .into_iter() @@ -327,7 +379,7 @@ fn operational_workflows_share_canonical_lane_and_protocol_version_flags() { "cf-integration", "live", "--lane", - "built-in-data-plane", + "builtin", "--protocol-version", "legacy", ]) @@ -398,7 +450,7 @@ fn conformance_accepts_repeatable_exact_lanes_and_protocol_eras() { "--lane", "fixture-direct", "--lane", - "external-data-plane", + "external", "--client-era", "legacy", "--client-era", @@ -417,10 +469,7 @@ fn conformance_accepts_repeatable_exact_lanes_and_protocol_eras() { else { panic!("expected conformance run") }; - assert_eq!( - args.lane, - [CliLane::FixtureDirect, CliLane::ExternalDataPlane] - ); + assert_eq!(args.lane, [CliLane::FixtureDirect, CliLane::External]); assert_eq!( args.client_era, [CliConformanceEra::Legacy, CliConformanceEra::Dual] diff --git a/src/conformance/results.rs b/src/conformance/results.rs index a61fc98..a0a535f 100644 --- a/src/conformance/results.rs +++ b/src/conformance/results.rs @@ -165,8 +165,8 @@ impl SemanticLane { pub(crate) const fn label(self) -> &'static str { match self { Self::FixtureDirect => "fixture direct", - Self::BuiltInDataPlane => "built-in dataplane", - Self::ExternalDataPlane => "external dataplane", + Self::BuiltInDataPlane => "builtin", + Self::ExternalDataPlane => "external", } } diff --git a/src/conformance/results_tests.rs b/src/conformance/results_tests.rs index 554dcec..2ff926e 100644 --- a/src/conformance/results_tests.rs +++ b/src/conformance/results_tests.rs @@ -24,11 +24,8 @@ const SPEC_REFERENCE: &str = #[test] fn semantic_lanes_have_one_shared_stable_vocabulary() { assert_eq!(SemanticLane::FixtureDirect.label(), "fixture direct"); - assert_eq!(SemanticLane::BuiltInDataPlane.label(), "built-in dataplane"); - assert_eq!( - SemanticLane::ExternalDataPlane.label(), - "external dataplane" - ); + assert_eq!(SemanticLane::BuiltInDataPlane.label(), "builtin"); + assert_eq!(SemanticLane::ExternalDataPlane.label(), "external"); } #[test] diff --git a/src/infrastructure/mode.rs b/src/infrastructure/mode.rs index 87ce6cb..0d0e031 100644 --- a/src/infrastructure/mode.rs +++ b/src/infrastructure/mode.rs @@ -8,21 +8,21 @@ pub(crate) enum StackMode { } impl StackMode { - /// Semantic topology name shown to users. + /// Semantic lane name shown to users. #[must_use] - pub(crate) const fn topology_label(self) -> &'static str { + pub(crate) const fn lane_label(self) -> &'static str { match self { - Self::Controlplane => "built-in dataplane", - Self::Dataplane => "external dataplane", + Self::Controlplane => "builtin", + Self::Dataplane => "external", } } - /// Canonical physical topology value accepted by stack commands. + /// Canonical semantic lane value accepted by public commands. #[must_use] - pub(crate) const fn cli_value(self) -> &'static str { + pub(crate) const fn lane_value(self) -> &'static str { match self { - Self::Controlplane => "controlplane", - Self::Dataplane => "dataplane", + Self::Controlplane => "builtin", + Self::Dataplane => "external", } } } diff --git a/src/runtime/conformance/mod.rs b/src/runtime/conformance/mod.rs index 035436b..4b1a24a 100644 --- a/src/runtime/conformance/mod.rs +++ b/src/runtime/conformance/mod.rs @@ -712,7 +712,7 @@ impl RuntimeContext { if !topologies.is_empty() && !interrupted { let cleanup_progress = Activity::spinner("Clear prior integration stacks"); - let cleanup_result = self.cleanup(TopologySelection::All, CleanupKind::Reset); + let cleanup_result = self.cleanup(LaneSelection::All, CleanupKind::Reset); cleanup_progress.finish(cleanup_result.is_ok()); if let Err(error) = cleanup_result { failures.push(ConformanceOperationalFailure::server( @@ -729,8 +729,7 @@ impl RuntimeContext { } let target = conformance_target(topology); let run_routed = lanes.contains(&target); - let stack_progress = - Activity::spinner(format!("Prepare {}", topology.topology_label())); + let stack_progress = Activity::spinner(format!("Prepare {}", topology.lane_label())); let mut topology_failure = self.stack_up_for_conformance(topology, true).await.err(); let stack_started = topology_failure.is_none(); stack_progress.finish(topology_failure.is_none()); @@ -742,7 +741,7 @@ impl RuntimeContext { if topology_failure.is_none() { let fixture_progress = Activity::spinner(format!( "Start the official fixture for {}", - topology.topology_label() + topology.lane_label() )); let (start_result, start_interrupted) = finish_phase_after_interrupt( self.start_conformance_service(topology, server_era), @@ -783,7 +782,7 @@ impl RuntimeContext { Ok(client) => { let provision_progress = Activity::spinner(format!( "Register the official fixture for {}", - topology.topology_label() + topology.lane_label() )); let (provision_result, provision_interrupted) = finish_phase_after_interrupt( @@ -916,7 +915,7 @@ impl RuntimeContext { failures.push(ConformanceOperationalFailure::server( Some(target), "run", - format!("{} topology: {error}", topology.topology_label()), + format!("{} lane: {error}", topology.lane_label()), )); } if interrupted { @@ -1720,7 +1719,7 @@ fn combine_cleanup_results(first: AppResult<()>, second: AppResult<()>) -> AppRe fn fixture_registration_context(topology: StackMode, server_era: ConformanceServerEra) -> String { format!( "ContextForge could not register the official fixture for {} with server era {} [{}]; routed tests for this lane were skipped", - topology.topology_label(), + topology.lane_label(), server_era.label(), server_era.protocol_versions_label() ) @@ -1972,7 +1971,7 @@ mod tests { assert_eq!( context, - "ContextForge could not register the official fixture for built-in dataplane with server era modern [2026-07-28]; routed tests for this lane were skipped" + "ContextForge could not register the official fixture for builtin with server era modern [2026-07-28]; routed tests for this lane were skipped" ); } @@ -2040,7 +2039,7 @@ mod tests { assert_eq!( rendered, - "────────────\n MCP server conformance results: external dataplane\n Client era: modern [2026-07-28]\n Server era: legacy [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25]\n XFAIL (1/2) server::external-data-plane::failing\n PASS (2/2) server::external-data-plane::passing\n────────────\n Summary [ 1.250s] 1 passed, 1 xfailed, 0 xpassed, 0 failed, 0 skipped, 0 unknown" + "────────────\n MCP server conformance results: external\n Client era: modern [2026-07-28]\n Server era: legacy [2024-11-05, 2025-03-26, 2025-06-18, 2025-11-25]\n XFAIL (1/2) server::external-data-plane::failing\n PASS (2/2) server::external-data-plane::passing\n────────────\n Summary [ 1.250s] 1 passed, 1 xfailed, 0 xpassed, 0 failed, 0 skipped, 0 unknown" ); } @@ -2103,7 +2102,7 @@ mod tests { assert_eq!( rendered, - "────────────\n MCP server conformance results: external dataplane\n Client era: modern [2026-07-28]\n Server era: modern [2026-07-28]\n FAIL (1/2) server::external-data-plane::failing\n PASS (2/2) server::external-data-plane::passing\n────────────\n Summary [ 1.250s] 1 passed, 0 xfailed, 0 xpassed, 1 failed, 0 skipped, 0 unknown" + "────────────\n MCP server conformance results: external\n Client era: modern [2026-07-28]\n Server era: modern [2026-07-28]\n FAIL (1/2) server::external-data-plane::failing\n PASS (2/2) server::external-data-plane::passing\n────────────\n Summary [ 1.250s] 1 passed, 0 xfailed, 0 xpassed, 1 failed, 0 skipped, 0 unknown" ); } @@ -2118,7 +2117,7 @@ mod tests { OutputStyle::plain(), ); - assert!(rendered.contains("MCP client conformance results: external dataplane")); + assert!(rendered.contains("MCP client conformance results: external")); assert!(rendered.contains("Client era: modern [2026-07-28]")); assert!(rendered.contains("Server era: modern [2026-07-28]")); assert!(rendered.contains("client::external-data-plane::failing")); diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index 2d523d8..678789d 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -52,7 +52,7 @@ use crate::app::{ Action, CiAction, ConformanceAction, DebugAction, ResolvedLoadArgs, StackAction, selected_topologies, topology_selection, }; -use crate::cli::{LiveGroup, ProtocolVersion, TokenKind as CliTokenKind, TopologySelection}; +use crate::cli::{LaneSelection, LiveGroup, ProtocolVersion, TokenKind as CliTokenKind}; use crate::error::AppFailure; use crate::{Activity, OutputStyle, TestStatus}; @@ -321,7 +321,7 @@ async fn wait_for_http_endpoint( if now >= deadline { return Err(AppFailure::from(anyhow!( "{} public MCP endpoint {} was not ready within {:.3}s; last result: {last_failure}", - mode.topology_label(), + mode.lane_label(), endpoint, timeout.as_secs_f64() ))); diff --git a/src/runtime/performance/mod.rs b/src/runtime/performance/mod.rs index 693792f..373b494 100644 --- a/src/runtime/performance/mod.rs +++ b/src/runtime/performance/mod.rs @@ -52,7 +52,7 @@ impl RuntimeContext { "{}", OutputStyle::stdout().test_result( status, - &format!("performance::{}", args.topology.topology_label()), + &format!("performance::{}", args.topology.lane_label()), Some(elapsed), None, ) diff --git a/src/runtime/stack/mod.rs b/src/runtime/stack/mod.rs index f3236ae..146ae0c 100644 --- a/src/runtime/stack/mod.rs +++ b/src/runtime/stack/mod.rs @@ -39,8 +39,8 @@ impl RuntimeContext { Activity::completed("Integration stack ready"); self.print_stack_summary(topology, &conformance_endpoint) } - StackAction::Down { topology, volumes } => self.cleanup( - topology, + StackAction::Down { lane, volumes } => self.cleanup( + lane, if volumes { CleanupKind::Reset } else { @@ -166,7 +166,7 @@ impl RuntimeContext { if report_progress { println!( "{}", - OutputStyle::stdout().success(&format!("{} stack started.", mode.topology_label())) + OutputStyle::stdout().success(&format!("{} stack started.", mode.lane_label())) ); } Ok(()) @@ -184,7 +184,7 @@ impl RuntimeContext { OutputStyle::stderr().info(&format!( "Waiting up to {}s for the public {} MCP endpoint.", STACK_READY_TIMEOUT.as_secs(), - mode.topology_label() + mode.lane_label() )) ); } @@ -817,19 +817,19 @@ impl RuntimeContext { &self.config.controlplane_project().value, "CF_CONTROLPLANE_PROJECT", )?, - StackMode::Controlplane.topology_label(), + StackMode::Controlplane.lane_label(), ), StackMode::Controlplane => ( required_text( &self.config.integration_project().value, "CF_INTEGRATION_PROJECT", )?, - StackMode::Dataplane.topology_label(), + StackMode::Dataplane.lane_label(), ), }; if self.project_has_running_containers(other)? { return Err(AppFailure::from(anyhow!( - "the {label} stack is running on the same host ports; run `cf-integration stack down --topology all` first" + "the {label} stack is running on the same host ports; run `cf-integration stack down --lane all` first" ))); } Ok(()) @@ -846,13 +846,13 @@ impl RuntimeContext { .is_empty()) } - pub(super) fn cleanup(&self, selection: TopologySelection, kind: CleanupKind) -> AppResult<()> { + pub(super) fn cleanup(&self, selection: LaneSelection, kind: CleanupKind) -> AppResult<()> { self.cleanup_with_output(selection, kind, true) } pub(super) fn cleanup_quiet( &self, - selection: TopologySelection, + selection: LaneSelection, kind: CleanupKind, ) -> AppResult<()> { self.cleanup_with_output(selection, kind, false) @@ -860,7 +860,7 @@ impl RuntimeContext { fn cleanup_with_output( &self, - selection: TopologySelection, + selection: LaneSelection, kind: CleanupKind, inherit_output: bool, ) -> AppResult<()> { diff --git a/src/runtime/stack/sources.rs b/src/runtime/stack/sources.rs index 93d7c82..e97249c 100644 --- a/src/runtime/stack/sources.rs +++ b/src/runtime/stack/sources.rs @@ -7,9 +7,9 @@ impl RuntimeContext { let controlplane_compose = self.config.controlplane_dir().join("docker-compose.yml"); if !controlplane_compose.is_file() { return Err(AppFailure::from(anyhow!( - "control-plane checkout is unavailable at {}; run `cf-integration stack up --topology {}` first", + "control-plane checkout is unavailable at {}; run `cf-integration stack up --lane {}` first", self.config.controlplane_dir().display(), - mode.cli_value() + mode.lane_value() ))); } if mode == StackMode::Dataplane @@ -17,7 +17,7 @@ impl RuntimeContext { && !self.config.dataplane_dir().is_dir() { return Err(AppFailure::from(anyhow!( - "dataplane source checkout is unavailable at {}; run `cf-integration stack up --topology dataplane` first", + "dataplane source checkout is unavailable at {}; run `cf-integration stack up --lane external` first", self.config.dataplane_dir().display() ))); } From 5424f7e530e80877cddaa66bdf2bf1e09dd2cb30 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Thu, 3 Sep 2026 09:53:55 +0100 Subject: [PATCH 05/10] docs: add concise CLI command guide Signed-off-by: lucarlig --- .env.example | 6 +- README.md | 422 +++++++++++++++++++-------------------------------- 2 files changed, 158 insertions(+), 270 deletions(-) diff --git a/.env.example b/.env.example index a9469ec..e5f2780 100644 --- a/.env.example +++ b/.env.example @@ -112,9 +112,9 @@ NGINX_PORT=8080 # Direct public-origin override; otherwise derived from NGINX_PORT. # MCP_CLI_BASE_URL=http://127.0.0.1:8080 -# Optional global MCP protocol override. Probe, conformance, live-stack, and -# performance workflows all default to the latest supported protocol. -# MCP_PROTOCOL_VERSION=2026-07-28 +# Optional operational MCP protocol mode override for probe, load, live, and +# Inspector workflows. Possible: modern, legacy. Default: modern. +# MCP_PROTOCOL_VERSION=modern # Local integration administrator. Stable random signing/encryption secrets are # created automatically under CF_INTEGRATION_DIR when their overrides are unset. diff --git a/README.md b/README.md index fffb0d9..c304af5 100644 --- a/README.md +++ b/README.md @@ -1,319 +1,207 @@ # cf-integration -`cf-integration` is the standalone Rust CLI for exercising `cf-controlplane` -with either its built-in Python data plane or the external Rust -`cf-dataplane`. +`cf-integration` runs ContextForge stacks and tests against the built-in Python +dataplane or the external Rust dataplane. It manages Docker Compose, source +checkouts, MCP probes, Locust load tests, upstream live tests, and official MCP +conformance runs. -The routing contract is fixed: +`/servers/{virtual_host_id}/mcp` routes through `cf-dataplane`; raw `/mcp`, +UI, and API traffic route to `cf-controlplane`. The external dataplane fails +closed and never falls back to the built-in dataplane. -- `/servers/{virtual_host_id}/mcp` routes through `cf-dataplane`. -- Raw `/mcp`, UI, and API traffic route to `cf-controlplane`. -- The external dataplane fails closed and never falls back to the - control plane. - -The CLI owns Docker Compose overlays, nginx routing, source checkout -orchestration, MCP probes, Locust load tests, upstream live tests, and official -MCP conformance runs. - -## Install - -Release archives cover ARM64 and x86-64 Linux, macOS, and Windows. +## Install and requirements ```bash cargo binstall cf-integration -cf-integration --help +# or +cargo install cf-integration --locked ``` -To compile from crates.io or this checkout: +To run the current checkout, put `cargo run --` before any command: ```bash -cargo install cf-integration --locked -cargo install --path . --locked +cargo run -- probe --lane external --protocol-version modern ``` -The installed binary is repository-independent. Required Compose overlays, -runtime scripts, and conformance baselines are embedded in the executable. - -## Runtime assets and workspace resolution - -The CLI resolves the action before initializing state. Runtime-backed actions -resolve assets in this order: - -1. explicit `CF_INTEGRATION_ROOT`, which must be a valid developer checkout; -2. the current directory, when it contains a valid checkout; -3. a versioned embedded-asset tree beneath `CF_INTEGRATION_DIR`. - -Embedded assets are materialized atomically, verified byte-for-byte, marked -read-only, and reused. Concurrent first runs converge on one complete tree. A -corrupt or incomplete versioned tree fails closed. - -`.env` is loaded from `CF_INTEGRATION_ROOT` when set, otherwise the current -directory. Relative paths resolve from that workspace. Generated checkouts, -assets, secrets, reports, and runtime state default to `.integration/`. - -`conformance report` and `debug token` do not materialize assets or generate -local Compose secrets. Compose-backed actions initialize them lazily. - -## Requirements - -Runtime requirements depend on the command: - -- Docker Engine with Docker Compose v2 for stack-backed workflows; -- Git for managed source checkouts; -- Node.js 22.7.5 or newer with `npx` for Inspector, live, and conformance; -- the control-plane checkout's Python/Locust dependencies for load tests; -- Rust 1.97 only when compiling the CLI or local source images. - -Published control-plane and data-plane images are used by default. Local -data-plane builds require an explicit `CF_DATAPLANE_REF`. - -## CLI - -```text -cf-integration -├── stack -│ ├── up -│ ├── down -│ ├── status -│ ├── logs -│ └── config -├── probe -├── load -├── live -├── conformance -│ ├── run -│ └── report -└── debug - ├── inspect - └── token -``` +Runtime requirements are Docker with Compose v2, Git, and Node.js 22.7.5 or +newer with `npx`. Load tests also need the control-plane checkout's Python and +Locust dependencies. Rust 1.97 is required only to compile the CLI or a local +source image. + +Published images are used by default. Set `CF_DATAPLANE_REF` to build an +external dataplane source ref. + +## Common selectors + +Wherever `--lane` is accepted, use these values: + +- `builtin`: Python built-in dataplane. +- `external`: external Rust dataplane. +- `fixture-direct`: reference fixture without ContextForge; available only to + conformance and `live --group protocol`. + +Routed commands default to `CF_MCP_LANE`, then `external`, and run one lane +at a time. Run them once per lane when comparing `builtin` and `external`. +`stack down` also accepts `all`; conformance accepts repeated `--lane` +options. No command accepts `--topology`. -Use `--help` at any level for the authoritative interface. +`probe`, `load`, `live`, and `debug inspect` accept +`--protocol-version modern|legacy`: -Every resolved command reports its lifecycle on standard error using the same -description: `⠋` while active, `✓` in green on success, and `✗` in red on -failure. Test results use aligned nextest-style labels: green `PASS`, yellow -`XFAIL`, red `XPASS` and `FAIL`, and yellow `SKIP`. `NO_COLOR` and -`CARGO_TERM_COLOR` control ANSI output. Command data such as tokens, Compose -configuration, and report paths remains on standard output for scripting. +- `modern`: latest per-request, stateless MCP revision. +- `legacy`: latest initialization-based MCP revision. -Every stack and workflow selector uses semantic `--lane` values. Stack -commands accept `builtin`, `external`, or `all` where both lanes are valid: +The default is `MCP_PROTOCOL_VERSION`, then `modern`. Dated revisions are +internal wire values, not operational CLI options. + +Use `--help` at any level for the authoritative interface, such as +`cf-integration stack --help` or `cf-integration load --help`. + +## Commands + +Test workflows prepare and clean up their required stack. Use `stack` when you +want a persistent stack for manual work. + +### `stack` ```bash -cf-integration stack up --lane external +# Start one lane +cf-integration stack up --lane builtin cf-integration stack up --lane external --fresh + +# Inspect one lane cf-integration stack status --lane external +cf-integration stack logs --lane external +cf-integration stack logs --lane external nginx cf-integration stack config --lane external + +# Stop one or both lanes +cf-integration stack down --lane builtin cf-integration stack down --lane all cf-integration stack down --lane all --volumes ``` -`stack down --volumes` is the explicit destructive reset. Managed workflows -preserve the primary failure, attempt every token and stack cleanup, and report -all cleanup failures. +`up --fresh` removes existing volumes before starting. `logs` follows all +services unless service names are supplied. `config` prints merged Compose +configuration. `down --volumes` also removes persistent volumes. + +### `probe` + +Probe one public MCP route, including discovery or initialization, +`tools/list`, a safe `tools/call`, authentication, and backend identity. + +```bash +cf-integration probe [--lane builtin|external] \ + [--protocol-version modern|legacy] +``` + +### `load` -Probe, load, and Inspector use the same routed lanes: +Run Locust against one public MCP route: ```bash -cf-integration probe --lane external --protocol-version modern -cf-integration load --lane builtin --smoke -cf-integration debug inspect --lane external --method tools/list +cf-integration load [--lane builtin|external] \ + [--protocol-version modern|legacy] [--smoke] \ + [--users N] [--spawn-rate N] [--run-time DURATION] + +# Compare both lanes for two minutes +cf-integration load --lane builtin --protocol-version legacy \ + --users 10 --spawn-rate 2 --run-time 2m +cf-integration load --lane external --protocol-version legacy \ + --users 10 --spawn-rate 2 --run-time 2m ``` -Live and conformance additionally support the direct `fixture-direct` lane. +`--smoke` selects a short smoke workload. Duration accepts positive `h`, +`m`, and `s` groups such as `2m30s` or `1h30m`. Defaults come from +`LOCUST_USERS`, `LOCUST_SPAWN_RATE`, and `LOCUST_RUN_TIME`. + +### `live` + +Run the managed upstream control-plane test groups: `mcp` for Fast Time MCP +routes, `rbac` for authorization and transports, `protocol` for +protocol-specific behavior, or `all` (the default). ```bash -cf-integration live --lane external --group mcp -cf-integration live --lane fixture-direct --group protocol \ - --protocol-version legacy +cf-integration live [--lane fixture-direct|builtin|external] \ + [--protocol-version modern|legacy] [--group mcp|rbac|protocol|all] + +cf-integration live --lane builtin --protocol-version legacy --group all +cf-integration live --lane fixture-direct \ + --protocol-version legacy --group protocol +``` +### `conformance` + +`run` executes the pinned official suite and compares it with checked-in +baselines. With no options it runs all three lanes using a modern client against +legacy and modern fixture servers. + +```bash cf-integration conformance run + +# Repeat selectors to build a matrix cf-integration conformance run \ - --client-era legacy \ - --client-era modern \ - --server-era legacy \ - --server-era modern + --lane fixture-direct --lane builtin --lane external \ + --client-era legacy --client-era modern \ + --server-era legacy --server-era modern + +# Replace selected baselines only after every selected run succeeds cf-integration conformance run --server-era dual --bless -cf-integration conformance report -cf-integration --version ``` -No command accepts `--topology`. The direct fixture spelling is only -`fixture-direct`. Probe, load, live, and Inspector use `--protocol-version`; -conformance uses the explicit `--client-era` and `--server-era` matrix axes. - -Operational protocol selection is semantic: `modern` maps to the latest -per-request revision and `legacy` maps to the latest initialization-based -revision. Exact date revisions remain internal wire values and conformance -matrix dimensions. - -## MCP and conformance behavior - -One MCP client owns endpoint construction, authorization, sessions, stateful -and stateless headers, JSON/SSE parsing, backend identity validation, response -limits, timeouts, and secret redaction. - -The session-oriented probe performs initialize, `notifications/initialized`, -`tools/list`, and one safe `tools/call`. The stateless probe performs -`server/discover`, attaches `Mcp-Method` and `Mcp-Name` routing headers, and -performs the same safe checks without a session. Both verify unauthenticated -rejection and external dataplane backend identity. - -The official runner is pinned to -`@modelcontextprotocol/conformance@0.2.0-alpha.11`. Its TypeScript fixture is -built from revision `c321dd32035556e6769d3724a8ee97d87c3faaac`. A default run -starts workflow-owned stacks and runs both conformance directions. The server -suite sends the official client directly to the fixture and through the -built-in and external dataplane routes. For protocol `2026-07-28`, the client -suite also makes the external dataplane send requests to the official scenario -servers. The four downstream scenarios are `tools_call`, `request-metadata`, -`http-standard-headers`, and `http-custom-headers`; they run automatically -whenever the `external` lane is selected. The workflow records raw official -results without suppression, writes deterministic comparisons, and continues -through every expanded client-revision/server-era combination before returning -one aggregated result. `dual` is supported only when selected explicitly. - -The client and fixture-server era selections are independent: +`--client-era` and `--server-era` accept `legacy`, `modern`, or `dual`. +`--results-dir`, `--baseline-dir`, and `--output-dir` override artifact +locations. + +`report` regenerates Markdown comparisons from existing results without +running the suite: ```bash -cf-integration conformance run \ - --client-era legacy \ - --client-era modern \ - --server-era legacy \ - --server-era modern +cf-integration conformance report +cf-integration conformance report \ + --results-dir .integration/conformance --output-dir reports/conformance ``` -Client `legacy` expands to `2025-06-18` and `2025-11-25`; `modern` expands to -`2026-07-28`; and `dual` expands to all three verified client revisions. The -expanded client revisions and selected server eras form a Cartesian product. -Artifacts default below `CF_INTEGRATION_DIR/conformance///` -and reports below `reports/conformance///`. -Server artifacts retain the lane directly below the era. Client artifacts and -reports use `client/external-data-plane/` below the era. `--results-dir`, -`--baseline-dir`, and `--output-dir` override those roots. - -Baselines use this strict layout: - -```text -tests/conformance/baselines/ - / - / - fixture-direct.yml - built-in-data-plane.yml - external-data-plane.yml - client/ - external-data-plane.yml -``` +### `debug` -Each file contains sorted `FAILURE` and `WARNING` check identities. They are -required to distinguish expected failures from regressions and are embedded -for installed binaries. Every completed lane is printed in nextest style even -when a later lane fails operationally. The direct fixture is gated -independently; findings reproduced there are subtracted from routed lanes -before server comparison. Client findings are gated independently without -fixture subtraction. Unexpected, stale, unknown, malformed, incomplete, -missing, and operational results fail the matrix. `--bless` replaces all -selected server and client baselines in one directory transaction only after -every combination succeeds. Operational lane failures render as unconditional -`FAIL` rows, count in the nextest-style summary, and cannot be blessed. Outside -a developer checkout, an omitted -`--baseline-dir` writes blessed baselines beneath the current workspace rather -than modifying embedded assets. Server comparison regeneration discovers every -protocol/era partition beneath the selected result root and accepts -`--results-dir` and `--output-dir`. - -## Canonical configuration - -Copy `.env.example` to `.env`. Process values override the file. +`inspect` runs an MCP Inspector method against one routed lane. The method +defaults to `tools/list`, and the server defaults to the Fast Time fixture. ```bash -CF_INTEGRATION_ROOT=/path/to/contextforge-dev-tools -CF_INTEGRATION_DIR=.integration -CF_MCP_LANE=external - -CF_CONTROLPLANE_REPO=https://github.com/IBM/mcp-context-forge.git -CF_CONTROLPLANE_REF=main -CF_CONTROLPLANE_VERSION=main - -CF_DATAPLANE_REPO=https://github.com/contextforge-org/contextforge-data-plane.git -CF_DATAPLANE_REF= -CF_DATAPLANE_IMAGE=ghcr.io/contextforge-org/contextforge-data-plane:latest -CF_DATAPLANE_PLATFORM=auto - -CF_COMPOSE_BUILD=auto -CF_FAST_TIME_EXPECTED_IMAGE=ghcr.io/ibm/cfex-mcp-fast-time-server:latest -CF_FAST_TIME_SERVER_ID=9779b6698cbd4b4995ee04a4fab38737 - -MCP_CLI_BASE_URL=http://127.0.0.1:8080 -MCP_PROTOCOL_VERSION=modern -MCP_SERVER_ID=9779b6698cbd4b4995ee04a4fab38737 - -PLATFORM_ADMIN_EMAIL=admin@example.com -PLATFORM_ADMIN_PASSWORD= -MCPGATEWAY_BEARER_TOKEN= +cf-integration debug inspect --lane external \ + --protocol-version modern --method tools/list +cf-integration debug inspect --lane builtin \ + --protocol-version legacy --server-id ``` -`CF_COMPOSE_BUILD=auto` pulls or reuses prebuilt images and builds only an -explicit source data plane when required. `true` always builds; `false` never -builds. Published mode tracks both repositories' main-branch images. The -dataplane uses its floating `:latest` tag. The control plane uses the -commit-tagged image for the freshly fetched `origin/main` revision because -upstream reserves `:latest` for releases. Stack startup pulls changes; -incompatible main images make the workflow fail instead of selecting an older -pair. - -CI jobs that package the code under test as a local image can opt out of -registry access with `CF_CONTROLPLANE_PULL_POLICY=never` or -`CF_DATAPLANE_PULL_POLICY=never`. This is never the default: the selected image -must already be loaded in Docker, and startup fails if it is absent. - -Compose requires `JWT_SECRET_KEY` and `AUTH_ENCRYPTION_SECRET`. If either is -unset, a runtime-backed action generates stable values under -`CF_INTEGRATION_DIR`. Canonical configuration is exported internally as the -upstream Compose adapter names `IMAGE_LOCAL` and `FAST_TIME_IMAGE`; those names -are not accepted as inputs. - -Without `MCPGATEWAY_BEARER_TOKEN`, dataplane workflows issue a one-day -server-scoped catalog token and revoke it during session cleanup. A caller -supplied token is never revoked by the harness. - -## Package layout - -One root package publishes exactly one binary, `cf-integration`. All concern -modules remain private implementation details: - -```text -src/infrastructure/ config, assets, processes, checkouts, Compose plans -src/mcp/ unified MCP client, protocol, auth proxy, probe -src/conformance/ fixture, strict baselines, results, comparisons -src/performance/ Locust settings, commands, and report auditing -src/runtime/live/ upstream live-test workflow -src/runtime/stack/ stack lifecycle and source ownership -src/runtime/conformance/ conformance orchestration and reports -src/runtime/performance/ performance workflow orchestration -src/runtime/probe.rs probe workflow orchestration -src/runtime/session.rs shared managed stack and credential scope -src/runtime/mod.rs thin action dispatcher -docker/ embedded Compose and nginx assets -scripts/ embedded runtime adapters -tests/conformance/ embedded expected-result baselines +`token` prints a token from an already-running control plane. `scoped` +creates the minimum catalog token used by public MCP tests; `admin` creates a +platform-admin session token. `--server-id` is valid only for `scoped`. + +```bash +cf-integration debug token --kind scoped +cf-integration debug token --kind scoped --server-id +cf-integration debug token --kind admin ``` -The Bruno collection under `manual-tests/mcp-manual-test-tools/` is an -intentional lower stack layer for manual diagnosis. It remains in the -repository and is excluded from the published crate payload. +## Configuration and artifacts -## Development and release +Copy `.env.example` to `.env`; process environment values override it. -```bash -cargo fmt --all --check -cargo clippy --all-targets -- -D warnings -cargo test --all-targets -cargo package --locked -``` +| Variable | Purpose | Default | +| --- | --- | --- | +| `CF_MCP_LANE` | Routed lane | `external` | +| `MCP_PROTOCOL_VERSION` | Protocol mode | `modern` | +| `CF_INTEGRATION_DIR` | Checkouts, state, and load reports | `.integration` | +| `CF_DATAPLANE_REF` | Optional local dataplane Git ref | unset | +| `LOCUST_*` | Users, spawn rate, and duration | `100`, `10`, `5m` | + +See [`.env.example`](.env.example) for every setting. Missing Compose secrets +are generated under `CF_INTEGRATION_DIR`. Workflow-created tokens are revoked +during cleanup; a caller-supplied `MCPGATEWAY_BEARER_TOKEN` is never revoked. -Pull requests run this quality gate plus native tests on Linux, macOS, and -Windows. Releases build and smoke-test all six ARM64/x86-64 Linux, macOS, and -Windows candidates before publishing the crate or tag. Prevalidated archives, -SHA-256 files, and GitHub artifact attestations are published afterward. +Installed binaries embed their runtime assets. Set `CF_INTEGRATION_ROOT` to +force a developer checkout. Load reports default below +`CF_INTEGRATION_DIR/reports/load`; conformance results below +`CF_INTEGRATION_DIR/conformance`; and conformance Markdown below +`reports/conformance`. From 233ebd1ba8cdb3de5efd93f9c932c36d54f787fd Mon Sep 17 00:00:00 2001 From: lucarlig Date: Thu, 3 Sep 2026 11:56:32 +0100 Subject: [PATCH 06/10] feat: isolate standalone dataplane load tests Signed-off-by: lucarlig --- Cargo.toml | 1 + README.md | 14 +- docker/docker-compose.cf-dataplane.yaml | 6 +- docker/docker-compose.cf-integration.yaml | 32 ++++ scripts/locustfile_mcp.py | 7 +- scripts/prepare_standalone_config.py | 117 +++++++++++++++ src/app.rs | 13 +- src/app_tests.rs | 45 ++++++ src/cli.rs | 4 + src/cli_public_tests.rs | 20 +++ src/infrastructure/assets.rs | 1 + .../compose_integration_tests.rs | 10 ++ src/infrastructure/stack.rs | 24 +++ src/infrastructure/stack_integration_tests.rs | 21 +++ src/performance/python_adapter_tests.rs | 34 +++++ src/runtime/inspect.rs | 2 +- src/runtime/performance/mod.rs | 128 ++++++++-------- src/runtime/probe.rs | 2 +- src/runtime/session.rs | 137 ++++++++++++++++-- src/runtime/stack/mod.rs | 2 +- 20 files changed, 538 insertions(+), 82 deletions(-) create mode 100644 scripts/prepare_standalone_config.py diff --git a/Cargo.toml b/Cargo.toml index 3604396..f1d3121 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ include = [ "/src/**", "/docker/**", "/scripts/locustfile_mcp.py", + "/scripts/prepare_standalone_config.py", "/scripts/live_protocol/sitecustomize.py", "/scripts/conformance/write_client_config.py", "/tests/conformance/baselines/**", diff --git a/README.md b/README.md index c304af5..f77851b 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ Run Locust against one public MCP route: ```bash cf-integration load [--lane builtin|external] \ - [--protocol-version modern|legacy] [--smoke] \ + [--protocol-version modern|legacy] [--standalone] [--smoke] \ [--users N] [--spawn-rate N] [--run-time DURATION] # Compare both lanes for two minutes @@ -109,12 +109,24 @@ cf-integration load --lane builtin --protocol-version legacy \ --users 10 --spawn-rate 2 --run-time 2m cf-integration load --lane external --protocol-version legacy \ --users 10 --spawn-rate 2 --run-time 2m + +# Measure only the external dataplane request path +cargo run -- load --lane external --protocol-version legacy --standalone \ + --users 10 --spawn-rate 2 --run-time 2m ``` `--smoke` selects a short smoke workload. Duration accepts positive `h`, `m`, and `s` groups such as `2m30s` or `1h30m`. Defaults come from `LOCUST_USERS`, `LOCUST_SPAWN_RATE`, and `LOCUST_RUN_TIME`. +`--standalone` is valid only with `--lane external`. Each run starts the full +stack to issue a scoped token, starts an isolated current-protocol MCP fixture, +then stops the control-plane gateway before Locust begins. A fresh mock config +for the token subject is written through the running dataplane's own serializer +on every run, so Redis receives the dataplane's current MessagePack schema. The +snapshot is non-expiring for the load duration; traffic does not depend on the +control-plane publisher or its schema-sync timing. + ### `live` Run the managed upstream control-plane test groups: `mcp` for Fast Time MCP diff --git a/docker/docker-compose.cf-dataplane.yaml b/docker/docker-compose.cf-dataplane.yaml index 208f1f4..eafa479 100644 --- a/docker/docker-compose.cf-dataplane.yaml +++ b/docker/docker-compose.cf-dataplane.yaml @@ -27,6 +27,8 @@ services: # Requires a control-plane image with configurable publisher interval; # older images ignore the variable (60s behavior). DATAPLANE_PUBLISHER_INTERVAL_SECONDS: ${CF_DATAPLANE_PUBLISHER_INTERVAL_SECONDS:-2} + volumes: + - ${CF_INTEGRATION_ROOT:?Set CF_INTEGRATION_ROOT to the integration harness root}/scripts/prepare_standalone_config.py:/opt/contextforge-integration/prepare_standalone_config.py:ro dataplane: image: ${CF_DATAPLANE_IMAGE:?Set CF_DATAPLANE_IMAGE to the cf-dataplane image tag} @@ -51,8 +53,8 @@ services: CONTEXTFORGE_DATA_PLANE_TOKEN_SECRET: ${JWT_SECRET_KEY:-my-test-key-but-now-longer-than-32-bytes} # The published image currently includes its non-production `with_tools` # bootstrap routes, whose clap model requires an RSA signing-key path. - # This harness never exposes or calls those routes and uses control-plane - # catalog tokens, so satisfy the unused path without adding a test key. + # The standalone load helper calls only the internal user-config route; + # token creation stays disabled, so satisfy the unused path without a key. CONTEXTFORGE_DATA_PLANE_TOKEN_VERIFICATION_PRIVATE_KEY: /dev/null CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE: plain-text-or-tls # These two MCP transport settings intentionally retain the historical diff --git a/docker/docker-compose.cf-integration.yaml b/docker/docker-compose.cf-integration.yaml index c965efc..1ca0515 100644 --- a/docker/docker-compose.cf-integration.yaml +++ b/docker/docker-compose.cf-integration.yaml @@ -12,6 +12,37 @@ services: register_fast_time: condition: service_completed_successfully + # A current-protocol upstream for standalone dataplane load tests. The + # profile keeps it out of normal stack, live, and probe workflows. + standalone_load_backend: + profiles: ["standalone-load"] + image: cf-integration/mcp-conformance-server:0.2.0-alpha.11 + build: + context: ${CF_INTEGRATION_ROOT:?Set CF_INTEGRATION_ROOT to the integration harness root} + dockerfile: docker/mcp-conformance-server.Dockerfile + labels: + name: cf-standalone-load-backend + restart: "no" + environment: + PORT: "3000" + MCP_CONFORMANCE_SERVER_ERA: modern + expose: + - "3000" + networks: + mcpnet: + aliases: + - mcp_conformance_server + healthcheck: + test: + - CMD + - node + - -e + - fetch('http://127.0.0.1:3000/mcp').then(response => { if (response.status !== 400) process.exit(1); }).catch(() => process.exit(1)) + interval: 2s + timeout: 2s + retries: 30 + start_period: 2s + locust: volumes: # Harness locustfile with streamable-HTTP content negotiation; the @@ -31,6 +62,7 @@ services: - MCP_SERVER_ID=${MCP_SERVER_ID:-} - MCP_SERVER_IDS=${MCP_SERVER_IDS:-} - MCP_TOOL_NAMES=${MCP_TOOL_NAMES:-} + - MCP_SKIP_TOOL_LIST=${MCP_SKIP_TOOL_LIST:-false} - MCP_PROTOCOL_VERSION=${MCP_PROTOCOL_VERSION:-2026-07-28} - LOCUST_LOG_LEVEL=${LOCUST_LOG_LEVEL:-INFO} command: diff --git a/scripts/locustfile_mcp.py b/scripts/locustfile_mcp.py index cae4b7b..4b0bdec 100644 --- a/scripts/locustfile_mcp.py +++ b/scripts/locustfile_mcp.py @@ -10,6 +10,7 @@ MCP_SERVER_ID virtual server id (dataplane only) MCPGATEWAY_BEARER_TOKEN bearer token (required) MCP_TOOL_NAMES optional comma-separated tools to call + MCP_SKIP_TOOL_LIST true when direct tool aliases are supplied LOCUST_REQUEST_TIMEOUT_SECONDS positive finite per-request timeout (default 60) """ from __future__ import annotations @@ -44,6 +45,7 @@ def _request_timeout_seconds() -> float: REQUEST_TIMEOUT_SECONDS = _request_timeout_seconds() _TOOL_ARGUMENTS = { + "test_simple_text": {}, "echo": {"message": "cf-integration"}, "fast_time_echo": {"message": "cf-integration"}, "fast-time-echo": {"message": "cf-integration"}, @@ -182,6 +184,7 @@ def validate_result(method: str, result) -> dict: MCP_STACK_MODE = os.environ.get("MCP_STACK_MODE", "dataplane") BEARER_TOKEN = os.environ.get("MCPGATEWAY_BEARER_TOKEN", "") TOOL_NAMES = [name.strip() for name in os.environ.get("MCP_TOOL_NAMES", "").split(",") if name.strip()] +SKIP_TOOL_LIST = os.environ.get("MCP_SKIP_TOOL_LIST", "false").lower() == "true" def safe_diagnostic(value) -> str: @@ -248,7 +251,7 @@ def on_start(self): raise RuntimeError("initialize response did not include Mcp-Session-Id") if not STATELESS: self._mcp_notification("notifications/initialized", None, name="MCP initialized") - if not self._tool_names: + if not self._tool_names and not SKIP_TOOL_LIST: listed = self._mcp_request("tools/list", {}, name="MCP tools/list") if listed: self._tool_names = [ @@ -409,6 +412,8 @@ def _mcp_notification(self, method: str, params: dict | None, name: str) -> None @task(5) def tools_list(self): + if SKIP_TOOL_LIST: + return self._mcp_request("tools/list", {}, name="MCP tools/list") @task(10) diff --git a/scripts/prepare_standalone_config.py b/scripts/prepare_standalone_config.py new file mode 100644 index 0000000..bbabc05 --- /dev/null +++ b/scripts/prepare_standalone_config.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Publish an isolated load fixture through the dataplane's own serializer.""" + +from __future__ import annotations + +import base64 +import json +import os +import sys +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +DATAPLANE_CONFIG_URL = ( + "http://dataplane:4445/contextforge-rs/admin/userconfigs/{subject}" +) +BACKEND_URL = "http://mcp_conformance_server:3000/mcp" +BACKEND_NAME = "standalone-load" +TOOL_NAMES = ["test_simple_text"] + + +def token_subject(token: str) -> str: + parts = token.split(".") + if len(parts) != 3: + raise SystemExit("MCPGATEWAY_BEARER_TOKEN is not a JWT") + payload = parts[1] + ("=" * (-len(parts[1]) % 4)) + try: + claims = json.loads(base64.urlsafe_b64decode(payload)) + except (ValueError, json.JSONDecodeError) as error: + raise SystemExit("MCPGATEWAY_BEARER_TOKEN has invalid claims") from error + subject = claims.get("sub") + if not isinstance(subject, str) or not subject: + raise SystemExit("MCPGATEWAY_BEARER_TOKEN has no string subject") + return subject + + +def prepare_config(server_id: str, protocol_version: str) -> dict: + if not server_id: + raise SystemExit("virtual-host-id must not be empty") + return { + "virtual_hosts": { + server_id: { + "backends": { + BACKEND_NAME: { + "name": BACKEND_NAME, + "url": BACKEND_URL, + "mcp_protocol_version": protocol_version, + "passthrough_headers": [], + "add_headers": {}, + "remove_headers": [], + "tool_name_aliases": [ + { + "downstream_prefixed_name": name, + "upstream_name": name, + } + for name in TOOL_NAMES + ], + "resource_uri_aliases": [], + "prompt_name_aliases": [], + "completion": {}, + "tool_schemas": {name: {} for name in TOOL_NAMES}, + } + } + } + } + } + + +def publish_config(subject: str, config: dict) -> None: + endpoint = DATAPLANE_CONFIG_URL.format(subject=quote(subject, safe="")) + request = Request( + endpoint, + data=json.dumps(config).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=30) as response: + if response.status != 202: + raise SystemExit( + f"dataplane config serializer returned HTTP {response.status}" + ) + except HTTPError as error: + detail = error.read(512).decode(errors="replace").strip() + raise SystemExit( + f"dataplane config serializer returned HTTP {error.code}: {detail}" + ) from error + except URLError as error: + raise SystemExit(f"dataplane config serializer is unavailable: {error.reason}") from error + + +def main() -> None: + import msgpack + import redis + + if len(sys.argv) != 3: + raise SystemExit( + "usage: prepare_standalone_config.py " + ) + server_id, protocol_version = sys.argv[1:] + token = os.environ.get("MCPGATEWAY_BEARER_TOKEN", "") + if not token: + raise SystemExit("MCPGATEWAY_BEARER_TOKEN is required") + subject = token_subject(token) + key = msgpack.dumps(("UserConfig", subject), use_bin_type=True) + client = redis.Redis.from_url( + os.environ.get("REDIS_URL", "redis://redis:6379/0"), + decode_responses=False, + ) + publish_config(subject, prepare_config(server_id, protocol_version)) + if client.ttl(key) != -1: + raise SystemExit("dataplane serializer did not persist the Redis snapshot") + print(json.dumps(TOOL_NAMES, separators=(",", ":"))) + + +if __name__ == "__main__": + main() diff --git a/src/app.rs b/src/app.rs index 4f4ba22..2c5992c 100644 --- a/src/app.rs +++ b/src/app.rs @@ -77,7 +77,13 @@ impl Action { protocol_version, .. }) => lane_and_protocol(*topology, protocol_version), - Self::Load(args) => lane_and_protocol(args.topology, &args.protocol_version), + Self::Load(args) => { + let mut summary = lane_and_protocol(args.topology, &args.protocol_version); + if args.standalone { + summary.push_str("\nControl plane: disabled during load"); + } + summary + } Self::Live { lane, protocol_version, @@ -226,6 +232,7 @@ pub(crate) enum StackAction { pub(crate) struct ResolvedLoadArgs { pub(crate) topology: StackMode, pub(crate) protocol_version: ProtocolVersion, + pub(crate) standalone: bool, pub(crate) request: LoadRequest, } @@ -302,6 +309,9 @@ pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result { let topology = resolve_lane(args.target.lane, environment)?; + if args.standalone && topology != StackMode::Dataplane { + bail!("--standalone requires --lane external"); + } Ok(Action::Load(ResolvedLoadArgs { topology, protocol_version: resolve_protocol_version( @@ -309,6 +319,7 @@ pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result Self { + Self { + command: project.command(["stop", "--timeout", "5", service]), + } + } + + /// Builds a Compose command that restarts one previously stopped service. + #[must_use] + pub(crate) fn start_service(project: ComposeProject, service: &str) -> Self { + Self { + command: project.command(["start", service]), + } + } + + /// Builds a Compose command that restarts one service without its dependencies. + #[must_use] + pub(crate) fn restart_service(project: ComposeProject, service: &str) -> Self { + Self { + command: project.command(["restart", "--timeout", "5", service]), + } + } + /// Builds a Compose cleanup command. #[must_use] pub(crate) fn cleanup(project: ComposeProject, kind: CleanupKind) -> Self { diff --git a/src/infrastructure/stack_integration_tests.rs b/src/infrastructure/stack_integration_tests.rs index f5c52cc..4ca3e29 100644 --- a/src/infrastructure/stack_integration_tests.rs +++ b/src/infrastructure/stack_integration_tests.rs @@ -176,6 +176,27 @@ fn controlplane_up_does_not_activate_locust_profile_when_ui_is_disabled() { #[test] fn cleanup_status_logs_and_config_use_typed_compose_commands() { let dataplane_project = project(StackMode::Dataplane); + assert!(ends_with( + &args(StackCommandPlan::stop_service( + dataplane_project.clone(), + "gateway" + )), + &["stop", "--timeout", "5", "gateway"] + )); + assert!(ends_with( + &args(StackCommandPlan::start_service( + dataplane_project.clone(), + "gateway" + )), + &["start", "gateway"] + )); + assert!(ends_with( + &args(StackCommandPlan::restart_service( + dataplane_project.clone(), + "dataplane" + )), + &["restart", "--timeout", "5", "dataplane"] + )); let down = StackCommandPlan::cleanup(dataplane_project.clone(), CleanupKind::Down); assert!(ends_with( &args(down.clone()), diff --git a/src/performance/python_adapter_tests.rs b/src/performance/python_adapter_tests.rs index a8d546c..888c633 100644 --- a/src/performance/python_adapter_tests.rs +++ b/src/performance/python_adapter_tests.rs @@ -38,11 +38,45 @@ fn locust_adapter_and_compose_overlay_do_not_reference_the_removed_helper() { "the load container receives a bearer token and must not receive the signing key" ); assert!(compose.contains("MCP_PROTOCOL_VERSION=${MCP_PROTOCOL_VERSION:-2026-07-28}")); + assert!(compose.contains("standalone_load_backend:")); + assert!(compose.contains("profiles: [\"standalone-load\"]")); + assert!(compose.contains("MCP_CONFORMANCE_SERVER_ERA: modern")); assert!( compose.contains("LOCUST_REQUEST_TIMEOUT_SECONDS=${LOCUST_REQUEST_TIMEOUT_SECONDS:-60}") ); } +#[test] +fn standalone_config_helper_uses_token_subject_and_selected_protocol() { + let code = r#" +import base64 +import json +import prepare_standalone_config as helper + +claims = base64.urlsafe_b64encode(json.dumps({"sub": "user-123"}).encode()).decode().rstrip("=") +assert helper.token_subject(f"header.{claims}.signature") == "user-123" + +prepared = helper.prepare_config("server-123", "2026-07-28") +backend = prepared["virtual_hosts"]["server-123"]["backends"]["standalone-load"] +assert backend["url"] == "http://mcp_conformance_server:3000/mcp" +assert backend["mcp_protocol_version"] == "2026-07-28" +assert backend["tool_name_aliases"] == [{"downstream_prefixed_name": "test_simple_text", "upstream_name": "test_simple_text"}] +assert backend["tool_schemas"] == {"test_simple_text": {}} +"#; + let output = Command::new(python()) + .arg("-c") + .arg(code) + .env("PYTHONPATH", scripts_dir()) + .output() + .expect("Python helper test should run"); + + assert!( + output.status.success(), + "standalone config helper failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + fn locust_stub() -> TempDir { let directory = tempfile::tempdir().expect("temporary Python stub should be created"); fs::write( diff --git a/src/runtime/inspect.rs b/src/runtime/inspect.rs index bdfb431..97e9d13 100644 --- a/src/runtime/inspect.rs +++ b/src/runtime/inspect.rs @@ -29,7 +29,7 @@ impl RuntimeContext { .unwrap_or_else(|| self.default_server_id()) .to_owned(); let operation_server_id = server_id.clone(); - self.with_managed_authenticated_target(mode, &server_id, |token| async move { + self.with_managed_authenticated_target(mode, &server_id, false, |token, _| async move { let endpoint = GatewayClient::new( gateway_topology(mode), self.base_url()?, diff --git a/src/runtime/performance/mod.rs b/src/runtime/performance/mod.rs index 373b494..a11e5ee 100644 --- a/src/runtime/performance/mod.rs +++ b/src/runtime/performance/mod.rs @@ -9,71 +9,81 @@ impl RuntimeContext { let server_id = self.default_server_id().to_owned(); let operation_server_id = server_id.clone(); let preparation = Activity::spinner("Preparing performance stack"); - self.with_managed_authenticated_target(args.topology, &server_id, |token| async move { - let command = LocustCommand::new_with_protocol_version( - &self.config, - args.topology, - &settings, - &token, - (args.topology == StackMode::Dataplane).then_some(operation_server_id.as_str()), - args.protocol_version.wire_version(), - ) - .map_err(AppFailure::from)?; - let command_spec = - self.compose_environment(command.command().clone(), args.topology, true)?; - let output_log = command.report_dir().join("locust.log"); - fs::write(&output_log, []) - .with_context(|| format!("failed to clear Locust output log {output_log:?}")) + self.with_managed_authenticated_target( + args.topology, + &server_id, + args.standalone, + |token, standalone_tool_names| async move { + let command = LocustCommand::new_with_protocol_version( + &self.config, + args.topology, + &settings, + &token, + (args.topology == StackMode::Dataplane).then_some(operation_server_id.as_str()), + args.protocol_version.wire_version(), + ) .map_err(AppFailure::from)?; - preparation.finish(true); + let mut command_spec = + self.compose_environment(command.command().clone(), args.topology, true)?; + if args.standalone { + command_spec = command_spec + .env("MCP_TOOL_NAMES", standalone_tool_names.join(",")) + .env("MCP_SKIP_TOOL_LIST", "true"); + } + let output_log = command.report_dir().join("locust.log"); + fs::write(&output_log, []) + .with_context(|| format!("failed to clear Locust output log {output_log:?}")) + .map_err(AppFailure::from)?; + preparation.finish(true); - let description = format!( - "Running load test ({} users, {}/s, {})", - settings.users(), - settings.spawn_rate(), - settings.run_time(), - ); - let activity = Activity::spinner(description); - let started = std::time::Instant::now(); - let process_result = self - .runner - .run_to_log(&command_spec, &output_log) - .map_err(AppFailure::from); - let result = finalize_locust_run(process_result, command.report_dir(), &token); - let elapsed = started.elapsed(); - activity.finish(result.is_ok()); + let description = format!( + "Running load test ({} users, {}/s, {})", + settings.users(), + settings.spawn_rate(), + settings.run_time(), + ); + let activity = Activity::spinner(description); + let started = std::time::Instant::now(); + let process_result = self + .runner + .run_to_log(&command_spec, &output_log) + .map_err(AppFailure::from); + let result = finalize_locust_run(process_result, command.report_dir(), &token); + let elapsed = started.elapsed(); + activity.finish(result.is_ok()); - let status = if result.is_ok() { - TestStatus::Pass - } else { - TestStatus::Fail - }; - println!( - "{}", - OutputStyle::stdout().test_result( - status, - &format!("performance::{}", args.topology.lane_label()), - Some(elapsed), - None, - ) - ); - if result.is_ok() { + let status = if result.is_ok() { + TestStatus::Pass + } else { + TestStatus::Fail + }; println!( "{}", - OutputStyle::stdout().info(&format!( - "Report: {}", - command.report_dir().join("locust_report.html").display() - )) - ); - } else if output_log.is_file() { - eprintln!( - "{}", - OutputStyle::stderr() - .failure(&format!("Load output: {}", output_log.display())) + OutputStyle::stdout().test_result( + status, + &format!("performance::{}", args.topology.lane_label()), + Some(elapsed), + None, + ) ); - } - result - }) + if result.is_ok() { + println!( + "{}", + OutputStyle::stdout().info(&format!( + "Report: {}", + command.report_dir().join("locust_report.html").display() + )) + ); + } else if output_log.is_file() { + eprintln!( + "{}", + OutputStyle::stderr() + .failure(&format!("Load output: {}", output_log.display())) + ); + } + result + }, + ) .await } } diff --git a/src/runtime/probe.rs b/src/runtime/probe.rs index 760f4f5..1f2f50a 100644 --- a/src/runtime/probe.rs +++ b/src/runtime/probe.rs @@ -9,7 +9,7 @@ impl RuntimeContext { protocol_version: &ProtocolVersion, ) -> AppResult<()> { let server_id = self.default_server_id().to_owned(); - self.with_managed_authenticated_target(topology, &server_id, |token| async { + self.with_managed_authenticated_target(topology, &server_id, false, |token, _| async { let config = ProbeConfig { mode: gateway_topology(topology), base_url: self.base_url()?.to_owned(), diff --git a/src/runtime/session.rs b/src/runtime/session.rs index 58e67fb..e5f97af 100644 --- a/src/runtime/session.rs +++ b/src/runtime/session.rs @@ -21,20 +21,31 @@ return 0 struct ManagedSessionScope<'a, R> { runtime: &'a RuntimeContext, topology: StackMode, + standalone: bool, token: Option, } impl<'a, R: ProcessRunner> ManagedSessionScope<'a, R> { - fn new(runtime: &'a RuntimeContext, topology: StackMode) -> Self { + fn new(runtime: &'a RuntimeContext, topology: StackMode, standalone: bool) -> Self { Self { runtime, topology, + standalone, token: None, } } async fn finish(self, primary: AppResult<()>) -> AppResult<()> { let mut cleanup_failures = Vec::new(); + if self.standalone + && self + .token + .as_ref() + .is_some_and(|token| token.catalog_id.is_some()) + && let Err(error) = self.runtime.restore_control_plane_gateway().await + { + cleanup_failures.push(error); + } if let Some(token) = self.token.as_ref() && let Err(error) = self.runtime.revoke_managed_token(token).await { @@ -61,7 +72,7 @@ impl RuntimeContext { F: FnOnce() -> Fut, Fut: Future>, { - let scope = ManagedSessionScope::new(self, topology); + let scope = ManagedSessionScope::new(self, topology, false); let primary = match self.stack_up(topology, false).await { Ok(()) => match self.prepare_test_target(topology, server_id).await { Ok(()) => operation().await, @@ -76,20 +87,37 @@ impl RuntimeContext { &self, topology: StackMode, server_id: &str, + standalone: bool, operation: F, ) -> AppResult<()> where - F: FnOnce(String) -> Fut, + F: FnOnce(String, Vec) -> Fut, Fut: Future>, { - let mut scope = ManagedSessionScope::new(self, topology); + if standalone && topology != StackMode::Dataplane { + return Err(AppFailure::from(anyhow!( + "standalone mode requires the external lane" + ))); + } + let mut scope = ManagedSessionScope::new(self, topology, standalone); let primary = match self.stack_up(topology, false).await { - Ok(()) => match self.prepare_test_target(topology, server_id).await { + Ok(()) => match self + .prepare_authenticated_target(topology, server_id, standalone) + .await + { Ok(()) => match self.managed_bearer_token(topology, server_id).await { Ok(token) => { let value = token.value.clone(); scope.token = Some(token); - operation(value).await + let tool_names = if standalone { + self.isolate_external_dataplane(server_id, &value).await + } else { + Ok(Vec::new()) + }; + match tool_names { + Ok(tool_names) => operation(value, tool_names).await, + Err(error) => Err(error), + } } Err(error) => Err(error), }, @@ -100,6 +128,19 @@ impl RuntimeContext { scope.finish(primary).await } + async fn prepare_authenticated_target( + &self, + topology: StackMode, + server_id: &str, + standalone: bool, + ) -> AppResult<()> { + if standalone { + self.ensure_other_stack_stopped(topology)?; + return Ok(()); + } + self.prepare_test_target(topology, server_id).await + } + pub(super) async fn prepare_test_target( &self, topology: StackMode, @@ -114,15 +155,7 @@ impl RuntimeContext { pub(super) async fn wait_for_publisher_snapshot(&self, server_id: &str) -> AppResult<()> { let timeout_seconds = self.environment_u64("CF_PUBLISHER_WAIT_SECONDS", 90)?; - let project = required_text( - &self.config.integration_project().value, - "CF_INTEGRATION_PROJECT", - )?; - let redis = self.container_id(project, "redis", false)?.ok_or_else(|| { - AppFailure::from(anyhow!( - "cannot wait for publisher snapshot: the dataplane Redis container is not running" - )) - })?; + let redis = self.dataplane_redis_container()?; let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout_seconds); loop { let command = CommandSpec::new("docker").args([ @@ -152,6 +185,80 @@ impl RuntimeContext { } } + async fn isolate_external_dataplane( + &self, + server_id: &str, + token: &str, + ) -> AppResult> { + let project = self.compose_project(StackMode::Dataplane); + let command = project.command([ + "--profile", + "standalone-load", + "up", + "--detach", + "--wait", + "standalone_load_backend", + ]); + let command = self.compose_environment(command, StackMode::Dataplane, true)?; + self.runner.run(&command)?; + let command = StackCommandPlan::stop_service(project.clone(), "gateway"); + let command = + self.compose_environment(command.command().clone(), StackMode::Dataplane, true)?; + self.runner.run(&command)?; + let command = project.command([ + "run", + "--rm", + "--no-deps", + "-e", + "MCPGATEWAY_BEARER_TOKEN", + "--entrypoint", + "python3", + "gateway", + "/opt/contextforge-integration/prepare_standalone_config.py", + server_id, + ProtocolVersion::Modern.wire_version(), + ]); + let command = self + .compose_environment(command, StackMode::Dataplane, true)? + .env("MCPGATEWAY_BEARER_TOKEN", token); + let tool_names = self.capture_text(&command)?; + let tool_names = serde_json::from_str::>(&tool_names) + .context("standalone config helper returned invalid tool names") + .map_err(AppFailure::from)?; + if tool_names.is_empty() { + return Err(AppFailure::from(anyhow!( + "standalone Redis config for server {server_id} contains no tools" + ))); + } + let command = StackCommandPlan::restart_service(project, "dataplane"); + let command = + self.compose_environment(command.command().clone(), StackMode::Dataplane, true)?; + self.runner.run(&command)?; + self.wait_for_public_endpoint(StackMode::Dataplane, false) + .await?; + Ok(tool_names) + } + + async fn restore_control_plane_gateway(&self) -> AppResult<()> { + let project = self.compose_project(StackMode::Dataplane); + let command = StackCommandPlan::start_service(project, "gateway"); + let command = + self.compose_environment(command.command().clone(), StackMode::Dataplane, true)?; + self.runner.run(&command)?; + self.wait_for_public_endpoint(StackMode::Controlplane, false) + .await + } + + fn dataplane_redis_container(&self) -> AppResult { + let project = required_text( + &self.config.integration_project().value, + "CF_INTEGRATION_PROJECT", + )?; + self.container_id(project, "redis", false)?.ok_or_else(|| { + AppFailure::from(anyhow!("the external lane Redis container is not running")) + }) + } + pub(super) fn environment_u64(&self, key: &str, default: u64) -> AppResult { self.environment_text(key).map_or(Ok(default), |value| { value diff --git a/src/runtime/stack/mod.rs b/src/runtime/stack/mod.rs index 146ae0c..a16df3e 100644 --- a/src/runtime/stack/mod.rs +++ b/src/runtime/stack/mod.rs @@ -172,7 +172,7 @@ impl RuntimeContext { Ok(()) } - async fn wait_for_public_endpoint( + pub(super) async fn wait_for_public_endpoint( &self, mode: StackMode, report_progress: bool, From 0349219ac914180144b45d4982aa2821f16014ad Mon Sep 17 00:00:00 2001 From: lucarlig Date: Thu, 3 Sep 2026 14:41:08 +0100 Subject: [PATCH 07/10] feat: add ephemeral ClickStack observability Signed-off-by: lucarlig --- README.md | 18 ++- docker/clickstack/collector.yaml | 12 ++ ...compose.cf-controlplane-observability.yaml | 15 ++ ...er-compose.cf-dataplane-observability.yaml | 14 ++ docker/docker-compose.cf-telemetry.yaml | 34 +++++ scripts/prepare_standalone_config.py | 13 +- src/app.rs | 5 + src/app_tests.rs | 16 ++ src/cli.rs | 4 + src/cli_public_tests.rs | 11 ++ src/infrastructure/compose.rs | 31 ++++ .../compose_integration_tests.rs | 139 +++++++++++++++++- src/infrastructure/stack.rs | 10 +- src/infrastructure/stack_integration_tests.rs | 14 +- src/performance/python_adapter_tests.rs | 5 + src/runtime/conformance/mod.rs | 8 +- src/runtime/performance/mod.rs | 3 +- src/runtime/session.rs | 49 +++++- src/runtime/stack/mod.rs | 41 +++++- 19 files changed, 423 insertions(+), 19 deletions(-) create mode 100644 docker/clickstack/collector.yaml create mode 100644 docker/docker-compose.cf-controlplane-observability.yaml create mode 100644 docker/docker-compose.cf-dataplane-observability.yaml create mode 100644 docker/docker-compose.cf-telemetry.yaml diff --git a/README.md b/README.md index f77851b..fe65137 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,15 @@ cf-integration stack down --lane all --volumes services unless service names are supplied. `config` prints merged Compose configuration. `down --volumes` also removes persistent volumes. +ClickStack starts by default for `stack`, `probe`, `live`, `conformance`, and +`debug inspect`. While the stack or test is running, open the HyperDX UI at + to inspect traces and metrics. On first access, create +a temporary local user; its sources are configured automatically. The external +dataplane exports the exact +HTTP counters, latency histograms, in-flight gauge, and body sizes recorded by +its `HttpMetricsLayer`; allow 30 seconds for its first export. All telemetry +storage is ephemeral and disappears with the stack. + ### `probe` Probe one public MCP route, including discovery or initialization, @@ -102,7 +111,7 @@ Run Locust against one public MCP route: ```bash cf-integration load [--lane builtin|external] \ [--protocol-version modern|legacy] [--standalone] [--smoke] \ - [--users N] [--spawn-rate N] [--run-time DURATION] + [--observability] [--users N] [--spawn-rate N] [--run-time DURATION] # Compare both lanes for two minutes cf-integration load --lane builtin --protocol-version legacy \ @@ -113,11 +122,18 @@ cf-integration load --lane external --protocol-version legacy \ # Measure only the external dataplane request path cargo run -- load --lane external --protocol-version legacy --standalone \ --users 10 --spawn-rate 2 --run-time 2m + +# Inspect traces and native HTTP metrics while using the mocked Redis snapshot +cargo run -- load --lane external --protocol-version legacy --standalone \ + --observability --users 10 --spawn-rate 2 --run-time 2m ``` `--smoke` selects a short smoke workload. Duration accepts positive `h`, `m`, and `s` groups such as `2m30s` or `1h30m`. Defaults come from `LOCUST_USERS`, `LOCUST_SPAWN_RATE`, and `LOCUST_RUN_TIME`. +Observability is disabled for load tests by default to avoid skewing +performance results; pass `--observability` when diagnostics are more important +than an uncontaminated benchmark. `--standalone` is valid only with `--lane external`. Each run starts the full stack to issue a scoped token, starts an isolated current-protocol MCP fixture, diff --git a/docker/clickstack/collector.yaml b/docker/clickstack/collector.yaml new file mode 100644 index 0000000..c4321f1 --- /dev/null +++ b/docker/clickstack/collector.yaml @@ -0,0 +1,12 @@ +# Accept harness telemetry immediately, without depending on HyperDX onboarding. + +service: + pipelines: + traces/integration: + receivers: [otlp/hyperdx] + processors: [memory_limiter, batch] + exporters: [clickhouse] + metrics/integration: + receivers: [otlp/hyperdx] + processors: [memory_limiter, batch] + exporters: [clickhouse] diff --git a/docker/docker-compose.cf-controlplane-observability.yaml b/docker/docker-compose.cf-controlplane-observability.yaml new file mode 100644 index 0000000..24112bf --- /dev/null +++ b/docker/docker-compose.cf-controlplane-observability.yaml @@ -0,0 +1,15 @@ +# OpenTelemetry trace export for the control plane. + +services: + gateway: + environment: + OTEL_ENABLE_OBSERVABILITY: "true" + OTEL_TRACES_EXPORTER: otlp + OTEL_EXPORTER_OTLP_ENDPOINT: http://clickstack:4317 + OTEL_EXPORTER_OTLP_PROTOCOL: grpc + OTEL_EXPORTER_OTLP_INSECURE: "true" + OTEL_SERVICE_NAME: cf-controlplane + OTEL_RESOURCE_ATTRIBUTES: deployment.environment=integration,service.namespace=contextforge + depends_on: + clickstack: + condition: service_healthy diff --git a/docker/docker-compose.cf-dataplane-observability.yaml b/docker/docker-compose.cf-dataplane-observability.yaml new file mode 100644 index 0000000..26ba9b1 --- /dev/null +++ b/docker/docker-compose.cf-dataplane-observability.yaml @@ -0,0 +1,14 @@ +# OpenTelemetry trace export for the external Rust dataplane. + +services: + dataplane: + environment: + CONTEXTFORGE_DATA_PLANE_ENABLE_OPEN_TELEMETRY: "true" + CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_ENDPOINT: http://clickstack:4318/v1/traces + CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_PROTOCOL: http-protobuf + CONTEXTFORGE_DATA_PLANE_OTEL_SERVICE_NAME: cf-dataplane + CONTEXTFORGE_DATA_PLANE_ENABLE_OTEL_METRICS: "true" + CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: http://clickstack:4318/v1/metrics + depends_on: + clickstack: + condition: service_healthy diff --git a/docker/docker-compose.cf-telemetry.yaml b/docker/docker-compose.cf-telemetry.yaml new file mode 100644 index 0000000..40a56fb --- /dev/null +++ b/docker/docker-compose.cf-telemetry.yaml @@ -0,0 +1,34 @@ +# Ephemeral all-in-one ClickStack observability for integration tests. + +services: + clickstack: + image: ${CF_CLICKSTACK_IMAGE:-clickhouse/clickstack-all-in-one:2.37.0@sha256:16650781330f42fea6b02b15144a2233383077c799ab5bc06b131abe23e89f47} + labels: + name: cf-clickstack + restart: unless-stopped + environment: + CUSTOM_OTELCOL_CONFIG_FILE: /etc/otelcol-contrib/custom.config.yaml + volumes: + - ${CF_INTEGRATION_ROOT:?Set CF_INTEGRATION_ROOT to the integration harness root}/docker/clickstack/collector.yaml:/etc/otelcol-contrib/custom.config.yaml:ro + ports: + - "127.0.0.1:${CF_OBSERVABILITY_UI_PORT:-3000}:8080" + expose: + - "4317" + - "4318" + tmpfs: + - /data/db + - /var/lib/clickhouse + - /var/log/clickhouse-server + networks: + - mcpnet + healthcheck: + test: + - CMD-SHELL + - curl -fsS http://127.0.0.1:8080/api/health >/dev/null && curl -fsS http://127.0.0.1:13133/ >/dev/null + interval: 2s + timeout: 2s + retries: 60 + start_period: 20s + +networks: + mcpnet: diff --git a/scripts/prepare_standalone_config.py b/scripts/prepare_standalone_config.py index bbabc05..456d17d 100644 --- a/scripts/prepare_standalone_config.py +++ b/scripts/prepare_standalone_config.py @@ -37,6 +37,10 @@ def token_subject(token: str) -> str: def prepare_config(server_id: str, protocol_version: str) -> dict: if not server_id: raise SystemExit("virtual-host-id must not be empty") + routes = { + name: {"backend_name": BACKEND_NAME, "upstream_name": name} + for name in TOOL_NAMES + } return { "virtual_hosts": { server_id: { @@ -60,7 +64,14 @@ def prepare_config(server_id: str, protocol_version: str) -> dict: "completion": {}, "tool_schemas": {name: {} for name in TOOL_NAMES}, } - } + }, + # Published dataplane images can trail the control-plane schema. + # The dataplane serializer ignores fields it does not understand, + # so publish both routing shapes while the two releases overlap. + "tools": routes, + "resources": {}, + "resource_templates": {}, + "prompts": {}, } } } diff --git a/src/app.rs b/src/app.rs index 2c5992c..6bcee5e 100644 --- a/src/app.rs +++ b/src/app.rs @@ -82,6 +82,9 @@ impl Action { if args.standalone { summary.push_str("\nControl plane: disabled during load"); } + if args.observability { + summary.push_str("\nObservability: ClickStack enabled during load"); + } summary } Self::Live { @@ -233,6 +236,7 @@ pub(crate) struct ResolvedLoadArgs { pub(crate) topology: StackMode, pub(crate) protocol_version: ProtocolVersion, pub(crate) standalone: bool, + pub(crate) observability: bool, pub(crate) request: LoadRequest, } @@ -320,6 +324,7 @@ pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result Self { + for name in [ + "docker-compose.cf-telemetry.yaml", + "docker-compose.cf-controlplane-observability.yaml", + ] { + let overlay = repository_root.join("docker").join(name); + if !self.files.contains(&overlay) { + self.files.push(overlay); + } + } + if include_dataplane { + let overlay = repository_root + .join("docker") + .join("docker-compose.cf-dataplane-observability.yaml"); + if !self.files.contains(&overlay) { + self.files.push(overlay); + } + } + self + } + /// Enables the isolated official MCP conformance server fixture. #[must_use] pub(crate) fn with_conformance_fixture(self, repository_root: &Path) -> Self { diff --git a/src/infrastructure/compose_integration_tests.rs b/src/infrastructure/compose_integration_tests.rs index ff42165..9d80c63 100644 --- a/src/infrastructure/compose_integration_tests.rs +++ b/src/infrastructure/compose_integration_tests.rs @@ -188,6 +188,11 @@ fn compose_overlays_assign_short_container_display_names() { .expect("read standalone conformance fixture Compose file"); let fixture: yaml_serde::Value = yaml_serde::from_str(&fixture).expect("parse standalone conformance fixture Compose file"); + let observability = + fs::read_to_string(workspace_root().join("docker/docker-compose.cf-telemetry.yaml")) + .expect("read observability Compose overlay"); + let observability: yaml_serde::Value = + yaml_serde::from_str(&observability).expect("parse observability Compose overlay"); for (service, expected_name) in [ ("gateway", "cf-controlplane"), @@ -225,6 +230,10 @@ fn compose_overlays_assign_short_container_display_names() { conformance["services"]["mcp_conformance_proxy"]["labels"]["name"].as_str(), Some("cf-conformance-proxy") ); + assert_eq!( + observability["services"]["clickstack"]["labels"]["name"].as_str(), + Some("cf-clickstack") + ); } #[test] @@ -413,13 +422,137 @@ fn conformance_fixture_is_an_explicit_overlay_and_profile() { ComposeProject::conformance_fixture(Path::new("/repo"), OsString::from("fixture")); assert_eq!( standalone.files(), - [PathBuf::from( - "/repo/docker/docker-compose.cf-conformance-fixture.yaml" - )] + [ + PathBuf::from("/repo/docker/docker-compose.cf-conformance-fixture.yaml"), + PathBuf::from("/repo/docker/docker-compose.cf-telemetry.yaml"), + ] ); assert_eq!(standalone.profiles(), ["conformance"]); } +#[test] +fn observability_overlays_are_explicit_and_include_the_selected_lane() { + let root = Path::new("/repo"); + let checkout = Path::new("/checkout"); + let built_in = ComposeProject::controlplane(root, checkout, OsString::from("built-in"), false) + .with_observability(root, false); + let external = ComposeProject::dataplane(root, checkout, OsString::from("external"), false) + .with_observability(root, true); + + assert!(built_in.files().ends_with(&[ + PathBuf::from("/repo/docker/docker-compose.cf-telemetry.yaml"), + PathBuf::from("/repo/docker/docker-compose.cf-controlplane-observability.yaml"), + ])); + assert!(external.files().ends_with(&[ + PathBuf::from("/repo/docker/docker-compose.cf-telemetry.yaml"), + PathBuf::from("/repo/docker/docker-compose.cf-controlplane-observability.yaml"), + PathBuf::from("/repo/docker/docker-compose.cf-dataplane-observability.yaml"), + ])); +} + +#[test] +fn observability_is_ephemeral_and_exports_both_routed_services() { + let root = workspace_root(); + let compose = fs::read_to_string(root.join("docker/docker-compose.cf-telemetry.yaml")) + .expect("read observability Compose overlay"); + let compose: yaml_serde::Value = + yaml_serde::from_str(&compose).expect("parse observability Compose overlay"); + let services = compose["services"] + .as_mapping() + .expect("telemetry services must be a mapping"); + let clickstack = &compose["services"]["clickstack"]; + + assert_eq!(services.len(), 1); + assert!( + clickstack["image"] + .as_str() + .expect("ClickStack image must be text") + .contains("clickstack-all-in-one:2.37.0@sha256:") + ); + assert_eq!( + clickstack["ports"][0].as_str(), + Some("127.0.0.1:${CF_OBSERVABILITY_UI_PORT:-3000}:8080") + ); + assert_eq!( + clickstack["tmpfs"] + .as_sequence() + .expect("ClickStack storage must use tmpfs") + .iter() + .filter_map(yaml_serde::Value::as_str) + .collect::>(), + [ + "/data/db", + "/var/lib/clickhouse", + "/var/log/clickhouse-server" + ] + ); + assert_eq!( + clickstack["environment"]["CUSTOM_OTELCOL_CONFIG_FILE"].as_str(), + Some("/etc/otelcol-contrib/custom.config.yaml") + ); + assert_eq!( + clickstack["volumes"][0].as_str(), + Some( + "${CF_INTEGRATION_ROOT:?Set CF_INTEGRATION_ROOT to the integration harness root}/docker/clickstack/collector.yaml:/etc/otelcol-contrib/custom.config.yaml:ro" + ) + ); + + let collector = fs::read_to_string(root.join("docker/clickstack/collector.yaml")) + .expect("read ClickStack collector extension"); + let collector: yaml_serde::Value = + yaml_serde::from_str(&collector).expect("parse ClickStack collector extension"); + for pipeline in ["traces/integration", "metrics/integration"] { + assert_eq!( + collector["service"]["pipelines"][pipeline]["receivers"][0].as_str(), + Some("otlp/hyperdx") + ); + assert_eq!( + collector["service"]["pipelines"][pipeline]["exporters"][0].as_str(), + Some("clickhouse") + ); + } + + let gateway = + fs::read_to_string(root.join("docker/docker-compose.cf-controlplane-observability.yaml")) + .expect("read gateway observability overlay"); + let gateway: yaml_serde::Value = + yaml_serde::from_str(&gateway).expect("parse gateway observability overlay"); + assert_eq!( + gateway["services"]["gateway"]["environment"]["OTEL_EXPORTER_OTLP_ENDPOINT"].as_str(), + Some("http://clickstack:4317") + ); + + let dataplane = + fs::read_to_string(root.join("docker/docker-compose.cf-dataplane-observability.yaml")) + .expect("read dataplane observability overlay"); + let dataplane: yaml_serde::Value = + yaml_serde::from_str(&dataplane).expect("parse dataplane observability overlay"); + assert_eq!( + dataplane["services"]["dataplane"]["environment"] + ["CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_ENDPOINT"] + .as_str(), + Some("http://clickstack:4318/v1/traces") + ); + assert_eq!( + dataplane["services"]["dataplane"]["environment"] + ["CONTEXTFORGE_DATA_PLANE_ENABLE_OTEL_METRICS"] + .as_str(), + Some("true") + ); + assert_eq!( + dataplane["services"]["dataplane"]["environment"] + ["CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"] + .as_str(), + Some("http://clickstack:4318/v1/metrics") + ); + assert_eq!( + dataplane["services"]["dataplane"]["environment"] + ["CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_PROTOCOL"] + .as_str(), + Some("http-protobuf") + ); +} + #[test] fn conformance_container_inputs_pin_the_runner_revision_and_protocol_fixture() { let root = workspace_root(); diff --git a/src/infrastructure/stack.rs b/src/infrastructure/stack.rs index 2377e6a..dd015e0 100644 --- a/src/infrastructure/stack.rs +++ b/src/infrastructure/stack.rs @@ -166,10 +166,11 @@ impl StackCommandPlan { start_locust_ui: bool, locust_workers: usize, ) -> Self { - let mut arguments = vec![OsString::from("up"), OsString::from("-d")]; - if mode == StackMode::Dataplane { - arguments.push(OsString::from("--remove-orphans")); - } + let mut arguments = vec![ + OsString::from("up"), + OsString::from("-d"), + OsString::from("--remove-orphans"), + ]; if build { arguments.push(OsString::from("--build")); } @@ -320,6 +321,7 @@ impl FreshnessSnapshot { for service in [ "gateway", "dataplane", + "clickstack", "nginx", "postgres", "pgbouncer", diff --git a/src/infrastructure/stack_integration_tests.rs b/src/infrastructure/stack_integration_tests.rs index 4ca3e29..b3fbb58 100644 --- a/src/infrastructure/stack_integration_tests.rs +++ b/src/infrastructure/stack_integration_tests.rs @@ -153,7 +153,7 @@ fn controlplane_up_does_not_activate_locust_profile_when_ui_is_disabled() { false, 3, )); - assert!(ends_with(&disabled, &["up", "-d"])); + assert!(ends_with(&disabled, &["up", "-d", "--remove-orphans"])); assert!( disabled .iter() @@ -169,7 +169,14 @@ fn controlplane_up_does_not_activate_locust_profile_when_ui_is_disabled() { )); assert!(ends_with( &enabled, - &["up", "-d", "--build", "--scale", "locust_worker=3"] + &[ + "up", + "-d", + "--remove-orphans", + "--build", + "--scale", + "locust_worker=3" + ] )); } @@ -243,6 +250,7 @@ fn cleanup_status_logs_and_config_use_typed_compose_commands() { OsString::from("cf-keycloak"), OsString::from("cf-conformance-server"), OsString::from("cf-conformance-proxy"), + OsString::from("cf-clickstack"), OsString::from("custom-service"), ] )), @@ -268,6 +276,7 @@ fn cleanup_status_logs_and_config_use_typed_compose_commands() { "keycloak", "mcp_conformance_server", "mcp_conformance_proxy", + "clickstack", "custom-service", ] )); @@ -297,6 +306,7 @@ fn current_snapshot() -> FreshnessSnapshot { let running = [ ("gateway", "cp-image", Some("cp-head")), ("dataplane", "dp-image", Some("dp-head")), + ("clickstack", "clickstack", None), ("nginx", "nginx", None), ("postgres", "postgres", None), ("pgbouncer", "pgbouncer", None), diff --git a/src/performance/python_adapter_tests.rs b/src/performance/python_adapter_tests.rs index 888c633..a96e67d 100644 --- a/src/performance/python_adapter_tests.rs +++ b/src/performance/python_adapter_tests.rs @@ -62,6 +62,11 @@ assert backend["url"] == "http://mcp_conformance_server:3000/mcp" assert backend["mcp_protocol_version"] == "2026-07-28" assert backend["tool_name_aliases"] == [{"downstream_prefixed_name": "test_simple_text", "upstream_name": "test_simple_text"}] assert backend["tool_schemas"] == {"test_simple_text": {}} +virtual_host = prepared["virtual_hosts"]["server-123"] +assert virtual_host["tools"] == {"test_simple_text": {"backend_name": "standalone-load", "upstream_name": "test_simple_text"}} +assert virtual_host["resources"] == {} +assert virtual_host["resource_templates"] == {} +assert virtual_host["prompts"] == {} "#; let output = Command::new(python()) .arg("-c") diff --git a/src/runtime/conformance/mod.rs b/src/runtime/conformance/mod.rs index 4b1a24a..4a06604 100644 --- a/src/runtime/conformance/mod.rs +++ b/src/runtime/conformance/mod.rs @@ -130,7 +130,13 @@ impl RuntimeContext { self.runner.run_async(&build).await?; let up = self.standalone_conformance_environment( - project.command(["up", "-d", "--wait", OFFICIAL_CONFORMANCE_SERVICE]), + project.command([ + "up", + "-d", + "--wait", + OFFICIAL_CONFORMANCE_SERVICE, + "clickstack", + ]), server_era, ); self.runner.run_async(&up).await.map_err(AppFailure::from) diff --git a/src/runtime/performance/mod.rs b/src/runtime/performance/mod.rs index a11e5ee..fed45d2 100644 --- a/src/runtime/performance/mod.rs +++ b/src/runtime/performance/mod.rs @@ -9,10 +9,11 @@ impl RuntimeContext { let server_id = self.default_server_id().to_owned(); let operation_server_id = server_id.clone(); let preparation = Activity::spinner("Preparing performance stack"); - self.with_managed_authenticated_target( + self.with_managed_performance_target( args.topology, &server_id, args.standalone, + args.observability, |token, standalone_tool_names| async move { let command = LocustCommand::new_with_protocol_version( &self.config, diff --git a/src/runtime/session.rs b/src/runtime/session.rs index e5f97af..9036e25 100644 --- a/src/runtime/session.rs +++ b/src/runtime/session.rs @@ -90,6 +90,50 @@ impl RuntimeContext { standalone: bool, operation: F, ) -> AppResult<()> + where + F: FnOnce(String, Vec) -> Fut, + Fut: Future>, + { + self.with_managed_authenticated_target_project( + topology, + server_id, + standalone, + self.compose_project(topology), + operation, + ) + .await + } + + pub(super) async fn with_managed_performance_target( + &self, + topology: StackMode, + server_id: &str, + standalone: bool, + observability: bool, + operation: F, + ) -> AppResult<()> + where + F: FnOnce(String, Vec) -> Fut, + Fut: Future>, + { + self.with_managed_authenticated_target_project( + topology, + server_id, + standalone, + self.performance_compose_project(topology, observability), + operation, + ) + .await + } + + async fn with_managed_authenticated_target_project( + &self, + topology: StackMode, + server_id: &str, + standalone: bool, + project: ComposeProject, + operation: F, + ) -> AppResult<()> where F: FnOnce(String, Vec) -> Fut, Fut: Future>, @@ -100,7 +144,10 @@ impl RuntimeContext { ))); } let mut scope = ManagedSessionScope::new(self, topology, standalone); - let primary = match self.stack_up(topology, false).await { + let primary = match self + .stack_up_with_project(topology, false, project, false) + .await + { Ok(()) => match self .prepare_authenticated_target(topology, server_id, standalone) .await diff --git a/src/runtime/stack/mod.rs b/src/runtime/stack/mod.rs index a16df3e..89db00c 100644 --- a/src/runtime/stack/mod.rs +++ b/src/runtime/stack/mod.rs @@ -99,7 +99,7 @@ impl RuntimeContext { .await } - async fn stack_up_with_project( + pub(super) async fn stack_up_with_project( &self, mode: StackMode, fresh: bool, @@ -209,16 +209,41 @@ impl RuntimeContext { mode: StackMode, conformance_endpoint: &url::Url, ) -> AppResult<()> { + let observability_ui = format!( + "http://127.0.0.1:{}", + self.environment_text("CF_OBSERVABILITY_UI_PORT") + .filter(|port| !port.is_empty()) + .unwrap_or("3000") + ); let summary = format_stack_endpoint_summary( self.base_url()?, &self.public_mcp_endpoint(mode)?, conformance_endpoint, + &observability_ui, ); println!("{}", OutputStyle::stdout().info(&summary)); Ok(()) } pub(super) fn compose_project(&self, mode: StackMode) -> ComposeProject { + self.routed_compose_project(mode) + .with_observability(self.config.asset_root(), mode == StackMode::Dataplane) + } + + pub(super) fn performance_compose_project( + &self, + mode: StackMode, + observability: bool, + ) -> ComposeProject { + let project = self.routed_compose_project(mode); + if observability { + project.with_observability(self.config.asset_root(), mode == StackMode::Dataplane) + } else { + project + } + } + + fn routed_compose_project(&self, mode: StackMode) -> ComposeProject { let project = match mode { StackMode::Dataplane => ComposeProject::dataplane( self.config.asset_root(), @@ -665,6 +690,7 @@ impl RuntimeContext { for service in [ "gateway", "dataplane", + "clickstack", "nginx", "postgres", "pgbouncer", @@ -1114,9 +1140,10 @@ fn format_stack_endpoint_summary( public_origin: &str, public_mcp_endpoint: &url::Url, conformance_endpoint: &url::Url, + observability_ui: &str, ) -> String { format!( - "Gateway/API: {public_origin}\nPublic MCP: {public_mcp_endpoint}\nConformance MCP (direct): {conformance_endpoint}" + "Gateway/API: {public_origin}\nPublic MCP: {public_mcp_endpoint}\nConformance MCP (direct): {conformance_endpoint}\nObservability: {observability_ui}" ) } @@ -1304,12 +1331,16 @@ mod tests { url::Url::parse("http://127.0.0.1:8080/servers/server-id/mcp").expect("public URL"); let conformance = url::Url::parse("http://127.0.0.1:49152/mcp").expect("conformance URL"); - let summary = - format_stack_endpoint_summary("http://127.0.0.1:8080", &public_mcp, &conformance); + let summary = format_stack_endpoint_summary( + "http://127.0.0.1:8080", + &public_mcp, + &conformance, + "http://127.0.0.1:3000", + ); assert_eq!( summary, - "Gateway/API: http://127.0.0.1:8080\nPublic MCP: http://127.0.0.1:8080/servers/server-id/mcp\nConformance MCP (direct): http://127.0.0.1:49152/mcp" + "Gateway/API: http://127.0.0.1:8080\nPublic MCP: http://127.0.0.1:8080/servers/server-id/mcp\nConformance MCP (direct): http://127.0.0.1:49152/mcp\nObservability: http://127.0.0.1:3000" ); } From f7705ffb9e4e42ebe21499a0cf459bff46502f1e Mon Sep 17 00:00:00 2001 From: lucarlig Date: Thu, 3 Sep 2026 15:19:24 +0100 Subject: [PATCH 08/10] fix: retain no-auth ClickStack after managed runs Signed-off-by: lucarlig --- README.md | 10 +- ...compose.cf-controlplane-observability.yaml | 10 +- ...er-compose.cf-dataplane-observability.yaml | 10 +- docker/docker-compose.cf-telemetry.yaml | 10 +- src/infrastructure/compose.rs | 26 +++-- .../compose_integration_tests.rs | 41 +++++--- src/infrastructure/stack.rs | 1 - src/infrastructure/stack_integration_tests.rs | 1 - src/runtime/conformance/mod.rs | 9 +- src/runtime/session.rs | 5 +- src/runtime/stack/mod.rs | 97 +++++++++++++++++-- 11 files changed, 162 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index fe65137..c9fa8e6 100644 --- a/README.md +++ b/README.md @@ -86,13 +86,13 @@ services unless service names are supplied. `config` prints merged Compose configuration. `down --volumes` also removes persistent volumes. ClickStack starts by default for `stack`, `probe`, `live`, `conformance`, and -`debug inspect`. While the stack or test is running, open the HyperDX UI at - to inspect traces and metrics. On first access, create -a temporary local user; its sources are configured automatically. The external -dataplane exports the exact +`debug inspect`. Open the no-login HyperDX UI at to +inspect traces and metrics. Managed test cleanup leaves ClickStack running, so +the UI remains available after a command finishes; `stack down --lane all` +removes it. The external dataplane exports the exact HTTP counters, latency histograms, in-flight gauge, and body sizes recorded by its `HttpMetricsLayer`; allow 30 seconds for its first export. All telemetry -storage is ephemeral and disappears with the stack. +storage is ephemeral and disappears when ClickStack is removed. ### `probe` diff --git a/docker/docker-compose.cf-controlplane-observability.yaml b/docker/docker-compose.cf-controlplane-observability.yaml index 24112bf..7a83372 100644 --- a/docker/docker-compose.cf-controlplane-observability.yaml +++ b/docker/docker-compose.cf-controlplane-observability.yaml @@ -10,6 +10,10 @@ services: OTEL_EXPORTER_OTLP_INSECURE: "true" OTEL_SERVICE_NAME: cf-controlplane OTEL_RESOURCE_ATTRIBUTES: deployment.environment=integration,service.namespace=contextforge - depends_on: - clickstack: - condition: service_healthy + networks: + observability: + +networks: + observability: + name: ${CF_OBSERVABILITY_NETWORK:-cf-observability} + external: true diff --git a/docker/docker-compose.cf-dataplane-observability.yaml b/docker/docker-compose.cf-dataplane-observability.yaml index 26ba9b1..a1b39fd 100644 --- a/docker/docker-compose.cf-dataplane-observability.yaml +++ b/docker/docker-compose.cf-dataplane-observability.yaml @@ -9,6 +9,10 @@ services: CONTEXTFORGE_DATA_PLANE_OTEL_SERVICE_NAME: cf-dataplane CONTEXTFORGE_DATA_PLANE_ENABLE_OTEL_METRICS: "true" CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: http://clickstack:4318/v1/metrics - depends_on: - clickstack: - condition: service_healthy + networks: + observability: + +networks: + observability: + name: ${CF_OBSERVABILITY_NETWORK:-cf-observability} + external: true diff --git a/docker/docker-compose.cf-telemetry.yaml b/docker/docker-compose.cf-telemetry.yaml index 40a56fb..dc517e7 100644 --- a/docker/docker-compose.cf-telemetry.yaml +++ b/docker/docker-compose.cf-telemetry.yaml @@ -2,7 +2,7 @@ services: clickstack: - image: ${CF_CLICKSTACK_IMAGE:-clickhouse/clickstack-all-in-one:2.37.0@sha256:16650781330f42fea6b02b15144a2233383077c799ab5bc06b131abe23e89f47} + image: ${CF_CLICKSTACK_IMAGE:-clickhouse/clickstack-local:2.37.0@sha256:1b67e1a3667be097c704913d18b96e388eb4d85c1e8aa8eec805dc9fdad98426} labels: name: cf-clickstack restart: unless-stopped @@ -16,11 +16,12 @@ services: - "4317" - "4318" tmpfs: - - /data/db - /var/lib/clickhouse - /var/log/clickhouse-server networks: - - mcpnet + observability: + aliases: + - clickstack healthcheck: test: - CMD-SHELL @@ -31,4 +32,5 @@ services: start_period: 20s networks: - mcpnet: + observability: + name: ${CF_OBSERVABILITY_NETWORK:-cf-observability} diff --git a/src/infrastructure/compose.rs b/src/infrastructure/compose.rs index e104d59..f7246e4 100644 --- a/src/infrastructure/compose.rs +++ b/src/infrastructure/compose.rs @@ -55,11 +55,22 @@ impl ComposeProject { repository_root .join("docker") .join("docker-compose.cf-conformance-fixture.yaml"), + ], + profiles: vec![OsString::from("conformance")], + } + } + + /// Builds the independently managed local ClickStack project. + #[must_use] + pub(crate) fn observability(repository_root: &Path, project_name: OsString) -> Self { + Self { + project_name, + files: vec![ repository_root .join("docker") .join("docker-compose.cf-telemetry.yaml"), ], - profiles: vec![OsString::from("conformance")], + profiles: Vec::new(), } } @@ -187,14 +198,11 @@ impl ComposeProject { repository_root: &Path, include_dataplane: bool, ) -> Self { - for name in [ - "docker-compose.cf-telemetry.yaml", - "docker-compose.cf-controlplane-observability.yaml", - ] { - let overlay = repository_root.join("docker").join(name); - if !self.files.contains(&overlay) { - self.files.push(overlay); - } + let controlplane = repository_root + .join("docker") + .join("docker-compose.cf-controlplane-observability.yaml"); + if !self.files.contains(&controlplane) { + self.files.push(controlplane); } if include_dataplane { let overlay = repository_root diff --git a/src/infrastructure/compose_integration_tests.rs b/src/infrastructure/compose_integration_tests.rs index 9d80c63..e821a0a 100644 --- a/src/infrastructure/compose_integration_tests.rs +++ b/src/infrastructure/compose_integration_tests.rs @@ -422,12 +422,21 @@ fn conformance_fixture_is_an_explicit_overlay_and_profile() { ComposeProject::conformance_fixture(Path::new("/repo"), OsString::from("fixture")); assert_eq!( standalone.files(), - [ - PathBuf::from("/repo/docker/docker-compose.cf-conformance-fixture.yaml"), - PathBuf::from("/repo/docker/docker-compose.cf-telemetry.yaml"), - ] + [PathBuf::from( + "/repo/docker/docker-compose.cf-conformance-fixture.yaml" + )] ); assert_eq!(standalone.profiles(), ["conformance"]); + + let observability = + ComposeProject::observability(Path::new("/repo"), OsString::from("observability")); + assert_eq!( + observability.files(), + [PathBuf::from( + "/repo/docker/docker-compose.cf-telemetry.yaml" + )] + ); + assert!(observability.profiles().is_empty()); } #[test] @@ -439,12 +448,10 @@ fn observability_overlays_are_explicit_and_include_the_selected_lane() { let external = ComposeProject::dataplane(root, checkout, OsString::from("external"), false) .with_observability(root, true); - assert!(built_in.files().ends_with(&[ - PathBuf::from("/repo/docker/docker-compose.cf-telemetry.yaml"), - PathBuf::from("/repo/docker/docker-compose.cf-controlplane-observability.yaml"), - ])); + assert!(built_in.files().ends_with(&[PathBuf::from( + "/repo/docker/docker-compose.cf-controlplane-observability.yaml" + )])); assert!(external.files().ends_with(&[ - PathBuf::from("/repo/docker/docker-compose.cf-telemetry.yaml"), PathBuf::from("/repo/docker/docker-compose.cf-controlplane-observability.yaml"), PathBuf::from("/repo/docker/docker-compose.cf-dataplane-observability.yaml"), ])); @@ -467,7 +474,7 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { clickstack["image"] .as_str() .expect("ClickStack image must be text") - .contains("clickstack-all-in-one:2.37.0@sha256:") + .contains("clickstack-local:2.37.0@sha256:") ); assert_eq!( clickstack["ports"][0].as_str(), @@ -480,11 +487,7 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { .iter() .filter_map(yaml_serde::Value::as_str) .collect::>(), - [ - "/data/db", - "/var/lib/clickhouse", - "/var/log/clickhouse-server" - ] + ["/var/lib/clickhouse", "/var/log/clickhouse-server"] ); assert_eq!( clickstack["environment"]["CUSTOM_OTELCOL_CONFIG_FILE"].as_str(), @@ -521,6 +524,10 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { gateway["services"]["gateway"]["environment"]["OTEL_EXPORTER_OTLP_ENDPOINT"].as_str(), Some("http://clickstack:4317") ); + assert_eq!( + gateway["networks"]["observability"]["external"].as_bool(), + Some(true) + ); let dataplane = fs::read_to_string(root.join("docker/docker-compose.cf-dataplane-observability.yaml")) @@ -551,6 +558,10 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { .as_str(), Some("http-protobuf") ); + assert_eq!( + dataplane["networks"]["observability"]["external"].as_bool(), + Some(true) + ); } #[test] diff --git a/src/infrastructure/stack.rs b/src/infrastructure/stack.rs index dd015e0..aaa3801 100644 --- a/src/infrastructure/stack.rs +++ b/src/infrastructure/stack.rs @@ -321,7 +321,6 @@ impl FreshnessSnapshot { for service in [ "gateway", "dataplane", - "clickstack", "nginx", "postgres", "pgbouncer", diff --git a/src/infrastructure/stack_integration_tests.rs b/src/infrastructure/stack_integration_tests.rs index b3fbb58..2de678e 100644 --- a/src/infrastructure/stack_integration_tests.rs +++ b/src/infrastructure/stack_integration_tests.rs @@ -306,7 +306,6 @@ fn current_snapshot() -> FreshnessSnapshot { let running = [ ("gateway", "cp-image", Some("cp-head")), ("dataplane", "dp-image", Some("dp-head")), - ("clickstack", "clickstack", None), ("nginx", "nginx", None), ("postgres", "postgres", None), ("pgbouncer", "pgbouncer", None), diff --git a/src/runtime/conformance/mod.rs b/src/runtime/conformance/mod.rs index 4a06604..70aea68 100644 --- a/src/runtime/conformance/mod.rs +++ b/src/runtime/conformance/mod.rs @@ -122,6 +122,7 @@ impl RuntimeContext { &self, server_era: ConformanceServerEra, ) -> AppResult<()> { + self.start_observability()?; let project = self.standalone_conformance_project(); let build = self.standalone_conformance_environment( project.command(["build", OFFICIAL_CONFORMANCE_SERVICE]), @@ -130,13 +131,7 @@ impl RuntimeContext { self.runner.run_async(&build).await?; let up = self.standalone_conformance_environment( - project.command([ - "up", - "-d", - "--wait", - OFFICIAL_CONFORMANCE_SERVICE, - "clickstack", - ]), + project.command(["up", "-d", "--wait", OFFICIAL_CONFORMANCE_SERVICE]), server_era, ); self.runner.run_async(&up).await.map_err(AppFailure::from) diff --git a/src/runtime/session.rs b/src/runtime/session.rs index 9036e25..4f56b0a 100644 --- a/src/runtime/session.rs +++ b/src/runtime/session.rs @@ -99,6 +99,7 @@ impl RuntimeContext { server_id, standalone, self.compose_project(topology), + true, operation, ) .await @@ -121,6 +122,7 @@ impl RuntimeContext { server_id, standalone, self.performance_compose_project(topology, observability), + observability, operation, ) .await @@ -132,6 +134,7 @@ impl RuntimeContext { server_id: &str, standalone: bool, project: ComposeProject, + observability: bool, operation: F, ) -> AppResult<()> where @@ -145,7 +148,7 @@ impl RuntimeContext { } let mut scope = ManagedSessionScope::new(self, topology, standalone); let primary = match self - .stack_up_with_project(topology, false, project, false) + .stack_up_with_project(topology, false, project, false, observability) .await { Ok(()) => match self diff --git a/src/runtime/stack/mod.rs b/src/runtime/stack/mod.rs index 89db00c..04f00ed 100644 --- a/src/runtime/stack/mod.rs +++ b/src/runtime/stack/mod.rs @@ -39,14 +39,15 @@ impl RuntimeContext { Activity::completed("Integration stack ready"); self.print_stack_summary(topology, &conformance_endpoint) } - StackAction::Down { lane, volumes } => self.cleanup( - lane, - if volumes { + StackAction::Down { lane, volumes } => { + let kind = if volumes { CleanupKind::Reset } else { CleanupKind::Down - }, - ), + }; + let primary = self.cleanup(lane, kind).err(); + finish_with_cleanup(primary, self.cleanup_observability(kind, true)) + } StackAction::Status(mode) => { self.require_mode_sources(mode)?; let command = StackCommandPlan::status(self.conformance_compose_project(mode)); @@ -86,7 +87,7 @@ impl RuntimeContext { } pub(super) async fn stack_up(&self, mode: StackMode, fresh: bool) -> AppResult<()> { - self.stack_up_with_project(mode, fresh, self.compose_project(mode), false) + self.stack_up_with_project(mode, fresh, self.compose_project(mode), false, true) .await } @@ -95,8 +96,14 @@ impl RuntimeContext { mode: StackMode, fresh: bool, ) -> AppResult<()> { - self.stack_up_with_project(mode, fresh, self.conformance_runtime_project(mode), false) - .await + self.stack_up_with_project( + mode, + fresh, + self.conformance_runtime_project(mode), + false, + true, + ) + .await } pub(super) async fn stack_up_with_project( @@ -105,6 +112,7 @@ impl RuntimeContext { fresh: bool, project: ComposeProject, report_progress: bool, + observability: bool, ) -> AppResult<()> { self.ensure_mode_sources(mode)?; if mode == StackMode::Dataplane { @@ -114,6 +122,7 @@ impl RuntimeContext { let build = self.resolve_build(mode, report_progress)?; self.pull_images(mode, build, report_progress)?; if mode == StackMode::Dataplane + && !observability && !fresh && !self.environment_flag("CF_FORCE_STACK_RESTART", false) && !build @@ -133,6 +142,9 @@ impl RuntimeContext { self.cleanup(topology_selection(mode), CleanupKind::Reset)?; } self.ensure_other_stack_stopped(mode)?; + if observability { + self.start_observability()?; + } if mode == StackMode::Controlplane { fs::create_dir_all(self.config.controlplane_dir().join("reports")) .context("failed to create control-plane report directory") @@ -271,6 +283,55 @@ impl RuntimeContext { .with_conformance_runtime(self.config.asset_root()) } + pub(super) fn start_observability(&self) -> AppResult<()> { + let command = self.observability_compose_project().command([ + "up", + "-d", + "--wait", + "--remove-orphans", + ]); + Ok(self.runner.run(&self.observability_environment(command))?) + } + + fn observability_compose_project(&self) -> ComposeProject { + ComposeProject::observability( + self.config.asset_root(), + format!( + "{}-observability", + self.config.integration_project().value.to_string_lossy() + ) + .into(), + ) + } + + fn observability_environment(&self, command: CommandSpec) -> CommandSpec { + let command_environment = command.environment().clone(); + let mut command = command.cwd(self.config.root()); + for (key, value) in self.config.environment().iter() { + if !command_environment.contains_key(key) { + command = command.env(key.clone(), value.value.clone()); + } + } + command + .env("CF_INTEGRATION_ROOT", self.config.asset_root().as_os_str()) + .env("CF_OBSERVABILITY_NETWORK", self.observability_network()) + } + + fn observability_network(&self) -> OsString { + self.environment_text("CF_OBSERVABILITY_NETWORK") + .filter(|name| !name.is_empty()) + .map_or_else( + || { + format!( + "{}-observability", + self.config.integration_project().value.to_string_lossy() + ) + .into() + }, + OsString::from, + ) + } + pub(super) fn compose_environment( &self, command: CommandSpec, @@ -304,6 +365,7 @@ impl RuntimeContext { ); command = command .env("CF_INTEGRATION_ROOT", self.config.asset_root().as_os_str()) + .env("CF_OBSERVABILITY_NETWORK", self.observability_network()) .env( "CF_INTEGRATION_DIR", self.config.integration_dir().as_os_str(), @@ -690,7 +752,6 @@ impl RuntimeContext { for service in [ "gateway", "dataplane", - "clickstack", "nginx", "postgres", "pgbouncer", @@ -927,6 +988,24 @@ impl RuntimeContext { finish_with_cleanup_failures(None, cleanup_failures) } + fn cleanup_observability(&self, kind: CleanupKind, inherit_output: bool) -> AppResult<()> { + let project = self.observability_compose_project(); + let command = StackCommandPlan::cleanup(project, kind); + let command = self.observability_environment(command.command().clone()); + let primary = self + .run_cleanup_command(&command, inherit_output) + .map_err(AppFailure::from) + .err(); + let project = format!( + "{}-observability", + self.config.integration_project().value.to_string_lossy() + ); + finish_with_cleanup( + primary, + self.remove_project_by_label(&project, kind, inherit_output), + ) + } + fn remove_project_by_label( &self, project: &str, From d39b68892969549c679614af76f76c61957459bf Mon Sep 17 00:00:00 2001 From: lucarlig Date: Thu, 3 Sep 2026 15:31:09 +0100 Subject: [PATCH 09/10] feat: capture routed service logs in ClickStack Signed-off-by: lucarlig --- README.md | 7 +++- docker/clickstack/collector.yaml | 24 +++++++---- ...compose.cf-controlplane-observability.yaml | 7 ++++ ...er-compose.cf-dataplane-observability.yaml | 7 ++++ docker/docker-compose.cf-telemetry.yaml | 1 + .../compose_integration_tests.rs | 42 ++++++++++++++----- 6 files changed, 68 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c9fa8e6..c78b5b8 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,11 @@ inspect traces and metrics. Managed test cleanup leaves ClickStack running, so the UI remains available after a command finishes; `stack down --lane all` removes it. The external dataplane exports the exact HTTP counters, latency histograms, in-flight gauge, and body sizes recorded by -its `HttpMetricsLayer`; allow 30 seconds for its first export. All telemetry -storage is ephemeral and disappears when ClickStack is removed. +its `HttpMetricsLayer`. HyperDX opens on live application logs. For metrics, +open **Chart Explorer**, select the **Metrics** data source and a metric, then +click **Run**. Run a load for at least 60 seconds so its 30-second cumulative +export interval produces the two samples needed for a non-zero chart. All +telemetry storage is ephemeral and disappears when ClickStack is removed. ### `probe` diff --git a/docker/clickstack/collector.yaml b/docker/clickstack/collector.yaml index c4321f1..591f575 100644 --- a/docker/clickstack/collector.yaml +++ b/docker/clickstack/collector.yaml @@ -1,12 +1,20 @@ -# Accept harness telemetry immediately, without depending on HyperDX onboarding. +# Add Docker stdout/stderr logs to ClickStack's built-in OTLP pipelines. + +receivers: + fluent_forward/docker: + endpoint: 0.0.0.0:24224 + +processors: + transform/docker_logs: + error_mode: ignore + log_statements: + - context: log + statements: + - set(resource.attributes["service.name"], attributes["fluent.tag"]) service: pipelines: - traces/integration: - receivers: [otlp/hyperdx] - processors: [memory_limiter, batch] - exporters: [clickhouse] - metrics/integration: - receivers: [otlp/hyperdx] - processors: [memory_limiter, batch] + logs/docker: + receivers: [fluent_forward/docker] + processors: [transform/docker_logs, memory_limiter, batch] exporters: [clickhouse] diff --git a/docker/docker-compose.cf-controlplane-observability.yaml b/docker/docker-compose.cf-controlplane-observability.yaml index 7a83372..594a4e6 100644 --- a/docker/docker-compose.cf-controlplane-observability.yaml +++ b/docker/docker-compose.cf-controlplane-observability.yaml @@ -12,6 +12,13 @@ services: OTEL_RESOURCE_ATTRIBUTES: deployment.environment=integration,service.namespace=contextforge networks: observability: + logging: + driver: fluentd + options: + fluentd-address: tcp://127.0.0.1:${CF_OBSERVABILITY_LOG_PORT:-24224} + fluentd-async: "true" + fluentd-sub-second-precision: "true" + tag: "{{.Name}}" networks: observability: diff --git a/docker/docker-compose.cf-dataplane-observability.yaml b/docker/docker-compose.cf-dataplane-observability.yaml index a1b39fd..a5921a6 100644 --- a/docker/docker-compose.cf-dataplane-observability.yaml +++ b/docker/docker-compose.cf-dataplane-observability.yaml @@ -11,6 +11,13 @@ services: CONTEXTFORGE_DATA_PLANE_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: http://clickstack:4318/v1/metrics networks: observability: + logging: + driver: fluentd + options: + fluentd-address: tcp://127.0.0.1:${CF_OBSERVABILITY_LOG_PORT:-24224} + fluentd-async: "true" + fluentd-sub-second-precision: "true" + tag: "{{.Name}}" networks: observability: diff --git a/docker/docker-compose.cf-telemetry.yaml b/docker/docker-compose.cf-telemetry.yaml index dc517e7..8b23f74 100644 --- a/docker/docker-compose.cf-telemetry.yaml +++ b/docker/docker-compose.cf-telemetry.yaml @@ -12,6 +12,7 @@ services: - ${CF_INTEGRATION_ROOT:?Set CF_INTEGRATION_ROOT to the integration harness root}/docker/clickstack/collector.yaml:/etc/otelcol-contrib/custom.config.yaml:ro ports: - "127.0.0.1:${CF_OBSERVABILITY_UI_PORT:-3000}:8080" + - "127.0.0.1:${CF_OBSERVABILITY_LOG_PORT:-24224}:24224" expose: - "4317" - "4318" diff --git a/src/infrastructure/compose_integration_tests.rs b/src/infrastructure/compose_integration_tests.rs index e821a0a..1f2270a 100644 --- a/src/infrastructure/compose_integration_tests.rs +++ b/src/infrastructure/compose_integration_tests.rs @@ -480,6 +480,10 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { clickstack["ports"][0].as_str(), Some("127.0.0.1:${CF_OBSERVABILITY_UI_PORT:-3000}:8080") ); + assert_eq!( + clickstack["ports"][1].as_str(), + Some("127.0.0.1:${CF_OBSERVABILITY_LOG_PORT:-24224}:24224") + ); assert_eq!( clickstack["tmpfs"] .as_sequence() @@ -504,16 +508,26 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { .expect("read ClickStack collector extension"); let collector: yaml_serde::Value = yaml_serde::from_str(&collector).expect("parse ClickStack collector extension"); - for pipeline in ["traces/integration", "metrics/integration"] { - assert_eq!( - collector["service"]["pipelines"][pipeline]["receivers"][0].as_str(), - Some("otlp/hyperdx") - ); - assert_eq!( - collector["service"]["pipelines"][pipeline]["exporters"][0].as_str(), - Some("clickhouse") - ); - } + assert_eq!( + collector["receivers"]["fluent_forward/docker"]["endpoint"].as_str(), + Some("0.0.0.0:24224") + ); + assert_eq!( + collector["service"]["pipelines"]["logs/docker"]["receivers"][0].as_str(), + Some("fluent_forward/docker") + ); + assert_eq!( + collector["service"]["pipelines"] + .as_mapping() + .expect("collector pipelines must be a mapping") + .len(), + 1, + "the extension must not duplicate ClickStack's built-in OTLP pipelines" + ); + assert_eq!( + collector["service"]["pipelines"]["logs/docker"]["exporters"][0].as_str(), + Some("clickhouse") + ); let gateway = fs::read_to_string(root.join("docker/docker-compose.cf-controlplane-observability.yaml")) @@ -524,6 +538,10 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { gateway["services"]["gateway"]["environment"]["OTEL_EXPORTER_OTLP_ENDPOINT"].as_str(), Some("http://clickstack:4317") ); + assert_eq!( + gateway["services"]["gateway"]["logging"]["driver"].as_str(), + Some("fluentd") + ); assert_eq!( gateway["networks"]["observability"]["external"].as_bool(), Some(true) @@ -558,6 +576,10 @@ fn observability_is_ephemeral_and_exports_both_routed_services() { .as_str(), Some("http-protobuf") ); + assert_eq!( + dataplane["services"]["dataplane"]["logging"]["driver"].as_str(), + Some("fluentd") + ); assert_eq!( dataplane["networks"]["observability"]["external"].as_bool(), Some(true) From 92c06d00adf8406f1ac7263938a7360d85b0b340 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Thu, 3 Sep 2026 17:17:56 +0100 Subject: [PATCH 10/10] feat: prepare version 0.3.0 Signed-off-by: lucarlig --- AGENTS.md | 1 + CHANGELOG.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 3 ++- 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.md diff --git a/AGENTS.md b/AGENTS.md index 526bbcd..b1ebeeb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,3 +10,4 @@ Scope: - Keep generated checkout/build/runtime state under `.integration/` or `CF_INTEGRATION_DIR`. - Preserve the public routing contract: `/servers/{virtual_host_id}/mcp` goes to `cf-dataplane`; raw `/mcp` and UI/API traffic go to `cf-controlplane`. - Use published `cf-dataplane` images by default. Local builds should be explicit overrides. +- Keep `CHANGELOG.md` current for user-visible behavior, CLI, workflow, and packaging changes. Add normal changes under `Unreleased`; when bumping the package version, move those entries into a dated version section. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..cf474ce --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,74 @@ +# Changelog + +All notable changes to `cf-integration` are recorded here. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.3.0] - 2026-09-03 + +### Added + +- Added semantic `modern` and `legacy` MCP protocol selectors across commands. +- Standardized routed workflow selection on `builtin`, `external`, and, where + applicable, `fixture-direct` lanes. +- Added standalone external-dataplane load tests that disable the control plane + during the measured phase and populate a per-run mocked Redis snapshot using + the dataplane's current routing schema. +- Added no-login ClickStack observability with control-plane and dataplane + traces, native dataplane HTTP metrics, and routed-service logs. +- Added a concise command guide covering stack, probe, load, live, conformance, + CI, and debug workflows. + +### Changed + +- Reused the external conformance stack between compatible server and client + phases while preserving setup, execution, and cleanup failures. +- Moved ClickStack into an independent Compose lifecycle so managed test cleanup + leaves telemetry available for inspection; explicit `stack down` removes it. +- Made ClickStack the default for non-performance workflows and an explicit + opt-in for load tests to avoid skewing benchmark results. + +### Fixed + +- Flushed conformance results before propagating a failed child-process exit. +- Prevented duplicate ClickStack trace and metric ingestion by using its built-in + OTLP pipelines once. + +## [0.2.0] - 2026-09-01 + +### Added + +- Added embedded runtime assets so the published crate works outside its source + checkout. +- Added the three-lane MCP conformance matrix, checked-in baselines, isolated + official fixtures, and client-conformance coverage for the external dataplane. +- Added native release binaries and automated crate publishing. + +### Changed + +- Consolidated the harness into the `cf-integration` package and decomposed its + runtime into focused stack, MCP, conformance, and performance workflows. +- Standardized terminal progress and conformance result reporting. + +### Fixed + +- Made runtime paths, Docker Compose invocation, image pulls, and source-image + builds portable across supported hosts. +- Preserved all conformance lane and cleanup failures in final results. + +## [0.1.0] - 2026-08-28 + +### Added + +- Initial Rust CLI release for ContextForge stack orchestration, routed MCP + probing, live tests, load tests, and official conformance execution. +- Added builtin and external dataplane routing through reusable Docker Compose + overlays. + +[Unreleased]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.1.0...v0.2.0 +[0.1.0]: https://github.com/contextforge-org/contextforge-dev-tools/releases/tag/v0.1.0 diff --git a/Cargo.lock b/Cargo.lock index 3df804e..8d2e0f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,7 @@ dependencies = [ [[package]] name = "cf-integration" -version = "0.2.1" +version = "0.3.0" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index f1d3121..f8cf40d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cf-integration" -version = "0.2.1" +version = "0.3.0" edition = "2024" rust-version = "1.97" license = "Apache-2.0" @@ -23,6 +23,7 @@ include = [ "/scripts/conformance/write_client_config.py", "/tests/conformance/baselines/**", "/README.md", + "/CHANGELOG.md", "/LICENSE", ]