diff --git a/crates/tracedecay-cli/src/cli/help.rs b/crates/tracedecay-cli/src/cli/help.rs index b1bdb2bfe4..b0b3b47b50 100644 --- a/crates/tracedecay-cli/src/cli/help.rs +++ b/crates/tracedecay-cli/src/cli/help.rs @@ -364,7 +364,10 @@ Exit status (a completed binary upgrade stays installed in every case): 75 the refresh waits on an operator step: an interactive host step (Kimi Code's `/plugins install`), or a tracked host whose CLI is not installed (install it, or `tracedecay uninstall --agent ` to - stop tracking it); act on the printed step, then rerun + stop tracking it); act on the printed step, then rerun. Also 75 when + the restored daemon serves a store whose persisted shape this binary + does not open: it names the store and the exact reset command + (`tracedecay wipe --all --yes`), which nothing runs on your behalf Related: tracedecay upgrade (refresh only after a real install), tracedecay update-plugin (plugins only), tracedecay channel."; @@ -456,6 +459,12 @@ pub(crate) const DOCTOR_AFTER_HELP: &str = "\ Examples: tracedecay doctor Check everything +Exit status: + 0 no issue found (warnings may still be printed) + 1 an issue was found + 75 no issue, but the daemon serves a store in its reset-required state; + Doctor names the store and the exact reset command to run + Related: tracedecay install (fix missing integration), tracedecay daemon status, tracedecay status (index health)."; diff --git a/crates/tracedecay-cli/src/main.rs b/crates/tracedecay-cli/src/main.rs index 547c3fbbda..fa79cd4f57 100644 --- a/crates/tracedecay-cli/src/main.rs +++ b/crates/tracedecay-cli/src/main.rs @@ -1221,10 +1221,7 @@ async fn dispatch_command( dispatch_configuration_command(profile, command).await?; Ok(CommandOutcome::Success) } - CommandFamily::Diagnostics => { - dispatch_diagnostics_command(profile, command).await?; - Ok(CommandOutcome::Success) - } + CommandFamily::Diagnostics => dispatch_diagnostics_command(profile, command).await, CommandFamily::Knowledge => { dispatch_knowledge_command(profile, command).await?; Ok(CommandOutcome::Success) @@ -1915,14 +1912,19 @@ async fn dispatch_configuration_command( async fn dispatch_diagnostics_command( profile: &ProfileRoot, command: Commands, -) -> tracedecay_domain::errors::Result<()> { +) -> tracedecay_domain::errors::Result { match command { Commands::Doctor => { - hotpath::future!( + let completion = hotpath::future!( tracedecay::doctor::run_doctor(profile, crate::cloud::doctor_network_probes(),), label = "cli.doctor.run" ) .await?; + if completion == tracedecay::doctor::DoctorCompletion::PendingOperatorAction { + return Ok(CommandOutcome::Exit( + agent_cmd::PENDING_OPERATOR_ACTION_EXIT_CODE, + )); + } } Commands::Cost { range, @@ -1952,7 +1954,7 @@ async fn dispatch_diagnostics_command( } _ => unreachable!("non-diagnostics command passed to diagnostics dispatcher"), } - Ok(()) + Ok(CommandOutcome::Success) } async fn dispatch_knowledge_command( diff --git a/crates/tracedecay-cli/src/update_cmd.rs b/crates/tracedecay-cli/src/update_cmd.rs index 0711aa1435..b97e7a51c6 100644 --- a/crates/tracedecay-cli/src/update_cmd.rs +++ b/crates/tracedecay-cli/src/update_cmd.rs @@ -13,6 +13,7 @@ use tracedecay_runtime_core::config::ProfileRoot; use crate::agent_cmd::{HostLifecycleCompletion, HostLifecycleSummary}; use crate::upgrade::UpgradeOutcome; use tracedecay_daemon_control as daemon_control; +use tracedecay_domain::errors::StoreResetRequiredV1; use tracedecay_session_memory::user_config::UserConfig; /// Rewrites the installed daemon service while preserving its captured @@ -319,34 +320,41 @@ pub(crate) async fn run_update_command( profile: &ProfileRoot, no_reinstall: bool, ) -> tracedecay_domain::errors::Result { - let refresh = run_update_flow(profile, "update", RefreshPolicy::Always, no_reinstall).await?; - update_completion(refresh) + let outcome = run_update_flow(profile, "update", RefreshPolicy::Always, no_reinstall).await?; + update_completion(outcome.refresh, &outcome.reset_required) } /// `update` keeps a successful binary upgrade while reporting the refresh as -/// what it was: a failed refresh fails the command, a pending host step -/// exits with the pending-operator-action status. +/// what it was: a failed refresh fails the command, a pending host step or a +/// store the restored daemon serves reset-required exits with the +/// pending-operator-action status. fn update_completion( refresh: Option, + reset_required: &[StoreResetRequiredV1], ) -> tracedecay_domain::errors::Result { - match refresh { - None | Some(PluginRefreshOutcome::Complete) => Ok(HostLifecycleCompletion::Complete), - Some(PluginRefreshOutcome::PendingOperatorAction) => { - eprintln!( - "\nThe TraceDecay binary is up to date; the plugin refresh is waiting on the \ - operator action listed above." - ); - Ok(HostLifecycleCompletion::PendingOperatorAction) - } - Some(PluginRefreshOutcome::Failed) => { - Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "the TraceDecay binary is up to date, but the plugin and \ - agent-integration refresh failed (see above); fix it and run \ - `tracedecay update` again" - .to_string(), - }) - } + if refresh == Some(PluginRefreshOutcome::Failed) { + return Err(tracedecay_domain::errors::TraceDecayError::Config { + message: "the TraceDecay binary is up to date, but the plugin and \ + agent-integration refresh failed (see above); fix it and run \ + `tracedecay update` again" + .to_string(), + }); } + if !reset_required.is_empty() { + eprintln!( + "\nThe TraceDecay binary and daemon are up to date; the daemon serves the stores \ + listed above in their reset-required state until the operator runs the named reset." + ); + return Ok(HostLifecycleCompletion::PendingOperatorAction); + } + if refresh == Some(PluginRefreshOutcome::PendingOperatorAction) { + eprintln!( + "\nThe TraceDecay binary is up to date; the plugin refresh is waiting on the \ + operator action listed above." + ); + return Ok(HostLifecycleCompletion::PendingOperatorAction); + } + Ok(HostLifecycleCompletion::Complete) } #[hotpath::measure(label = "cli.upgrade.run", future = true)] @@ -361,7 +369,14 @@ pub(crate) async fn run_upgrade_command( no_reinstall, ) .await - .map(|_| ()) + .map(drop) +} + +/// What one `update` / `upgrade` run concluded after the daemon restore. +struct UpdateFlowOutcome { + refresh: Option, + /// Stores the restored daemon serves in their typed reset-required state. + reset_required: Vec, } async fn run_update_flow( @@ -369,8 +384,8 @@ async fn run_update_flow( operation: &str, refresh_policy: RefreshPolicy, no_reinstall: bool, -) -> tracedecay_domain::errors::Result> { - let refresh = daemon_control::with_exclusive_maintenance_window( +) -> tracedecay_domain::errors::Result { + let (refresh, installed_version) = daemon_control::with_exclusive_maintenance_window( profile, operation, crate::product_runtime::PRODUCT_BUILD_VERSION, @@ -384,88 +399,64 @@ async fn run_update_flow( // validates the binary it actually starts, not the one that was // running before the upgrade. Ok(daemon_control::MaintenanceWindowOutcome { - value: outcome.refresh, + value: (outcome.refresh, outcome.installed_version.clone()), installed_version: outcome.installed_version, }) }, )?; - reset_refused_profile_authorities(profile, operation).await?; - Ok(refresh) -} - -/// What the post-window profile probe decided. -#[derive(Debug, PartialEq, Eq)] -enum ProfileResetDecision { - /// The restored daemon opens the profile; nothing to reset. - Opens, - /// The daemon refused the profile with a typed reset state. - Reset { authority: String, reason: String }, - /// The probe could not decide (no reachable daemon, transport failure); - /// reported, never acted on. - Undecided(String), -} - -fn profile_reset_decision( - probe: tracedecay_domain::errors::Result, -) -> ProfileResetDecision { - match probe { - Ok(_) => ProfileResetDecision::Opens, - Err(error) => match error.reset_required_context() { - Some((authority, reason)) => ProfileResetDecision::Reset { - authority: authority.to_owned(), - reason: reason.to_owned(), - }, - None => ProfileResetDecision::Undecided(error.to_string()), - }, - } + let reset_required = restored_daemon_pending_resets( + profile, + operation, + installed_version + .as_deref() + .unwrap_or(crate::product_runtime::PRODUCT_BUILD_VERSION), + ); + Ok(UpdateFlowOutcome { + refresh, + reset_required, + }) } -/// The upgrade journey ends with core tools that work, not with a typed -/// refusal on the first tool call. Once the maintenance window has restored -/// the daemon on the new binary, a projectless probe asks it to open the -/// profile registry; a typed reset refusal is acted on here by resetting the -/// complete profile database state, the one reset authority a profile-scoped -/// refusal has. Old shapes are deleted, never migrated or backed up. -async fn reset_refused_profile_authorities( +/// A restored daemon serves a store it cannot open in that store's typed +/// reset-required state. The upgrade journey never deletes profile data on +/// its own: each such store is reported with the exact reset command as the +/// operator's pending action. +fn restored_daemon_pending_resets( profile: &ProfileRoot, operation: &str, -) -> tracedecay_domain::errors::Result<()> { + expected_version: &str, +) -> Vec { if !daemon_control::daemon_reachable(profile) { eprintln!( "No reachable TraceDecay daemon after {operation}; the profile's persisted shape is \ checked on the daemon's next open." ); - return Ok(()); + return Vec::new(); } - let probe = crate::commands::daemon_tool_json( - profile, - None, - "tracedecay_project_list", - serde_json::json!({ "limit": 1, "format": "json" }), - ) - .await; - match profile_reset_decision(probe) { - ProfileResetDecision::Opens => Ok(()), - ProfileResetDecision::Reset { authority, reason } => { - eprintln!( - "\n\x1b[33mThe upgraded daemon refuses the persisted {authority} shape: \ - {reason}\x1b[0m\n\ - Resetting the complete profile database state so core tools work on this \ - binary; refused authorities are never migrated or backed up. Re-run \ - `tracedecay init ` for each project afterwards." - ); - crate::commands::handle_wipe(profile, true, true).await + match daemon_control::daemon_reset_required_stores(profile, expected_version) { + Ok(stores) => { + for store in &stores { + eprintln!("{}", pending_reset_line(store)); + } + stores } - ProfileResetDecision::Undecided(detail) => { + Err(error) => { eprintln!( - " \x1b[33mwarning:\x1b[0m could not verify that the profile opens after \ - {operation}: {detail}" + " \x1b[33mwarning:\x1b[0m could not verify that the daemon serves every store \ + after {operation}: {error}" ); - Ok(()) + Vec::new() } } } +fn pending_reset_line(store: &StoreResetRequiredV1) -> String { + format!( + " \x1b[33mpending operator action:\x1b[0m {} requires reset ({}); run `{}`", + store.store, store.reason, store.remedy + ) +} + fn combine_operation_and_restore( operation: &str, operation_result: tracedecay_domain::errors::Result, @@ -781,45 +772,54 @@ mod tests { use std::path::{Path, PathBuf}; use super::{ - InstallThenRefresh, PluginRefreshOutcome, ProfileResetDecision, RefreshPolicy, - current_tracedecay_exe_from, install_pass_covers_tracked_agents, post_update_binary, - post_update_binary_from, prepare_post_update_lease, profile_reset_decision, - restart_daemon_service_with, run_install_then_refresh, update_completion, + InstallThenRefresh, PluginRefreshOutcome, RefreshPolicy, current_tracedecay_exe_from, + install_pass_covers_tracked_agents, pending_reset_line, post_update_binary, + post_update_binary_from, prepare_post_update_lease, restart_daemon_service_with, + run_install_then_refresh, update_completion, }; use crate::agent_cmd::HostLifecycleCompletion; use crate::upgrade::UpgradeOutcome; use tempfile::TempDir; use tracedecay_daemon_control as daemon_control; - /// Only the typed reset state authorizes deleting profile data after an - /// update; an opening profile is left alone and every other failure is - /// reported without acting. + fn git_correlation_reset() -> tracedecay_domain::errors::StoreResetRequiredV1 { + tracedecay_domain::errors::TraceDecayError::ProfileResetRequired { + component: "git correlation", + found_version: Some(5), + required_version: 6, + } + .store_reset_required("profile sessions") + .expect("a versioned profile refusal is a reset-required store") + } + + /// A restored daemon serving a reset-required store completes the update + /// with the pending-operator-action status and names the exact reset; + /// nothing deletes the store on the operator's behalf. #[test] - fn post_update_profile_reset_acts_only_on_the_typed_reset_state() { + fn update_with_a_reset_required_store_is_pending_on_the_named_reset() { + let reset = git_correlation_reset(); assert_eq!( - profile_reset_decision(Ok(serde_json::json!({ "projects": [] }))), - ProfileResetDecision::Opens + update_completion( + Some(PluginRefreshOutcome::Complete), + std::slice::from_ref(&reset) + ) + .unwrap(), + HostLifecycleCompletion::PendingOperatorAction ); assert_eq!( - profile_reset_decision(Err( - tracedecay_domain::errors::TraceDecayError::reset_required( - "session temporal", - "published v3 shape", - ) - )), - ProfileResetDecision::Reset { - authority: "session temporal".to_owned(), - reason: "published v3 shape".to_owned(), - } + update_completion(None, std::slice::from_ref(&reset)).unwrap(), + HostLifecycleCompletion::PendingOperatorAction ); assert_eq!( - profile_reset_decision(Err(tracedecay_domain::errors::TraceDecayError::Config { - message: "daemon tool call failed: storage unavailable".to_owned(), - })), - ProfileResetDecision::Undecided( - "config error: daemon tool call failed: storage unavailable".to_owned() - ) + pending_reset_line(&reset), + " \x1b[33mpending operator action:\x1b[0m profile sessions requires reset \ + (git correlation profile schema 5 is incompatible with required schema 6; reset \ + the profile); run `tracedecay wipe --all --yes`" ); + let failed = update_completion(Some(PluginRefreshOutcome::Failed), &[reset]) + .expect_err("a failed refresh still fails `update`") + .to_string(); + assert!(failed.contains("refresh failed"), "{failed}"); } #[test] @@ -1190,7 +1190,7 @@ mod tests { } ); assert_eq!(calls.into_inner(), vec!["upgrade", "post-update"]); - let failed = update_completion(outcome.refresh) + let failed = update_completion(outcome.refresh, &[]) .expect_err("`update` still fails the refresh it could not run") .to_string(); assert_eq!( @@ -1203,18 +1203,18 @@ mod tests { #[test] fn update_exit_reports_the_refresh_truthfully() { assert_eq!( - update_completion(None).unwrap(), + update_completion(None, &[]).unwrap(), HostLifecycleCompletion::Complete ); assert_eq!( - update_completion(Some(PluginRefreshOutcome::Complete)).unwrap(), + update_completion(Some(PluginRefreshOutcome::Complete), &[]).unwrap(), HostLifecycleCompletion::Complete ); assert_eq!( - update_completion(Some(PluginRefreshOutcome::PendingOperatorAction)).unwrap(), + update_completion(Some(PluginRefreshOutcome::PendingOperatorAction), &[]).unwrap(), HostLifecycleCompletion::PendingOperatorAction ); - let failed = update_completion(Some(PluginRefreshOutcome::Failed)) + let failed = update_completion(Some(PluginRefreshOutcome::Failed), &[]) .expect_err("a failed refresh must fail `update`") .to_string(); assert!(failed.contains("binary is up to date"), "{failed}"); diff --git a/crates/tracedecay-daemon-control/src/lib.rs b/crates/tracedecay-daemon-control/src/lib.rs index a6cd0533fe..cd85713f01 100644 --- a/crates/tracedecay-daemon-control/src/lib.rs +++ b/crates/tracedecay-daemon-control/src/lib.rs @@ -84,10 +84,10 @@ mod service; pub use service::{ DaemonProcessProofV1, DaemonServiceMemoryLimitsV1, DaemonServiceSpec, DaemonServiceState, - MaintenanceWindowOutcome, QuiescedDaemonLifecycle, daemon_reachable, daemon_socket_connectable, - default_socket_path, install_service, install_service_under_lease, - installed_service_process_proof, installed_service_socket_path, installed_service_state, - prepare_scoop_package_service, quiesce_installed_service_before_lease, + MaintenanceWindowOutcome, QuiescedDaemonLifecycle, daemon_reachable, + daemon_reset_required_stores, daemon_socket_connectable, default_socket_path, install_service, + install_service_under_lease, installed_service_process_proof, installed_service_socket_path, + installed_service_state, prepare_scoop_package_service, quiesce_installed_service_before_lease, refresh_installed_service_under_lease_with_state, restore_installed_service_after_update, restore_scoop_package_service, service_spec, service_spec_with_remote_tls, service_status, socket_path_or_default, start_service, stop_service, unavailable_daemon_socket_advice, diff --git a/crates/tracedecay-daemon-control/src/service.rs b/crates/tracedecay-daemon-control/src/service.rs index 18a3a0a722..a8dcb1e762 100644 --- a/crates/tracedecay-daemon-control/src/service.rs +++ b/crates/tracedecay-daemon-control/src/service.rs @@ -27,7 +27,9 @@ mod tests; #[allow(clippy::expect_used)] mod update_restore_tests; -pub use probe::{DaemonProcessProofV1, daemon_reachable, daemon_socket_connectable}; +pub use probe::{ + DaemonProcessProofV1, daemon_reachable, daemon_reset_required_stores, daemon_socket_connectable, +}; pub use unit_file::installed_service_socket_path; use probe::{ @@ -1610,7 +1612,7 @@ fn restored_service_matches( } if expected.is_running() { matches!(socket_state, DaemonSocketState::Connectable) - && matches!(protocol_state, DaemonProtocolState::Ready) + && matches!(protocol_state, DaemonProtocolState::Ready { .. }) } else { socket_state.is_proven_quiesced() } diff --git a/crates/tracedecay-daemon-control/src/service/probe.rs b/crates/tracedecay-daemon-control/src/service/probe.rs index 7a912ef51d..5e20060a1e 100644 --- a/crates/tracedecay-daemon-control/src/service/probe.rs +++ b/crates/tracedecay-daemon-control/src/service/probe.rs @@ -10,7 +10,7 @@ use std::time::Duration; use tracedecay_daemon_identity::authority; #[cfg(unix)] use tracedecay_daemon_identity::client_connection; -use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_domain::errors::{Result, StoreResetRequiredV1, TraceDecayError}; use tracedecay_runtime_core::config::ProfileRoot; use super::default_socket_path; @@ -163,7 +163,7 @@ pub(super) fn probe_daemon_process_with_timeout( fn proof_from_protocol(protocol: DaemonProtocolState) -> DaemonProcessProofV1 { match protocol { - DaemonProtocolState::Ready => DaemonProcessProofV1::Ready, + DaemonProtocolState::Ready { .. } => DaemonProcessProofV1::Ready, DaemonProtocolState::IdentityMismatch { name, version, @@ -201,7 +201,12 @@ pub(super) enum DaemonSocketState { #[derive(Debug, PartialEq, Eq)] pub(super) enum DaemonProtocolState { NotRequired, - Ready, + /// initialize named this build. A ready daemon may still serve some + /// registered stores in their typed reset-required state; those are the + /// operator's pending reset, not a failed restore. + Ready { + reset_required_stores: Vec, + }, Unresponsive(String), IdentityMismatch { name: Option, @@ -214,7 +219,16 @@ impl std::fmt::Display for DaemonProtocolState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::NotRequired => f.write_str("not required"), - Self::Ready => f.write_str("ready"), + Self::Ready { + reset_required_stores, + } if reset_required_stores.is_empty() => f.write_str("ready"), + Self::Ready { + reset_required_stores, + } => write!( + f, + "ready ({} store(s) require reset)", + reset_required_stores.len() + ), Self::Unresponsive(error) => write!(f, "unresponsive ({error})"), Self::IdentityMismatch { name, @@ -245,23 +259,30 @@ pub(super) fn daemon_protocol_state_with_timeout( } fn classify_daemon_protocol_identity( - identity: Result<(Option, Option)>, + identity: Result, expected_version: &str, ) -> DaemonProtocolState { match identity { - Ok((name, version)) - if name.as_deref() == Some("tracedecay") - && version.as_deref().is_some_and(|version| { - tracedecay_daemon_protocol::versions_name_same_build(version, expected_version) - }) => - { - DaemonProtocolState::Ready - } - Ok((name, version)) => DaemonProtocolState::IdentityMismatch { + Ok(DaemonInitializeIdentity { name, version, - expected_version: expected_version.to_owned(), - }, + reset_required_stores, + }) if name.as_deref() == Some("tracedecay") + && version.as_deref().is_some_and(|version| { + tracedecay_daemon_protocol::versions_name_same_build(version, expected_version) + }) => + { + DaemonProtocolState::Ready { + reset_required_stores, + } + } + Ok(DaemonInitializeIdentity { name, version, .. }) => { + DaemonProtocolState::IdentityMismatch { + name, + version, + expected_version: expected_version.to_owned(), + } + } Err(error) => DaemonProtocolState::Unresponsive(error.to_string()), } } @@ -380,13 +401,21 @@ pub(super) fn daemon_readiness_probe( ) } +/// What a completed initialize exchange reported. +#[derive(Debug)] +pub(super) struct DaemonInitializeIdentity { + name: Option, + version: Option, + reset_required_stores: Vec, +} + fn query_daemon_identity_stream( profile: &ProfileRoot, mut stream: impl ProbeStream, auth_token: &str, client_version: &str, deadline: std::time::Instant, -) -> Result<(Option, Option)> { +) -> Result { const REQUEST_ID: i64 = 1; let handshake = crate::handshake_for_current_client(profile, client_version, None, None, false, false)?; @@ -436,7 +465,48 @@ fn query_daemon_identity_stream( .pointer("/result/serverInfo/version") .and_then(serde_json::Value::as_str) .map(str::to_string); - return Ok((name, version)); + // A daemon predating the typed reset-required state names none. + let reset_required_stores = match response + .pointer("/result/_meta") + .and_then(|meta| meta.get(tracedecay_daemon_protocol::RESET_REQUIRED_STORES_META_KEY)) + { + Some(stores) => serde_json::from_value(stores.clone())?, + None => Vec::new(), + }; + return Ok(DaemonInitializeIdentity { + name, + version, + reset_required_stores, + }); + } +} + +/// The registered stores the default socket's daemon serves in their typed +/// reset-required state, each naming the command that resets it. A daemon +/// that does not complete initialize as this build is an error, never an +/// empty list. +pub fn daemon_reset_required_stores( + profile: &ProfileRoot, + expected_version: &str, +) -> Result> { + const RESET_REQUIRED_PROBE_TIMEOUT: Duration = Duration::from_secs(10); + let socket_path = default_socket_path(profile.data_dir())?; + match daemon_readiness_probe( + profile, + &socket_path, + expected_version, + RESET_REQUIRED_PROBE_TIMEOUT, + ) + .1 + { + DaemonProtocolState::Ready { + reset_required_stores, + } => Ok(reset_required_stores), + protocol => Err(TraceDecayError::Config { + message: format!( + "could not read the daemon's reset-required stores: protocol readiness is {protocol}" + ), + }), } } @@ -680,13 +750,17 @@ fn missing_loopback_authority() -> TraceDecayError { #[cfg(test)] mod identity_classification_tests { - use super::{DaemonProtocolState, classify_daemon_protocol_identity}; + use super::{DaemonInitializeIdentity, DaemonProtocolState, classify_daemon_protocol_identity}; const SHA: &str = "84598a0b9c841b914565f46b20bb6c765706e8e5"; /// The identity a `tracedecay` daemon reporting `version` answers with. - fn identity(version: &str) -> (Option, Option) { - (Some("tracedecay".to_owned()), Some(version.to_owned())) + fn identity(version: &str) -> DaemonInitializeIdentity { + DaemonInitializeIdentity { + name: Some("tracedecay".to_owned()), + version: Some(version.to_owned()), + reset_required_stores: Vec::new(), + } } /// `tracedecay update` installs a release and then waits for the daemon @@ -700,7 +774,9 @@ mod identity_classification_tests { Ok(identity(&format!("0.1.0-beta.47+{SHA}"))), "0.1.0-beta.47", ), - DaemonProtocolState::Ready + DaemonProtocolState::Ready { + reset_required_stores: Vec::new() + } ); } diff --git a/crates/tracedecay-daemon-control/src/service/tests.rs b/crates/tracedecay-daemon-control/src/service/tests.rs index e404a3dfcb..40e2d578f5 100644 --- a/crates/tracedecay-daemon-control/src/service/tests.rs +++ b/crates/tracedecay-daemon-control/src/service/tests.rs @@ -361,7 +361,9 @@ fn strict_restoration_requires_readiness_only_for_running_state() { DaemonServiceState::RunningEnabled, DaemonServiceState::RunningEnabled, super::probe::DaemonSocketState::Connectable, - &super::probe::DaemonProtocolState::Ready, + &super::probe::DaemonProtocolState::Ready { + reset_required_stores: Vec::new(), + }, )); assert!(!super::restored_service_matches( DaemonServiceState::RunningEnabled, @@ -379,7 +381,9 @@ fn strict_restoration_requires_readiness_only_for_running_state() { DaemonServiceState::StoppedEnabled, DaemonServiceState::RunningEnabled, super::probe::DaemonSocketState::Connectable, - &super::probe::DaemonProtocolState::Ready, + &super::probe::DaemonProtocolState::Ready { + reset_required_stores: Vec::new(), + }, )); assert!(!super::restored_service_matches( DaemonServiceState::RunningEnabled, @@ -452,6 +456,21 @@ pub(super) fn serve_probe_response( name: &'static str, version: &'static str, expected_auth_token: String, +) -> std::thread::JoinHandle<()> { + serve_probe_result( + listener, + serde_json::json!({ "serverInfo": {"name": name, "version": version} }), + expected_auth_token, + ) +} + +/// Answers one authenticated readiness probe with `result` as the +/// initialize result. +#[cfg(unix)] +pub(super) fn serve_probe_result( + listener: UnixListener, + result: serde_json::Value, + expected_auth_token: String, ) -> std::thread::JoinHandle<()> { std::thread::spawn(move || { let mut stream = accept_readiness_probe(&listener); @@ -467,9 +486,7 @@ pub(super) fn serve_probe_response( let response = serde_json::json!({ "jsonrpc": "2.0", "id": request["id"], - "result": { - "serverInfo": {"name": name, "version": version} - } + "result": result, }); writeln!(stream, "{response}").expect("write initialize response"); }) @@ -614,7 +631,9 @@ fn daemon_protocol_probe_requires_current_tracedecay_identity() { std::time::Duration::from_secs(10), ) .1, - super::probe::DaemonProtocolState::Ready + super::probe::DaemonProtocolState::Ready { + reset_required_stores: Vec::new(), + } ); ready_server.join().expect("join ready server"); @@ -1017,7 +1036,12 @@ fn running_service_snapshot_uses_one_authenticated_connection() { assert_eq!(snapshot.0, DaemonServiceState::RunningEnabled); assert_eq!(snapshot.2, super::probe::DaemonSocketState::Connectable); - assert_eq!(snapshot.3, super::probe::DaemonProtocolState::Ready); + assert_eq!( + snapshot.3, + super::probe::DaemonProtocolState::Ready { + reset_required_stores: Vec::new(), + } + ); assert_eq!(accepts.load(Ordering::SeqCst), 1); } diff --git a/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs b/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs index 6a463f2a37..61346e5f4a 100644 --- a/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs +++ b/crates/tracedecay-daemon-control/src/service/update_restore_tests.rs @@ -263,6 +263,66 @@ fn maintenance_window_waits_out_the_lease_of_the_daemon_it_just_stopped() { fixture.assert_daemon_running_after("a completed maintenance window"); } +/// A restored daemon that serves a registered store in its typed +/// reset-required state is restored: the restore check sees the unit +/// `RunningEnabled`, a connectable socket and initialize from this build, and +/// hands back the operator's pending reset instead of failing the update. +#[cfg(target_os = "linux")] +#[test] +fn restore_check_accepts_a_reset_required_daemon_and_returns_its_pending_reset() { + let _env_lock = lock_user_data_dir_test_env(); + let fixture = DrainingDaemonFixture::new(); + let authority = super::tests::seed_socket_authority(&fixture.socket_path); + let listener = UnixListener::bind(&fixture.socket_path).expect("bind restored daemon"); + let server = super::tests::serve_probe_result( + listener, + serde_json::json!({ + "serverInfo": {"name": "tracedecay", "version": super::tests::TEST_BUILD_VERSION}, + "_meta": { + "tracedecay/reset_required_stores": [{ + "store": "profile sessions", + "authority": "git correlation", + "found_version": 5, + "required_version": 6, + "reason": "git correlation profile schema 5 is incompatible with required schema 6; reset the profile", + "remedy": "tracedecay wipe --all --yes", + }], + }, + }), + authority.auth_token().to_owned(), + ); + + let (state, _, socket, protocol) = + super::installed_service_status_snapshot(&fixture.runner, super::tests::TEST_BUILD_VERSION) + .expect("restored service snapshot"); + server.join().expect("join restored daemon"); + + assert!( + super::restored_service_matches( + DaemonServiceState::RunningEnabled, + state, + socket, + &protocol + ), + "a reset-required daemon is restored: service {state:?}, socket {socket}, protocol {protocol}" + ); + assert_eq!( + protocol, + super::probe::DaemonProtocolState::Ready { + reset_required_stores: vec![tracedecay_domain::errors::StoreResetRequiredV1 { + store: "profile sessions".to_owned(), + authority: "git correlation".to_owned(), + found_version: Some(5), + required_version: Some(6), + reason: "git correlation profile schema 5 is incompatible with required schema 6; \ + reset the profile" + .to_owned(), + remedy: "tracedecay wipe --all --yes".to_owned(), + }], + } + ); +} + /// A holder that outlives the bound is contention, reported typed, and the /// failure path still restores the daemon it stopped: contention must never /// leave a previously healthy daemon stopped. diff --git a/crates/tracedecay-daemon-control/src/service/windows_task.rs b/crates/tracedecay-daemon-control/src/service/windows_task.rs index 8b67dd09fa..9de912c391 100644 --- a/crates/tracedecay-daemon-control/src/service/windows_task.rs +++ b/crates/tracedecay-daemon-control/src/service/windows_task.rs @@ -403,7 +403,7 @@ impl DaemonControlApi for NativeDaemonControl { timeout, ); ControlObservation { - satisfied: matches!(protocol, super::probe::DaemonProtocolState::Ready), + satisfied: matches!(protocol, super::probe::DaemonProtocolState::Ready { .. }), diagnostic: format!("protocol {protocol}"), } } diff --git a/crates/tracedecay-daemon-protocol/src/contract/mod.rs b/crates/tracedecay-daemon-protocol/src/contract/mod.rs index 9f7df41d0f..c2ec0af95b 100644 --- a/crates/tracedecay-daemon-protocol/src/contract/mod.rs +++ b/crates/tracedecay-daemon-protocol/src/contract/mod.rs @@ -91,6 +91,9 @@ pub const DAEMON_INVOCATION_REVISION: u16 = 1; /// Request method a control client sends over the closed protocol to ask the /// daemon to shut down. pub const DAEMON_SHUTDOWN_METHOD: &str = "tracedecay/daemon/shutdown"; +/// `initialize` result `_meta` key listing every registered store the daemon +/// serves in a typed reset-required state. +pub const RESET_REQUIRED_STORES_META_KEY: &str = "tracedecay/reset_required_stores"; const DAEMON_INVOCATION_CANCEL_OPERATION: &str = "invocation_cancel"; const DAEMON_INVOCATION_DELIVERY_ACK_OPERATION: &str = "invocation_delivery_ack"; diff --git a/crates/tracedecay-daemon-protocol/src/lib.rs b/crates/tracedecay-daemon-protocol/src/lib.rs index 763ba2f539..a0f65486a2 100644 --- a/crates/tracedecay-daemon-protocol/src/lib.rs +++ b/crates/tracedecay-daemon-protocol/src/lib.rs @@ -96,8 +96,8 @@ pub use contract::{ DaemonInvocationDeliveryAckResponseOutcome, DaemonInvocationOperation, DaemonInvocationOutcome, DaemonInvocationPayload, DaemonInvocationProblem, DaemonInvocationRequest, DaemonInvocationResponse, DaemonLspSessionAccess, HandoffApplicationInvocationV1, - HandoffApplicationOutcomeV1, WorkApplicationInvocationV1, WorkApplicationOutcomeV1, - WorkflowApplicationInvocation, WorkflowApplicationOutcome, + HandoffApplicationOutcomeV1, RESET_REQUIRED_STORES_META_KEY, WorkApplicationInvocationV1, + WorkApplicationOutcomeV1, WorkflowApplicationInvocation, WorkflowApplicationOutcome, parse_daemon_invocation_cancellation_request, parse_daemon_invocation_delivery_ack_request, parse_daemon_invocation_request, }; diff --git a/crates/tracedecay-domain/src/errors.rs b/crates/tracedecay-domain/src/errors.rs index f229e1e1fa..5d4d6a5c94 100644 --- a/crates/tracedecay-domain/src/errors.rs +++ b/crates/tracedecay-domain/src/errors.rs @@ -67,8 +67,9 @@ pub enum TraceDecayError { HostCliUnavailable { program: String, lifecycle: String }, #[error( - "{component} profile schema {found_version:?} is incompatible with required schema \ - {required_version}; reset the profile" + "{component} profile schema {} is incompatible with required schema \ + {required_version}; reset the profile", + .found_version.map_or_else(|| "unversioned".to_owned(), |version| version.to_string()) )] ProfileResetRequired { component: &'static str, @@ -112,6 +113,25 @@ pub enum TraceDecayError { pub type Result = std::result::Result; +/// The one command that resets every profile-scoped persisted shape. Refused +/// shapes are never migrated or backed up: the reset deletes the old data and +/// the next open creates the shape the running binary writes. +pub const PROFILE_RESET_COMMAND: &str = "tracedecay wipe --all --yes"; + +/// A persisted store the daemon keeps mounted in a typed reset-required state +/// instead of refusing to serve: every read against it returns the typed +/// refusal, stores that admit keep serving, and `remedy` is the exact command +/// the operator runs to reset it. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct StoreResetRequiredV1 { + pub store: String, + pub authority: String, + pub found_version: Option, + pub required_version: Option, + pub reason: String, + pub remedy: String, +} + /// Flatten an error and its [`std::error::Error::source`] chain into one /// message string. /// @@ -151,6 +171,32 @@ impl TraceDecayError { Some((authority, reason)) } + /// The typed reset-required state `store` is mounted in when this error is + /// a profile-scoped persisted-shape refusal, `None` for any other failure. + pub fn store_reset_required(&self, store: impl Into) -> Option { + let (authority, found_version, required_version) = match self { + Self::ResetRequired { authority, .. } => (authority.clone(), None, None), + Self::ProfileResetRequired { + component, + found_version, + required_version, + } => ( + (*component).to_owned(), + *found_version, + Some(*required_version), + ), + _ => return None, + }; + Some(StoreResetRequiredV1 { + store: store.into(), + authority, + found_version, + required_version, + reason: self.to_string(), + remedy: PROFILE_RESET_COMMAND.to_owned(), + }) + } + pub fn project_route( reason_code: impl Into, retryable: bool, diff --git a/crates/tracedecay-global-db/src/registered/git_correlation_schema_tests.rs b/crates/tracedecay-global-db/src/registered/git_correlation_schema_tests.rs index ccc606c55e..304aed367e 100644 --- a/crates/tracedecay-global-db/src/registered/git_correlation_schema_tests.rs +++ b/crates/tracedecay-global-db/src/registered/git_correlation_schema_tests.rs @@ -40,7 +40,11 @@ async fn an_older_git_correlation_schema_is_refused_without_mutation() { assert!(matches!( error, - TraceDecayError::ResetRequired { ref authority, .. } if authority == "git correlation" + TraceDecayError::ProfileResetRequired { + component: "git correlation", + found_version: Some(5), + required_version: 6, + } )); assert_eq!( fs::read(&database_path).unwrap(), diff --git a/crates/tracedecay-global-db/src/schema_stages.rs b/crates/tracedecay-global-db/src/schema_stages.rs index 7faaa3f39d..bd9b9267e7 100644 --- a/crates/tracedecay-global-db/src/schema_stages.rs +++ b/crates/tracedecay-global-db/src/schema_stages.rs @@ -471,7 +471,7 @@ async fn classify_registered_schema_authorities( /// Git evidence is stored as per-session rows since schema version 6. A store /// recorded at any other version holds a shape nothing converts, so it keeps -/// its data untouched behind the typed reset. +/// its data untouched behind the typed, versioned reset. async fn require_admissible_git_correlation_schema( connection: &impl QueryExecutor, ) -> tracedecay_domain::errors::Result<()> { @@ -479,16 +479,13 @@ async fn require_admissible_git_correlation_schema( .await .map_err(|error| global_db_operation_error("inspect git correlation schema", error))?; match recorded { - Some(found) if found != GIT_CORRELATION_SCHEMA_VERSION => { - Err(tracedecay_domain::errors::TraceDecayError::reset_required( - "git correlation", - format!( - "the store records Git correlation schema version {found}; this build \ - stores Git evidence as per-session rows at version \ - {GIT_CORRELATION_SCHEMA_VERSION}" - ), - )) - } + Some(found) if found != GIT_CORRELATION_SCHEMA_VERSION => Err( + tracedecay_domain::errors::TraceDecayError::ProfileResetRequired { + component: "git correlation", + found_version: Some(found), + required_version: GIT_CORRELATION_SCHEMA_VERSION, + }, + ), _ => Ok(()), } } diff --git a/crates/tracedecay-mcp/src/handlers/info/status.rs b/crates/tracedecay-mcp/src/handlers/info/status.rs index a8324db605..47ed13a632 100644 --- a/crates/tracedecay-mcp/src/handlers/info/status.rs +++ b/crates/tracedecay-mcp/src/handlers/info/status.rs @@ -377,6 +377,7 @@ pub async fn handle_status( &ctx.store_runtime() .registered_schema_convergence_observations(), ); + output["reset_required_stores"] = json!(ctx.store_runtime().reset_required_stores()); let freshness_payload = hotpath::future!( ctx.freshness(), label = "mcp.info.status.code_index_freshness" @@ -745,6 +746,16 @@ fn render_status_md(value: &Value) -> String { } Value::Array(a) => { md.field(k, &format!("{} item(s)", a.len())); + if k == "reset_required_stores" { + for store in a { + md.bullet(&format!( + "pending operator action: {} requires reset ({}), run `{}`", + store["store"].as_str().unwrap_or_default(), + store["reason"].as_str().unwrap_or_default(), + store["remedy"].as_str().unwrap_or_default(), + )); + } + } } Value::Object(o) => { if let Some(status) = o.get("status").and_then(Value::as_str) { diff --git a/crates/tracedecay-mcp/src/lib.rs b/crates/tracedecay-mcp/src/lib.rs index 8db47fd838..8629edf1cd 100644 --- a/crates/tracedecay-mcp/src/lib.rs +++ b/crates/tracedecay-mcp/src/lib.rs @@ -98,9 +98,9 @@ pub use tool_context::{ McpToolContext, RequestControls, }; pub use tool_errors::{ - mark_semantic_tool_error, reset_required_command, reset_required_remedy, - semantic_failure_reason, serialize_response_line, structured_hook_error_data, - tool_error_response, tool_result_has_semantic_error, + mark_semantic_tool_error, reset_required_command, reset_required_context, + reset_required_remedy, semantic_failure_reason, serialize_response_line, + structured_hook_error_data, tool_error_response, tool_result_has_semantic_error, }; pub use tools::render::format_relative_time; pub use tools::{ diff --git a/crates/tracedecay-mcp/src/tool_errors.rs b/crates/tracedecay-mcp/src/tool_errors.rs index 671b686f68..01cf46a80d 100644 --- a/crates/tracedecay-mcp/src/tool_errors.rs +++ b/crates/tracedecay-mcp/src/tool_errors.rs @@ -1,7 +1,7 @@ //! Semantic tool-failure classification and JSON-RPC error-response mapping. use serde_json::{Value, json}; -use tracedecay_domain::errors::TraceDecayError; +use tracedecay_domain::errors::{PROFILE_RESET_COMMAND, TraceDecayError}; use tracedecay_sessions::admission::HostAdmissionStatus; use crate::response_handles::RESPONSE_RETRIEVE_TOOL; @@ -368,14 +368,13 @@ pub fn reset_required_command(authority: &str, project_root: Option<&std::path:: } else if is_host_artifact_authority(authority) { "delete the block or package directory named in the refusal".to_string() } else { - "tracedecay wipe --all --yes".to_string() + PROFILE_RESET_COMMAND.to_string() } } /// Operator rendering of [`reset_required_command`]: the refused authority, /// what the reset deletes, the command, and the re-initialization that -/// follows. `tracedecay update` performs the profile reset itself after -/// refreshing the binary and daemon. +/// follows. pub fn reset_required_remedy(authority: &str, project_root: Option<&std::path::Path>) -> String { let command = reset_required_command(authority, project_root); if is_project_store_authority(authority) { @@ -401,16 +400,14 @@ pub fn reset_required_remedy(authority: &str, project_root: Option<&std::path::P "refused authority: {authority}\n\ this binary does not open or migrate that shape; reset it (its old data is deleted, \ nothing is backed up):\n \ - tracedecay update refreshes the binary and daemon, then resets every \ - refused profile authority\n \ - {command} resets the complete profile database state now\n\ + {command} resets the complete profile database state\n\ then re-run `tracedecay init ` for each project" ) } /// The refused authority and its reason for both typed reset states: the -/// generic persisted-shape refusal and the LCM profile schema refusal. -fn reset_required_context(error: &TraceDecayError) -> Option<(String, String)> { +/// generic persisted-shape refusal and the versioned profile schema refusal. +pub fn reset_required_context(error: &TraceDecayError) -> Option<(String, String)> { match error { TraceDecayError::ResetRequired { authority, reason } => { Some((authority.clone(), reason.clone())) @@ -520,7 +517,7 @@ mod tests { let profile = super::reset_required_remedy("session temporal", None); assert!(profile.contains("refused authority: session temporal")); - assert!(profile.contains("\n tracedecay update "), "{profile}"); + assert!(!profile.contains("tracedecay update"), "{profile}"); assert!( profile.contains("\n tracedecay wipe --all --yes"), "{profile}" diff --git a/crates/tracedecay-store-runtime/src/session_registry.rs b/crates/tracedecay-store-runtime/src/session_registry.rs index 786dfc849f..006dcabbf0 100644 --- a/crates/tracedecay-store-runtime/src/session_registry.rs +++ b/crates/tracedecay-store-runtime/src/session_registry.rs @@ -14,7 +14,7 @@ use tracedecay_sessions::observation::ObservationCancellation; use tracedecay_store::{AdmissionConfigV1, ProjectId, StoreIncarnationV1, StoreShardIdV1}; use tracedecay_daemon_identity::profile_identity::LocalProfileIdentityAuthorityV1; -use tracedecay_domain::errors::{Result, TraceDecayError}; +use tracedecay_domain::errors::{Result, StoreResetRequiredV1, TraceDecayError}; use tracedecay_global_db::{RegisteredGlobalDbLeaseV1, RegisteredGlobalDbOwnerV1}; use tracedecay_graph_db::{GraphDbOwnerAttachmentV1, GraphDbRetirementCommit}; use tracedecay_runtime_core::RuntimeOperationTaskOwnerV1; @@ -2394,6 +2394,10 @@ pub struct DaemonSessionRuntimeRegistryV1 { >, >, registered_schema_convergence: RegisteredSchemaConvergenceMaintenance, + /// Registered stores whose last attach was refused with a typed reset. + /// The refused store stays unmounted, so every open re-runs admission and + /// a reset store serves on its next open without a daemon restart. + reset_required_stores: StdMutex>, retained_hook_tasks: RetainedHookTasks, session_sync_service: Arc>>, diff --git a/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs b/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs index a847b00e47..eaaa58300a 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/maintenance.rs @@ -11,8 +11,9 @@ use tokio::sync::Semaphore; use tracedecay_contracts::storage::{ SchemaConvergenceFindingV1, SchemaConvergenceStageV1, SchemaConvergenceStateV1, }; +use tracedecay_domain::errors::{StoreResetRequiredV1, TraceDecayError}; use tracedecay_global_db::schema_stages::RegisteredSchemaConvergence; -use tracedecay_store::{StoreRuntimeBindingV1, StoreShardIdV1}; +use tracedecay_store::{StoreRuntimeBindingV1, StoreShardIdV1, StoreShardScopeV1}; use super::retained_hook_tasks::RetainedHookTaskJoin; @@ -544,7 +545,64 @@ impl DaemonSessionRuntimeRegistryV1 { runtime: StoreRuntimeClientLease, _operation: &'static str, ) -> Result { - self.attach_registered_inner(runtime).await + let shard_id = runtime.binding().shard_id.clone(); + let attached = self.attach_registered_inner(runtime).await; + self.record_registered_admission(shard_id, attached.as_ref().err()); + attached + } + + fn record_registered_admission( + &self, + shard_id: StoreShardIdV1, + refusal: Option<&TraceDecayError>, + ) { + let reset_required = refusal.and_then(|error| { + let store = match &shard_id.scope { + StoreShardScopeV1::Profile => "profile authority".to_owned(), + StoreShardScopeV1::ProfileSessions => "profile sessions".to_owned(), + StoreShardScopeV1::ProjectSessions { project_id } => { + format!("project sessions {project_id}") + } + scope => format!("{scope:?}"), + }; + error.store_reset_required(store) + }); + let mut stores = self + .reset_required_stores + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match reset_required { + Some(state) => { + if stores.get(&shard_id) != Some(&state) { + tracing::warn!( + event = "store_reset_required", + store = %state.store, + authority = %state.authority, + remedy = %state.remedy, + "registered store is served in its typed reset-required state" + ); + } + stores.insert(shard_id, state); + } + // Any other failure says nothing about the persisted shape, so + // only a successful attach clears a recorded reset. + None if refusal.is_none() => { + stores.remove(&shard_id); + } + None => {} + } + } + + /// Registered stores currently held in their typed reset-required state, + /// each with the exact command that resets it. + #[must_use] + pub fn reset_required_stores(&self) -> Vec { + self.reset_required_stores + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .values() + .cloned() + .collect() } // Erase the inner state machine before Hotpath wraps it by value. Boxing diff --git a/crates/tracedecay-store-runtime/src/session_registry/mounts.rs b/crates/tracedecay-store-runtime/src/session_registry/mounts.rs index 6876bc07c6..c2fe5e2922 100644 --- a/crates/tracedecay-store-runtime/src/session_registry/mounts.rs +++ b/crates/tracedecay-store-runtime/src/session_registry/mounts.rs @@ -215,6 +215,7 @@ impl DaemonSessionRuntimeRegistryV1 { project_owners: super::ProjectRuntimeOwnerRegistryV1::default(), code_graph_publication_gates: std::sync::Mutex::new(std::collections::BTreeMap::new()), registered_schema_convergence: RegisteredSchemaConvergenceMaintenance::new(), + reset_required_stores: std::sync::Mutex::new(BTreeMap::new()), retained_hook_tasks: RetainedHookTasks::new(), session_sync_service: Arc::new(std::sync::OnceLock::new()), remote_recovery_project_lifecycle: Arc::new(std::sync::OnceLock::new()), diff --git a/crates/tracedecay/src/daemon.rs b/crates/tracedecay/src/daemon.rs index ae626fd985..d34db24eab 100644 --- a/crates/tracedecay/src/daemon.rs +++ b/crates/tracedecay/src/daemon.rs @@ -245,8 +245,8 @@ pub fn error_is_read_deadline(error: &TraceDecayError) -> bool { mod bootstrap; mod bootstrap_route; use bootstrap_route::{ - apply_daemon_initialize_route, attach_initialize_route_metadata, cached_project_node_count, - daemon_bootstrap_response, prewarm_daemon_bootstrap_catalog, + apply_daemon_initialize_route, attach_initialize_route_metadata, attach_reset_required_stores, + cached_project_node_count, daemon_bootstrap_response, prewarm_daemon_bootstrap_catalog, }; mod branch_add; mod branch_admin; diff --git a/crates/tracedecay/src/daemon/bootstrap.rs b/crates/tracedecay/src/daemon/bootstrap.rs index 598a39472b..4b88c7e836 100644 --- a/crates/tracedecay/src/daemon/bootstrap.rs +++ b/crates/tracedecay/src/daemon/bootstrap.rs @@ -170,10 +170,8 @@ async fn run_foreground_loopback( } let lifecycle = DaemonLifecycle::default(); let sync_config = tracedecay_configuration::SyncConfig::default(); - let profile_database = store_administration.registered_profile_database().await?; let maintenance = maintenance::MaintenanceCoordinator::spawn( profile_root.clone(), - profile_database, store_administration.clone(), invocation.code_index_schedulers.clone(), sync_config.retention.clone(), @@ -636,13 +634,8 @@ async fn run_foreground_unix( ); } let sync_config = tracedecay_configuration::SyncConfig::default(); - let profile_database = engine - .store_administration - .registered_profile_database() - .await?; let maintenance = maintenance::MaintenanceCoordinator::spawn( profile_root.clone(), - profile_database, engine.store_administration.clone(), engine.invocation.code_index_schedulers.clone(), sync_config.retention.clone(), @@ -834,9 +827,31 @@ async fn install_profile_worker_plan( .profile_identity()? .profile_id() .clone(); - let database = store_administration + let database = match store_administration .registered_profile_session_database() - .await?; + .await + { + Ok(database) => database, + // A reset-required profile store is served as that typed state, not a + // dead daemon. Its persisted worker selection is unreadable, so the + // plan runs on the selection a reset profile initializes to until the + // operator resets it. + Err(error) if error.store_reset_required("profile sessions").is_some() => { + log_daemon_event( + "profile_worker_plan_reset_required", + &[ + ("selection", "automatic".to_owned()), + ("error", error.to_string()), + ], + ); + invocation.install_worker_selection( + store_administration, + tracedecay_domain::configuration::CodeIndexWorkerSelectionV1::default(), + )?; + return Ok(()); + } + Err(error) => return Err(error), + }; invocation .install_profile_worker_plan(store_administration, database, &profile_id) .await?; diff --git a/crates/tracedecay/src/daemon/bootstrap_route.rs b/crates/tracedecay/src/daemon/bootstrap_route.rs index 3bfa1a663d..d7e0759378 100644 --- a/crates/tracedecay/src/daemon/bootstrap_route.rs +++ b/crates/tracedecay/src/daemon/bootstrap_route.rs @@ -103,6 +103,34 @@ pub(super) fn attach_initialize_route_metadata( result["_meta"]["tracedecayInitializeRoute"] = json!(route); } +/// Names, in an `initialize` result, every registered store the daemon serves +/// in its typed reset-required state, so lifecycle probes that prove the +/// daemon ready also learn the reset the operator owes. A registry that +/// cannot be opened turns the answer into that typed error. +pub(super) async fn attach_reset_required_stores( + response: &mut JsonRpcResponse, + store_administration: &StoreAdministration, +) { + if response.result.is_none() { + return; + } + match store_administration.session_runtime_registry().await { + Ok(registry) => { + if let Some(result) = response.result.as_mut() { + result["_meta"][tracedecay_daemon_protocol::RESET_REQUIRED_STORES_META_KEY] = + json!(registry.reset_required_stores()); + } + } + Err(error) => { + *response = JsonRpcResponse::error( + response.id.clone(), + ErrorCode::InternalError, + error.to_string(), + ); + } + } +} + /// Returns `None` for project-dependent requests, `Some(None)` for handled /// notifications, and `Some(Some(response))` for static MCP bootstrap calls. pub(super) fn daemon_bootstrap_response( diff --git a/crates/tracedecay/src/daemon/connection_serving.rs b/crates/tracedecay/src/daemon/connection_serving.rs index 8920dae9de..8f06a42bc9 100644 --- a/crates/tracedecay/src/daemon/connection_serving.rs +++ b/crates/tracedecay/src/daemon/connection_serving.rs @@ -1461,6 +1461,17 @@ fn serve_broker_socket_client_inner( .id .clone() .map(|id| project_open_error_response(id, &error)); + } else if let Some(response) = response.as_mut() + && matches!( + classify_mcp_method(&request.method), + McpMethod::Initialize + ) + { + Box::pin(attach_reset_required_stores( + response, + &engine.store_administration, + )) + .await; } // Keep catalog-refresh bookkeeping consistent with the regular MCP // server path. Only a warming `tools/list` (no published node count) diff --git a/crates/tracedecay/src/daemon/core_doctor.rs b/crates/tracedecay/src/daemon/core_doctor.rs index 9efbe3d1fd..0a8dc769bd 100644 --- a/crates/tracedecay/src/daemon/core_doctor.rs +++ b/crates/tracedecay/src/daemon/core_doctor.rs @@ -104,6 +104,7 @@ fn core_status_request_id(request: Option<&JsonRpcRequest>) -> Option serde_json::Value { json!({ "project_root": handshake.project_path, @@ -116,6 +117,7 @@ fn project_open_status_value( "status": "unavailable", "findings": [], }, + "reset_required_stores": reset_required_stores, }) } @@ -598,7 +600,14 @@ where && project_open.state != ProjectOpenStatusStateV1::Completed { drop(setup_activity); - let result = doctor_runtime_tool_result(project_open_status_value(handshake, project_open)); + let reset_required_stores = Box::pin(store_administration.session_runtime_registry()) + .await? + .reset_required_stores(); + let result = doctor_runtime_tool_result(project_open_status_value( + handshake, + project_open, + &reset_required_stores, + )); Box::pin(write_json_rpc_response( transport, &JsonRpcResponse::success(id, result), @@ -612,13 +621,15 @@ where let report_ready = if request.doctor_report_requested() { match Box::pin(doctor_report_ready()).await { Ok(ready) => ready, - // Enrollment, warming, and a blocked repository walk are client - // states. Dropping the socket before any frame made Doctor report - // a closed connection and then print store-recovery guidance. + // Enrollment, warming, a blocked repository walk, and a + // reset-required store are client or operator states. Dropping + // the socket before any frame made Doctor report a closed + // connection and then print store-recovery guidance. Err(error) if super::error_is_project_not_enrolled(&error) || super::error_is_project_warming(&error) - || super::error_is_repository_discovery_deferred(&error) => + || super::error_is_repository_discovery_deferred(&error) + || tracedecay_mcp::reset_required_context(&error).is_some() => { drop(setup_activity); Box::pin(write_json_rpc_response( @@ -951,7 +962,23 @@ mod doctor_runtime_route_tests { output: String::new(), idle_before_write: false, }; - let store_administration = StoreAdministration::default(); + std::fs::create_dir_all(&profile).expect("fixture profile root"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&profile, std::fs::Permissions::from_mode(0o700)) + .expect("secure fixture profile root"); + } + let store_administration = StoreAdministration::default().with_profile_identity( + tracedecay_daemon_identity::profile_identity::load_or_create(&profile) + .expect("load fixture profile identity"), + ); + let _database_scope = tracedecay_runtime_core::db::enter_daemon_database_scope( + &profile, + REGISTERED_RUNTIME_NONCE.fetch_add(1, Ordering::Relaxed), + "core-doctor-status-project-open", + ) + .expect("enter daemon database scope"); let first_request = AuthenticatedFirstRequest::new(status_request_line()); let project_open = ProjectOpenStatusV1 { state: ProjectOpenStatusStateV1::Converging, @@ -982,6 +1009,11 @@ mod doctor_runtime_route_tests { .contains(r#""reason":"deferred_repository_discovery""#) ); assert!(transport.output.contains(r#""retry_after_ms":250"#)); + assert!( + transport.output.contains(r#""reset_required_stores":[]"#), + "{}", + transport.output + ); } #[tokio::test] diff --git a/crates/tracedecay/src/daemon/http_application_router.rs b/crates/tracedecay/src/daemon/http_application_router.rs index 96b78136cf..0f6e5b9673 100644 --- a/crates/tracedecay/src/daemon/http_application_router.rs +++ b/crates/tracedecay/src/daemon/http_application_router.rs @@ -4,6 +4,7 @@ //! resolver the registry uses to mount it on demand. use super::*; +use tracedecay_runtime_core::logging::log_daemon_event; use tracedecay_runtime_core::path_safety::canonical_existing_identity; #[hotpath::measure(label = "daemon.http.application.router_build")] @@ -112,7 +113,24 @@ pub(super) async fn install_remote_http_application_router( store_administration: &StoreAdministration, invocation: &DaemonInvocationState, ) -> Result<()> { - let runtime = store_administration.registered_runtime_registry().await?; + let runtime = match store_administration.registered_runtime_registry().await { + Ok(runtime) => runtime, + // The account-active guard reads the profile authority. While that + // store is reset-required no remote request can be admitted, so the + // remote protocol stays unmounted until the operator's reset restarts + // the daemon on a fresh profile. + Err(error) if error.store_reset_required("profile authority").is_some() => { + log_daemon_event( + "remote_protocol_router_unmounted", + &[ + ("reason", "profile_authority_reset_required".to_owned()), + ("error", error.to_string()), + ], + ); + return Ok(()); + } + Err(error) => return Err(error), + }; let credentials = runtime.remote_credential_authority(); let router = tracedecay_daemon_service::build_daemon_remote_protocol_router( Arc::clone(&credentials), diff --git a/crates/tracedecay/src/daemon/maintenance.rs b/crates/tracedecay/src/daemon/maintenance.rs index 7fdf0a2dff..1219c41343 100644 --- a/crates/tracedecay/src/daemon/maintenance.rs +++ b/crates/tracedecay/src/daemon/maintenance.rs @@ -338,7 +338,6 @@ impl MaintenanceCoordinator { #[hotpath::skip] pub(super) async fn spawn( profile_root: PathBuf, - profile_database: tracedecay_global_db::RegisteredGlobalDbLeaseV1, administration: StoreAdministration, code_index_schedulers: tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1, retention: tracedecay_configuration::RetentionConfig, @@ -413,7 +412,6 @@ impl MaintenanceCoordinator { task_owner .run( profile_root, - profile_database, administration, code_index_schedulers, retention, @@ -495,7 +493,6 @@ impl MaintenanceCoordinator { async fn run( &self, profile_root: PathBuf, - profile_database: tracedecay_global_db::RegisteredGlobalDbLeaseV1, administration: StoreAdministration, code_index_schedulers: tracedecay_code_index_runtime::code_index_scheduler::CodeIndexSchedulerRegistryV1, retention: tracedecay_configuration::RetentionConfig, @@ -503,15 +500,41 @@ impl MaintenanceCoordinator { interval: Duration, ) { run_maintenance_loop(&self.cancellation, &self.wake, interval, |continuation| { - self.run_tick( + let (profile_root, administration, code_index_schedulers, retention) = ( &profile_root, - profile_database.as_ref(), &administration, &code_index_schedulers, &retention, - branch_gc, - continuation, - ) + ); + async move { + // Resolved per tick: a reset-required profile authority keeps + // the daemon serving, and the tick after the operator's reset + // mounts the fresh store. + let profile_database = match administration.registered_profile_database().await { + Ok(database) => database, + Err(error) => { + log_daemon_event( + "retention_degraded", + &[ + ("pass", "maintenance_tick".to_owned()), + ("failure", "profile_authority_unavailable".to_owned()), + ("error", error.to_string()), + ], + ); + return MaintenanceTickOutcome::Retry; + } + }; + self.run_tick( + profile_root, + profile_database.as_ref(), + administration, + code_index_schedulers, + retention, + branch_gc, + continuation, + ) + .await + } }) .await; } diff --git a/crates/tracedecay/src/daemon/profile_retained.rs b/crates/tracedecay/src/daemon/profile_retained.rs index 35de1ed140..bfef1233aa 100644 --- a/crates/tracedecay/src/daemon/profile_retained.rs +++ b/crates/tracedecay/src/daemon/profile_retained.rs @@ -97,6 +97,15 @@ pub(super) async fn invoke_profile_retained( /// A profile authority that could not be mounted keeps its reason code and /// retry verdict as a typed unavailable terminal. fn authority_problem(error: &TraceDecayError) -> ApplicationProblem { + if let Some((authority, reason)) = tracedecay_mcp::reset_required_context(error) { + return ApplicationProblem::reset_required(SafeDiagnostic { + code: "application.retained.profile-reset-required".to_owned(), + message: format!( + "The {authority} requires an explicit reset: {reason}. Reset it with `{}`", + tracedecay_mcp::reset_required_command(&authority, None) + ), + }); + } let (code, retryable) = match error { TraceDecayError::ProjectRoute { reason_code, diff --git a/crates/tracedecay/src/daemon/project_composition.rs b/crates/tracedecay/src/daemon/project_composition.rs index 8b4ef95c8b..458e4f5632 100644 --- a/crates/tracedecay/src/daemon/project_composition.rs +++ b/crates/tracedecay/src/daemon/project_composition.rs @@ -1594,7 +1594,18 @@ impl ProjectOpenInputs<'_> { ) .await; full_server.publish_doctor_report(); - let code_index_status = match core.code_index_activation.automatic_admission() { + let code_index_status = self.activate_code_index(core); + self.log_phase( + "full_published", + Some(("code_index", code_index_status.to_owned())), + self.started, + ); + Ok(()) + } + + /// Start this route's code indexing under its automatic admission. + fn activate_code_index(&self, core: &ComposedCoreServer) -> &'static str { + match core.code_index_activation.automatic_admission() { code_index_scheduler::CodeIndexAutomaticAdmissionV1::Admitted => { if core.code_index_activation.activate() { "warming" @@ -1612,13 +1623,7 @@ impl ProjectOpenInputs<'_> { ); "linked_worktree_disabled" } - }; - self.log_phase( - "full_published", - Some(("code_index", code_index_status.to_owned())), - self.started, - ); - Ok(()) + } } /// A failed upgrade either degrades the route to the still-published core @@ -1672,6 +1677,17 @@ impl ProjectOpenInputs<'_> { ) .await; } + // A session store in its typed reset-required state keeps the + // full upgrade refused until the operator resets it, so the + // retained core serves the code index meanwhile. + if tracedecay_mcp::reset_required_context(&error).is_some() { + let code_index_status = self.activate_code_index(core); + self.log_phase( + "full_upgrade_reset_required", + Some(("code_index", code_index_status.to_owned())), + self.started, + ); + } self.log_phase( "full_upgrade_degraded", Some(("error", error.to_string())), diff --git a/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs b/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs index d521c861e4..dd1545388c 100644 --- a/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs +++ b/crates/tracedecay/src/daemon/project_composition/code_index_activation.rs @@ -148,6 +148,17 @@ fn spawn_query_authority_when_generation_ready(inputs: QueryAuthorityWaitInputs) let mut seats = authority_invocation .code_index_schedulers .subscribe_serving_seats(); + // A restored generation seats its graph only for a reader that + // demands the complete generation. The query authority is that + // reader: a route whose full upgrade is refused (a reset-required + // session store) has no advisory owner to make the demand. + tokio::select! { + biased; + () = authority_cancellation.cancelled() => return, + latest = authority_invocation + .code_index_schedulers + .latest_complete_ready_for_scope(&authority_scope) => drop(latest), + } let generation_ready = loop { if authority_invocation .code_index_schedulers diff --git a/crates/tracedecay/src/daemon/project_open_handshake.rs b/crates/tracedecay/src/daemon/project_open_handshake.rs index 75b5585f50..58ce8c578c 100644 --- a/crates/tracedecay/src/daemon/project_open_handshake.rs +++ b/crates/tracedecay/src/daemon/project_open_handshake.rs @@ -205,19 +205,17 @@ fn tool_call_open_refusal_response( if !matches!(classify_mcp_method(&request.method), McpMethod::ToolsCall) { return None; } - let TraceDecayError::ResetRequired { authority, reason } = error else { - return None; - }; + let (authority, reason) = tracedecay_mcp::reset_required_context(error)?; let id = request.id.clone()?; let tool_name = request.params.as_ref()?.get("name")?.as_str()?; let request_id = tracedecay_contracts::request_identity::mcp_connection_request_id(&id, connection_scope)?; - let reset_command = tracedecay_mcp::reset_required_command(authority, None); + let reset_command = tracedecay_mcp::reset_required_command(&authority, None); let envelope = tracedecay_daemon_service::application_surface::mcp_project_open_reset_refusal( tool_name, request_id, - authority, - reason, + &authority, + &reason, &reset_command, )?; let text = serde_json::to_string(&envelope).ok()?; @@ -290,18 +288,20 @@ pub(crate) fn project_open_error_response( })), ) } - TraceDecayError::ResetRequired { authority, reason } => JsonRpcResponse::error_with_data( - id, - ErrorCode::InternalError, - error.to_string(), - Some(json!({ - "kind": "reset_required", - "retryable": false, - "authority": authority, - "reason": reason, - })), - ), - _ => JsonRpcResponse::error(id, ErrorCode::InternalError, error.to_string()), + _ => match tracedecay_mcp::reset_required_context(error) { + Some((authority, reason)) => JsonRpcResponse::error_with_data( + id, + ErrorCode::InternalError, + error.to_string(), + Some(json!({ + "kind": "reset_required", + "retryable": false, + "authority": authority, + "reason": reason, + })), + ), + None => JsonRpcResponse::error(id, ErrorCode::InternalError, error.to_string()), + }, } } diff --git a/crates/tracedecay/src/daemon/project_routing.rs b/crates/tracedecay/src/daemon/project_routing.rs index 191cecdb7f..cd676d3c92 100644 --- a/crates/tracedecay/src/daemon/project_routing.rs +++ b/crates/tracedecay/src/daemon/project_routing.rs @@ -115,8 +115,18 @@ pub(super) async fn bind_authenticated_profile_identity( let scoped_administration = store_administration .clone() .with_profile_identity(profile_identity); - let profile_database = scoped_administration.registered_profile_database().await?; - let global_db_path = authority::canonical_identity_path(profile_database.db_path())?; + let global_db_path = match scoped_administration.registered_profile_database().await { + Ok(profile_database) => authority::canonical_identity_path(profile_database.db_path())?, + // A reset-required profile authority keeps its registered location. + // Binding the connection to it lets every request on it answer the + // typed refusal instead of closing without a frame. + Err(error) if error.store_reset_required("profile authority").is_some() => { + authority::canonical_identity_path( + &profile_root.join(tracedecay_runtime_core::config::GLOBAL_DB_FILENAME), + )? + } + Err(error) => return Err(error), + }; let supplied_global_db_path = authority::canonical_identity_path(&handshake.client_identity.global_db_path)?; if supplied_global_db_path != global_db_path { diff --git a/crates/tracedecay/src/daemon/project_server_lifecycle.rs b/crates/tracedecay/src/daemon/project_server_lifecycle.rs index c503128c46..4ebb13bdf7 100644 --- a/crates/tracedecay/src/daemon/project_server_lifecycle.rs +++ b/crates/tracedecay/src/daemon/project_server_lifecycle.rs @@ -280,6 +280,11 @@ pub(super) async fn ensure_user_profile_host_admission_replay_for_identity( .registered_profile_session_database() .await .map_err(|error| { + // A persisted-shape refusal is terminal until the operator's + // reset; it stays typed instead of reading as a retryable outage. + if tracedecay_mcp::reset_required_context(&error).is_some() { + return error; + } TraceDecayError::project_route( "registered_authority_unavailable", true, diff --git a/crates/tracedecay/src/daemon/projectless.rs b/crates/tracedecay/src/daemon/projectless.rs index f3b89a4e2e..5e11b71803 100644 --- a/crates/tracedecay/src/daemon/projectless.rs +++ b/crates/tracedecay/src/daemon/projectless.rs @@ -149,24 +149,34 @@ async fn projectless_response( ) -> Option { let id = request.id.clone()?; match request.method.as_str() { - "initialize" => Some(match tracedecay_project::version::build_version() { - Ok(version) => JsonRpcResponse::success( - id, - json!({ - "protocolVersion": "2024-11-05", - "capabilities": { - "tools": { - "listChanged": true + "initialize" => { + let mut response = match tracedecay_project::version::build_version() { + Ok(version) => JsonRpcResponse::success( + id, + json!({ + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": { + "listChanged": true + } + }, + "serverInfo": { + "name": "tracedecay", + "version": version } - }, - "serverInfo": { - "name": "tracedecay", - "version": version - } - }), - ), - Err(error) => JsonRpcResponse::error(id, ErrorCode::InternalError, error.to_string()), - }), + }), + ), + Err(error) => { + JsonRpcResponse::error(id, ErrorCode::InternalError, error.to_string()) + } + }; + boxed_projectless_phase(attach_reset_required_stores( + &mut response, + store_administration, + )) + .await; + Some(response) + } "tools/list" => Some(projectless_tools_list_response(id)), "tools/call" => { let started = timings_enabled.then(std::time::Instant::now); @@ -316,6 +326,9 @@ async fn projectless_tools_call_response_with_connection( } if let Err(error) = boxed_projectless_phase(store_administration.ensure_account_active()).await { + if error.store_reset_required("profile authority").is_some() { + return tool_error_response(id, tool_name, &error); + } return JsonRpcResponse::error(id, ErrorCode::InternalError, error.to_string()); } // Keep unrelated tool families out of one generated poll frame. Some handlers diff --git a/crates/tracedecay/src/daemon/remote_deletion.rs b/crates/tracedecay/src/daemon/remote_deletion.rs index e9da16e342..b2c68e05d9 100644 --- a/crates/tracedecay/src/daemon/remote_deletion.rs +++ b/crates/tracedecay/src/daemon/remote_deletion.rs @@ -7,6 +7,7 @@ use axum::http::{HeaderMap, StatusCode}; use axum::response::{IntoResponse, Response}; use serde::{Deserialize, Serialize}; use tracedecay_domain::ProjectId; +use tracedecay_runtime_core::logging::log_daemon_event; use super::{StoreAdministration, http_application::DaemonHttpApplicationRegistry}; pub(super) use tracedecay_global_db::{RemoteDeletionFailureCode, RemoteDeletionPhase}; @@ -29,11 +30,29 @@ pub(super) enum RemoteDeletionBootMode { pub(super) async fn resume_remote_account_deletion_for_boot( owners: &RemoteDeletionRuntimeOwners, ) -> tracedecay_domain::errors::Result { - let Some(tombstone) = owners + let tombstone = match owners .administration .remote_account_deletion_tombstone() - .await? - else { + .await + { + Ok(tombstone) => tombstone, + // A reset-required profile authority holds no readable tombstone; the + // daemon boots to serve that typed state, and every profile read, + // including the account-active guard, meets the same refusal until + // the operator's reset deletes the store. + Err(error) if error.store_reset_required("profile authority").is_some() => { + log_daemon_event( + "remote_account_deletion_resume", + &[ + ("outcome", "profile_authority_reset_required".to_owned()), + ("error", error.to_string()), + ], + ); + None + } + Err(error) => return Err(error), + }; + let Some(tombstone) = tombstone else { return Ok(RemoteDeletionBootMode::Ordinary); }; match owners diff --git a/crates/tracedecay/src/doctor.rs b/crates/tracedecay/src/doctor.rs index ac40ac71fe..a0899074bf 100644 --- a/crates/tracedecay/src/doctor.rs +++ b/crates/tracedecay/src/doctor.rs @@ -92,12 +92,21 @@ pub struct AdmittedDoctorNetworkProbes { pub fetch_latest_version: fn() -> Option, } +/// What a doctor run that found no issue concluded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DoctorCompletion { + Healthy, + /// The daemon serves a store in its typed reset-required state; the + /// operator owes the reset Doctor named. + PendingOperatorAction, +} + /// Runs a comprehensive health check of the tracedecay installation. #[hotpath::measure(label = "doctor.run", future = true)] pub async fn run_doctor( profile: &tracedecay_runtime_core::config::ProfileRoot, network: AdmittedDoctorNetworkProbes, -) -> tracedecay_domain::errors::Result<()> { +) -> tracedecay_domain::errors::Result { let _lifecycle_lease = match tracedecay_runtime_core::lifecycle_lease::acquire_shared_or_inherited( profile.data_dir(), @@ -116,6 +125,7 @@ pub async fn run_doctor( check_binary(&mut dc, build_version); check_daemon_service(&mut dc, profile, build_version); + let mut pending_reset = check_reset_required_stores(&mut dc, profile, build_version); eprintln!("\n\x1b[1mCurrent project\x1b[0m"); let project_path = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); @@ -146,7 +156,17 @@ pub async fn run_doctor( } } Err(error) => { - classify_daemon_status_error(&mut dc, profile.data_dir(), &project_path, error) + if let Some((authority, reason)) = tracedecay_mcp::reset_required_context(error) { + pending_reset = true; + dc.warn(&format!( + "Current project is not served: {authority} requires reset ({reason}). \ + Pending operator action: run `{}`", + tracedecay_mcp::reset_required_command(&authority, Some(&project_path)) + )); + DatabaseHealth::unknown("reset_required") + } else { + classify_daemon_status_error(&mut dc, profile.data_dir(), &project_path, error) + } } }; check_watcher(&mut dc, profile); @@ -191,7 +211,37 @@ pub async fn run_doctor( check_network(&mut dc, upload_enabled.as_ref(), network); print_summary(&dc); - doctor_result(&dc, &storage_health) + doctor_result(&dc, &storage_health, pending_reset) +} + +/// Lists every registered store the daemon serves in its typed +/// reset-required state, each with the exact reset command. Returns whether +/// any reset is pending. +fn check_reset_required_stores( + dc: &mut DoctorCounters, + profile: &tracedecay_runtime_core::config::ProfileRoot, + build_version: &str, +) -> bool { + if !tracedecay_daemon_control::daemon_reachable(profile) { + return false; + } + match tracedecay_daemon_control::daemon_reset_required_stores(profile, build_version) { + Ok(stores) => { + for store in &stores { + dc.warn(&format!( + "Store {} requires reset ({}). Pending operator action: run `{}`", + store.store, store.reason, store.remedy + )); + } + !stores.is_empty() + } + Err(error) => { + dc.warn(&format!( + "Daemon reset-required stores could not be read: {error}" + )); + false + } + } } fn render_project_open_status( @@ -399,11 +449,13 @@ fn database_health_from_storage_runtime_findings<'a>( /// /// Only an observed storage *failure* is fatal. `DatabaseHealth::Unknown`, a /// diagnostic that could not run, is reported to the user but never laundered -/// into a healthy verdict nor turned into a hard failure. +/// into a healthy verdict nor turned into a hard failure. With no issue, a +/// pending reset is the operator's action, not health. fn doctor_result( dc: &DoctorCounters, storage_health: &DatabaseHealth, -) -> tracedecay_domain::errors::Result<()> { + pending_reset: bool, +) -> tracedecay_domain::errors::Result { match storage_health { DatabaseHealth::Failed { reason } => { Err(tracedecay_domain::errors::TraceDecayError::Config { @@ -415,7 +467,10 @@ fn doctor_result( message: format!("doctor found {} issue(s)", dc.issues), }) } - DatabaseHealth::Healthy | DatabaseHealth::Unknown { .. } => Ok(()), + DatabaseHealth::Healthy | DatabaseHealth::Unknown { .. } if pending_reset => { + Ok(DoctorCompletion::PendingOperatorAction) + } + DatabaseHealth::Healthy | DatabaseHealth::Unknown { .. } => Ok(DoctorCompletion::Healthy), } } diff --git a/crates/tracedecay/src/doctor/tests.rs b/crates/tracedecay/src/doctor/tests.rs index 36b7375988..58556956f0 100644 --- a/crates/tracedecay/src/doctor/tests.rs +++ b/crates/tracedecay/src/doctor/tests.rs @@ -509,6 +509,7 @@ fn unavailable_canonical_report_is_an_issue_that_fails_the_doctor_exit() { let error = super::doctor_result( &counters, &DatabaseHealth::unknown("canonical_doctor_report_unavailable"), + true, ) .unwrap_err(); assert_eq!(error.to_string(), "config error: doctor found 1 issue(s)"); @@ -517,13 +518,28 @@ fn unavailable_canonical_report_is_an_issue_that_fails_the_doctor_exit() { #[test] fn doctor_result_treats_unavailable_canonical_report_as_unknown() { let counters = DoctorCounters::new(); - super::doctor_result( - &counters, - &DatabaseHealth::Unknown { - reason: "canonical_doctor_report_unavailable".to_string(), - }, - ) - .unwrap(); + assert_eq!( + super::doctor_result( + &counters, + &DatabaseHealth::Unknown { + reason: "canonical_doctor_report_unavailable".to_string(), + }, + false, + ) + .unwrap(), + super::DoctorCompletion::Healthy + ); +} + +/// A store the daemon serves reset-required is the operator's pending +/// action: with no issue, Doctor completes pending rather than healthy. +#[test] +fn doctor_result_reports_a_pending_reset_without_issues_as_pending() { + let counters = DoctorCounters::new(); + assert_eq!( + super::doctor_result(&counters, &DatabaseHealth::unknown("reset_required"), true).unwrap(), + super::DoctorCompletion::PendingOperatorAction + ); } #[test] diff --git a/crates/tracedecay/tests/mcp_suite/status_behavior_test.rs b/crates/tracedecay/tests/mcp_suite/status_behavior_test.rs index 5b4dee86a5..8dfc584f6f 100644 --- a/crates/tracedecay/tests/mcp_suite/status_behavior_test.rs +++ b/crates/tracedecay/tests/mcp_suite/status_behavior_test.rs @@ -292,6 +292,7 @@ async fn tracedecay_status_reports_the_sealed_branch_and_keeps_diagnostics_opt_i **memory.status:** nominal\n\ {owner_bullets}\ **project_root:** {root}\n\ + **reset_required_stores:** 0 item(s)\n\ **retrieval_serving.status:** serving\n\ **schema_convergence.status:** completed\n\ **server:** {{14 field(s)}}\n\ diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs index acc99fbfd9..3fce18ba1d 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance.rs @@ -36,6 +36,8 @@ //! final shape this binary creates, returning `ResetRequired` with a //! `Reset`-only legal action for every subsequent open of that store. use crate::common; +/// A reset-required registered store served as a typed state. +mod reset_required_serving; /// The HTTP, MCP-host, and Rust SDK legs of this same journey. mod transport_boundaries; diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/reset_required_serving.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/reset_required_serving.rs new file mode 100644 index 0000000000..e555b3f65d --- /dev/null +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/reset_required_serving.rs @@ -0,0 +1,197 @@ +//! A registered store whose persisted shape this binary does not open is +//! served as a typed reset-required state instead of a daemon that refuses to +//! start. +//! +//! The profile session store is stamped with the Git correlation schema a +//! released binary wrote (version 5, before per-session Git evidence rows) and +//! a physically spawned `tracedecay daemon run` is started over it. The daemon +//! must reach readiness, `tracedecay_status` must name the refused store with +//! its exact reset command, a profile session read must return the typed +//! `reset_required` problem, and a code-index read on the same project must +//! still answer. After the named reset the same read serves. + +use std::path::Path; +use std::process::Stdio; +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; + +use crate::common::{ + canonical_existing_path, spawn_tracedecay_daemon_with, tracedecay_command_with_home, +}; + +/// Bound on waiting for the first sealed code generation of a two-line fixture. +const CODE_INDEX_READY_TIMEOUT: Duration = Duration::from_secs(120); + +/// The first value under `key` anywhere in a tool payload, including JSON +/// rendered into MCP text blocks. +fn find_key(value: &Value, key: &str) -> Option { + match value { + Value::Object(map) => map + .get(key) + .cloned() + .or_else(|| map.values().find_map(|child| find_key(child, key))), + Value::Array(items) => items.iter().find_map(|child| find_key(child, key)), + Value::String(text) => serde_json::from_str::(text) + .ok() + .and_then(|parsed| find_key(&parsed, key)), + _ => None, + } +} + +fn names_symbol(value: &Value, name: &str) -> bool { + match value { + Value::Object(map) => { + map.get("name").and_then(Value::as_str) == Some(name) + || map.values().any(|child| names_symbol(child, name)) + } + Value::Array(items) => items.iter().any(|child| names_symbol(child, name)), + Value::String(text) => { + serde_json::from_str::(text).is_ok_and(|parsed| names_symbol(&parsed, name)) + } + _ => false, + } +} + +fn wait_for_code_index_hit(home: &Path, project: &Path, symbol: &str) { + let started = Instant::now(); + loop { + let result = super::tool_call( + home, + project, + "tracedecay_search", + &json!({ "query": symbol, "format": "json" }), + ); + if names_symbol(&result, symbol) { + return; + } + assert!( + started.elapsed() < CODE_INDEX_READY_TIMEOUT, + "code index never answered `{symbol}` within {CODE_INDEX_READY_TIMEOUT:?}: {result}" + ); + std::thread::sleep(Duration::from_millis(500)); + } +} + +/// Stamps the profile session store with the Git correlation schema version +/// a released binary recorded, leaving every other byte of its shape as this +/// binary wrote it. +fn stamp_profile_sessions_git_correlation_version(home: &Path, version: i64) { + let db_path = home.join(".tracedecay/user-sessions.db"); + assert!( + db_path.is_file(), + "the daemon should have created the profile session store at {}", + db_path.display() + ); + let stamped = rusqlite::Connection::open(&db_path) + .expect("open the profile session store") + .execute( + "UPDATE session_schema_migrations SET version = ?1 WHERE name = 'git_correlation'", + [version], + ) + .expect("stamp the released git correlation schema version"); + assert_eq!( + stamped, 1, + "the store records exactly one git correlation schema row" + ); +} + +fn profile_session_read(home: &Path, project: &Path) -> Value { + super::tool_call( + home, + project, + "tracedecay_lcm_status", + &json!({ "storage_scope": "user", "format": "json" }), + ) +} + +fn status_reset_required_stores(home: &Path, project: &Path) -> Value { + let status = super::tool_call( + home, + project, + "tracedecay_status", + &json!({ "format": "json" }), + ); + find_key(&status, "reset_required_stores") + .unwrap_or_else(|| panic!("tracedecay_status omitted reset_required_stores: {status}")) +} + +#[test] +fn reset_required_profile_session_store_is_served_typed_until_its_named_reset() { + let home = tempfile::TempDir::new().expect("isolated home"); + let home_path = canonical_existing_path(home.path()); + let project = tempfile::TempDir::new().expect("project"); + let project_path = canonical_existing_path(project.path()); + + let mut daemon = spawn_tracedecay_daemon_with(&home_path, |_| {}); + super::initialize_project(&home_path, &project_path, "reset-required-serving"); + wait_for_code_index_hit(&home_path, &project_path, "probe"); + daemon + .kill_and_wait() + .expect("stop the daemon that wrote the profile"); + + stamp_profile_sessions_git_correlation_version(&home_path, 5); + let mut daemon = spawn_tracedecay_daemon_with(&home_path, |_| {}); + assert!( + daemon + .try_wait() + .expect("inspect the restarted daemon") + .is_none(), + "the daemon must keep serving over a reset-required profile session store" + ); + + assert_eq!( + status_reset_required_stores(&home_path, &project_path), + json!([{ + "store": "profile sessions", + "authority": "git correlation", + "found_version": 5, + "required_version": 6, + "reason": "git correlation profile schema 5 is incompatible with required schema 6; \ + reset the profile", + "remedy": "tracedecay wipe --all --yes", + }]) + ); + super::assert_reset_required( + &super::cli_problem_envelope( + &profile_session_read(&home_path, &project_path), + "profile session read over a reset-required store", + ), + "profile session read over a reset-required store", + ); + wait_for_code_index_hit(&home_path, &project_path, "probe"); + + // `tracedecay wipe --all --yes` takes the profile offline by stopping the + // managed service and restarts it afterwards; this unmanaged daemon is + // stopped and started around the same command. + daemon + .kill_and_wait() + .expect("stop the daemon for the profile reset"); + let wipe = tracedecay_command_with_home(&home_path) + .args(["wipe", "--all", "--yes"]) + .current_dir(&project_path) + .stdin(Stdio::null()) + .output() + .expect("run the named reset"); + assert!( + wipe.status.success(), + "tracedecay wipe --all --yes failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&wipe.stdout), + String::from_utf8_lossy(&wipe.stderr) + ); + let mut daemon = spawn_tracedecay_daemon_with(&home_path, |_| {}); + + let served = profile_session_read(&home_path, &project_path); + assert!( + find_key(&served, "problem").is_none_or(|problem| problem.is_null()), + "the profile session read must serve after the named reset: {served}" + ); + super::initialize_project(&home_path, &project_path, "reset-required-serving"); + assert_eq!( + status_reset_required_stores(&home_path, &project_path), + json!([]), + "the reset store serves again" + ); + + let _ = daemon.kill_and_wait(); +}