Isolate provider fixtures and serialize Docker companion preparation

crates/freshell-supervisor/src/service.rs

@@ -1047,9 +1047,13 @@ impl Supervisor {
                     ),
                 })
                 .await
-                .map_err(map_backend)
                 .map_err(|error| {
-                    self.activation_failure(&prepared, state, "create_stopped_runtime", error)
+                    self.activation_failure_from_backend(
+                        &prepared,
+                        state,
+                        "create_stopped_runtime",
+                        error,
+                    )
                 })?;
             append_event(
                 &self.config.lifecycle_log,
@@ -1333,10 +1337,36 @@ impl Supervisor {
         stage: &'static str,
         error: RuntimeError,
     ) -> RuntimeError {
+        self.record_activation_failure(prepared, state, stage, error, None)
+    }
+
+    fn activation_failure_from_backend(
+        &self,
+        prepared: &PreparedLaunch,
+        state: LaunchState,
+        stage: &'static str,
+        error: BackendError,
+    ) -> RuntimeError {
+        let diagnostic = backend_failure_diagnostic(&error);
+        self.record_activation_failure(prepared, state, stage, map_backend(error), Some(diagnostic))
+    }
+
+    fn record_activation_failure(
+        &self,
+        prepared: &PreparedLaunch,
+        state: LaunchState,
+        stage: &'static str,
+        error: RuntimeError,
+        backend_failure: Option<serde_json::Value>,
+    ) -> RuntimeError {
+        let mut data = activation_failure_data(prepared, state, stage, &error);
+        if let Some(backend_failure) = backend_failure {
+            data["backendFailure"] = backend_failure;
+        }
         append_event(
             &self.config.lifecycle_log,
             "supervisor.activation_failed",
-            activation_failure_data(prepared, state, stage, &error),
+            data,
         );
         error
     }
@@ -4092,6 +4122,7 @@ mod release_scope_tests {
         classify_fresh_agent_operation_identity, lifecycle_secrets, release_qualified_workload,
         FreshAgentNativeIdentityUpdate,
     };
+    use crate::backend::BackendError;
     use crate::registry::PreparedLaunch;
     use freshell_runtime_protocol::{
         FreshAgentForkResult, IncarnationId, LaunchNonce, LaunchState, RuntimeError,
@@ -4268,6 +4299,66 @@ mod release_scope_tests {
         assert!(event["data"].get("errorMessage").is_none());
         assert!(!contents.contains("prepare provider bootstrap failed"));
     }
+
+    #[test]
+    fn backend_create_failure_records_bounded_redacted_backend_cause() {
+        let dir = tempfile::tempdir().unwrap();
+        let log = dir.path().join("backend-create-failure.jsonl");
+        let process_secret = "supervisor-control-secret";
+        let host_binary_path = dir.path().join("synthetic-host");
+        std::fs::write(&host_binary_path, "synthetic-host-artifact").unwrap();
+        let supervisor = super::Supervisor::new(
+            crate::registry::Registry::open(dir.path().join("registry"), None).unwrap(),
+            super::default_backend(dir.path().join("absent-docker.sock"), log.clone()),
+            super::SupervisorConfig {
+                runtime_root: dir.path().join("runtime"),
+                control_socket_path: dir.path().join("control.sock"),
+                host_binary_path,
+                image_ref: format!("sha256:{}", "a".repeat(64)),
+                test_run_id: "backend-failure-diagnostic".into(),
+                control_secret: process_secret.into(),
+                lifecycle_log: log.clone(),
+                admission: crate::admission::AdmissionPolicy::default(),
+            },
+        )
+        .unwrap();
+        let prepared = PreparedLaunch {
+            soul_id: SoulId::new(),
+            incarnation_id: IncarnationId::new(),
+            launch_nonce: LaunchNonce::new(),
+            state: LaunchState::Prepared,
+            intent_revision: 3,
+            existing_request: false,
+        };
+        let provider_secret = "sk-abcdefghijklmnopqrstuvwx";
+        let backend_error = BackendError::DockerHttp {
+            status: 400,
+            body: format!(
+                r#"{{"message":"invalid mount config {process_secret}","token":"{provider_secret}","detail":"{}"}}"#,
+                "x".repeat(700),
+            ),
+        };
+        let error = supervisor.activation_failure_from_backend(
+            &prepared,
+            LaunchState::Prepared,
+            "create_stopped_runtime",
+            backend_error,
+        );
+        assert_eq!(error.code, RuntimeErrorCode::BackendUnavailable);
+
+        let contents = std::fs::read_to_string(&log).unwrap();
+        assert!(!contents.contains(process_secret));
+        assert!(!contents.contains(provider_secret));
+        let event: serde_json::Value = serde_json::from_str(contents.trim()).unwrap();
+        assert_eq!(event["data"]["stage"], "create_stopped_runtime");
+        assert_eq!(event["data"]["backendFailure"]["kind"], "docker_http");
+        assert_eq!(event["data"]["backendFailure"]["httpStatus"], 400);
+        let detail = event["data"]["backendFailure"]["detail"].as_str().unwrap();
+        assert!(detail.contains("invalid mount config"));
+        assert!(detail.contains("***REDACTED***"));
+        assert!(detail.chars().count() <= 512);
+        assert_eq!(event["data"]["backendFailure"]["detailTruncated"], true);
+    }
 }
 
 fn semantic_launch_digest(
@@ -4344,6 +4435,36 @@ fn map_backend(error: BackendError) -> RuntimeError {
     RuntimeError::new(code, error.to_string())
 }
 
+fn backend_failure_diagnostic(error: &BackendError) -> serde_json::Value {
+    let (kind, http_status, detail) = match error {
+        BackendError::Unavailable(detail) => ("unavailable", None, detail.as_str()),
+        BackendError::OwnershipMismatch(detail) => ("ownership_mismatch", None, detail.as_str()),
+        BackendError::DockerHttp { status, body } => ("docker_http", Some(*status), body.as_str()),
+        BackendError::Malformed(detail) => ("malformed", None, detail.as_str()),
+        BackendError::InvalidConfig(detail) => ("invalid_config", None, detail.as_str()),
+    };
+    const MAX_DETAIL_CHARS: usize = 512;
+    let redacted = freshell_runtime_observability::scrub(detail, "");
+    let detail_truncated = redacted.chars().count() > MAX_DETAIL_CHARS;
+    let detail = redacted
+        .chars()
+        .map(|character| {
+            if character.is_control() {
+                ' '
+            } else {
+                character
+            }
+        })
+        .take(MAX_DETAIL_CHARS)
+        .collect::<String>();
+    serde_json::json!({
+        "kind": kind,
+        "httpStatus": http_status,
+        "detail": detail,
+        "detailTruncated": detail_truncated,
+    })
+}
+
 fn io_runtime(error: std::io::Error) -> RuntimeError {
     RuntimeError::new(RuntimeErrorCode::RegistryFailure, error.to_string())
 }

scripts/run-standard-tests.ts

@@ -147,18 +147,16 @@ export function resolveCloudCompanionScope(forwardedArgs: string[], localOnlyDir
 
 export function buildCloudCompanionRun({
   mode,
-  clientWorkers,
   scope,
 }: {
   mode: StandardTestMode
-  clientWorkers: string
   scope: string[]
 }): StandardTestRun {
   return {
     name: 'client-local-only',
     runner: 'vitest',
     configPath: DEFAULT_VITEST_CONFIG,
-    maxWorkers: clientWorkers,
+    maxWorkers: '1',
     priority: mode === 'aggressive' ? 'normal' : 'background',
     pathFilters: scope,
   }
@@ -464,10 +462,8 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
       } else if (companionScope.length === 0) {
         log('info', 'Local companion skipped: no requested selectors intersect the cloud-excluded tests', { localOnlyDirs })
       } else {
-        const workers = resolveDesktopWorkerPlan(availableParallelism())
         companionRun = buildCloudCompanionRun({
           mode: resolvedMode,
-          clientWorkers: workers.clientWorkers,
           scope: companionScope,
         })
         log('info', 'Starting local companion for cloud-excluded tests (runs in parallel with the cloud shards)', {

scripts/testing/runtime-sandbox.ts

@@ -13,6 +13,70 @@ export const PHASE1_RUNTIME_IMAGE_TAG = 'ubuntu:24.04'
 export const PHASE2_RUNTIME_IMAGE_TAG = 'freshell-managed-runtime:phase2'
 export const CONTROL_PROTOCOL_VERSION = 1
 
+type OwnedRuntimeImage = { tag: string; runId: string; candidateSha: string; digest?: string }
+
+const RUNTIME_IMAGE_LABELS = {
+  runId: 'org.freshell.runtime-test-run-id',
+  candidate: 'org.freshell.runtime-test-candidate',
+} as const
+
+function runOwnedRuntimeImageTag(runId: string, candidateSha: string): string {
+  const runToken = createHash('sha256').update(runId).digest('hex')
+  const repository = PHASE2_RUNTIME_IMAGE_TAG.slice(0, PHASE2_RUNTIME_IMAGE_TAG.indexOf(':'))
+  return `${repository}:phase2-${candidateSha.slice(0, 12)}-${runToken}`
+}
+
+function assertRunOwnedRuntimeImage(observed: unknown, context: OwnedRuntimeImage): void {
+  const images = Array.isArray(observed) ? observed : []
+  const image = images.length === 1 && images[0] && typeof images[0] === 'object'
+    ? images[0] as { Id?: unknown; RepoTags?: unknown; Config?: { Labels?: unknown } }
+    : undefined
+  const labels = image?.Config?.Labels
+  const repoTags = image?.RepoTags
+  if (!image || typeof image.Id !== 'string' || (context.digest && image.Id !== context.digest)
+    || !Array.isArray(repoTags) || !repoTags.includes(context.tag)
+    || !labels || typeof labels !== 'object'
+    || (labels as Record<string, unknown>)[RUNTIME_IMAGE_LABELS.runId] !== context.runId
+    || (labels as Record<string, unknown>)[RUNTIME_IMAGE_LABELS.candidate] !== context.candidateSha) {
+    throw new Error('runtime image is not the exact owned artifact')
+  }
+}
+
+function buildRunOwnedRuntimeImage(
+  repoRoot: string, context: OwnedRuntimeImage,
+): string {
+  execFileSync('docker', [
+    'build',
+    '--pull=false',
+    '--provenance=false',
+    '--sbom=false',
+    '--label', `${RUNTIME_IMAGE_LABELS.runId}=${context.runId}`,
+    '--label', `${RUNTIME_IMAGE_LABELS.candidate}=${context.candidateSha}`,
+    '-f',
+    'docker/runtime/Dockerfile',
+    '-t',
+    context.tag,
+    '.',
+  ], { cwd: repoRoot, stdio: 'inherit' })
+  const digest = String(execFileSync('docker', ['image', 'inspect', context.tag, '--format', '{{.Id}}'], { encoding: 'utf8' })).trim()
+  if (!/^sha256:[a-f0-9]{64}$/.test(digest)) throw new Error('run-owned runtime image has no pinned digest')
+  const observed: unknown = JSON.parse(String(execFileSync('docker', ['image', 'inspect', context.tag], { encoding: 'utf8' })))
+  assertRunOwnedRuntimeImage(observed, { ...context, digest })
+  const addressed = String(execFileSync('docker', ['image', 'inspect', digest, '--format', '{{.Id}}'], { encoding: 'utf8' })).trim()
+  if (addressed !== digest) throw new Error('run-owned runtime image digest is not locally addressable')
+  return digest
+}
+
+function removeRunOwnedRuntimeImage(context: OwnedRuntimeImage): void {
+  const imageTags = () => String(execFileSync('docker', ['image', 'ls', '--format', '{{.Repository}}:{{.Tag}}'], { encoding: 'utf8' }))
+    .trim().split('\n').filter(Boolean)
+  if (!imageTags().includes(context.tag)) return
+  const observed: unknown = JSON.parse(String(execFileSync('docker', ['image', 'inspect', context.tag], { encoding: 'utf8' })))
+  assertRunOwnedRuntimeImage(observed, context)
+  execFileSync('docker', ['image', 'rm', context.tag], { encoding: 'utf8' })
+  if (imageTags().includes(context.tag)) throw new Error('exact owned runtime image tag remains after cleanup')
+}
+
 export type RuntimeLimits = {
   cpuMilli: number
   memoryBytes: number
@@ -161,6 +225,7 @@ export class RuntimeHarness {
   private readonly trackedForeignContainers = new Set<string>()
   private readonly trackedVolumes = new Set<string>()
   private readonly assertions: AssertionRecord[] = []
+  private ownedRuntimeImage: OwnedRuntimeImage | undefined
   private brokerStarted = false
   private readonly fixtureInputs: RuntimeFixtureInputs | undefined
 
@@ -335,6 +400,22 @@ export class RuntimeHarness {
       try { await this.broker.close() } catch (error) { errors.push(`broker close: ${String(error)}`) }
       this.brokerStarted = false
     }
+    let runtimeImageCleanup: { tag: string; digest?: string; removed?: boolean } | undefined
+    if (this.ownedRuntimeImage) {
+      runtimeImageCleanup = { tag: this.ownedRuntimeImage.tag, digest: this.ownedRuntimeImage.digest }
+      try {
+        removeRunOwnedRuntimeImage(this.ownedRuntimeImage)
+        runtimeImageCleanup.removed = true
+        this.recordLifecycle('runtime.owned_image_removed', runtimeImageCleanup)
+      } catch (error) {
+        runtimeImageCleanup.removed = false
+        errors.push(`runtime image ${this.ownedRuntimeImage.tag}: ${String(error)}`)
+        this.recordLifecycle('runtime.owned_image_cleanup_failed', {
+          ...runtimeImageCleanup,
+          error: String(error),
+        })
+      }
+    }
     // Build copies are unique to this run, so retries/concurrent gates never
     // overwrite an executable that another still-running container/process
     // has mapped. Remove them only after every receipt-owned runtime stopped.
@@ -349,6 +430,7 @@ export class RuntimeHarness {
         ...this.broker?.receiptIds?.() ?? [],
       ])],
       volumes: [...new Set([...this.trackedVolumes, ...providerVolumes, ...indexedVolumes, ...candidateVolumes])],
+      runtimeImage: runtimeImageCleanup,
       unsafeBrokerAttempts: this.broker?.unsafeAttempts?.() ?? [],
       completedAt: new Date().toISOString(),
     }
@@ -1112,35 +1194,31 @@ export class RuntimeHarness {
   }
 
   private ensureRuntimeImage(): void {
-    const tag = this.phase === 1 ? PHASE1_RUNTIME_IMAGE_TAG : PHASE2_RUNTIME_IMAGE_TAG
-    let inspect = spawnSync('docker', ['image', 'inspect', tag, '--format', '{{.Id}}'], { encoding: 'utf8' })
     if (this.phase === 1) {
+      const tag = PHASE1_RUNTIME_IMAGE_TAG
+      let inspect = spawnSync('docker', ['image', 'inspect', tag, '--format', '{{.Id}}'], { encoding: 'utf8' })
       if (inspect.status !== 0) {
         docker(['pull', tag])
         inspect = spawnSync('docker', ['image', 'inspect', tag, '--format', '{{.Id}}'], { encoding: 'utf8' })
       }
-    } else {
-      // Always invoke the Phase 2 build so Docker validates the current
-      // Dockerfile/provider pins. Layer caching keeps unchanged rebuilds cheap.
-      // BuildKit provenance captures the whole context digest, including files
-      // the image never copies, and therefore changes the manifest-list ID for
-      // an otherwise identical workload image. Disable attestations here so
-      // the exact image identity is a reproducible digest of the image itself.
-      execFileSync('docker', [
-        'build',
-        '--pull=false',
-        '--provenance=false',
-        '--sbom=false',
-        '-f',
-        'docker/runtime/Dockerfile',
-        '-t',
-        tag,
-        '.',
-      ], { cwd: this.repoRoot, stdio: 'inherit' })
-      inspect = spawnSync('docker', ['image', 'inspect', tag, '--format', '{{.Id}}'], { encoding: 'utf8' })
+      this.imageRef = inspect.stdout.trim()
+      if (!/^sha256:[0-9a-f]{64}$/.test(this.imageRef)) throw new Error(`runtime image is not pinned: ${this.imageRef}`)
+      return
+    }
+
+    // Always build the pinned provider workload image. Give each harness its
+    // own tag so another worktree cannot retag or remove our captured digest.
+    // BuildKit attestations include irrelevant build-context files in the
+    // image identity, so disable them to keep the workload digest reproducible.
+    const ownedImage: OwnedRuntimeImage = {
+      tag: runOwnedRuntimeImageTag(this.runId, this.candidateSha),
+      runId: this.runId,
+      candidateSha: this.candidateSha,
     }
-    this.imageRef = inspect.stdout.trim()
-    if (!/^sha256:[0-9a-f]{64}$/.test(this.imageRef)) throw new Error(`runtime image is not pinned: ${this.imageRef}`)
+    this.ownedRuntimeImage = ownedImage
+    ownedImage.digest = buildRunOwnedRuntimeImage(this.repoRoot, ownedImage)
+    this.imageRef = ownedImage.digest
+    this.recordLifecycle('runtime.owned_image_built', { tag: ownedImage.tag, digest: ownedImage.digest })
   }
 
   private phase2BootstrapFiles(): string[] {

test/integration/server/fresh-agent-parity.test.ts

@@ -92,6 +92,9 @@ describe('hosted fresh-agent provider inputs', () => {
 
   beforeAll(async () => {
     homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fresh-provider-parity-home-'))
+    workspace = path.join(homeDir, 'workspace')
+    fs.mkdirSync(workspace)
+    execFileSync('git', ['init', '--quiet', workspace])
     onecli = fakeOnecliGrants(providers.map(row => row.provider))
     restoreOnecliEnvironment = exposeGrantsToHarness(onecli.serverEnv)
     const xdgConfigHome = path.join(homeDir, 'custom-xdg')
@@ -106,16 +109,21 @@ describe('hosted fresh-agent provider inputs', () => {
       providerSettings: {},
       freshAgentModes: providers.map(row => row.sessionType),
       fixtureFreshAgentModes: providers.map(row => row.sessionType),
-    }, homeDir)
+    }, homeDir, {
+      workspaceRoots: [workspace],
+      providerInputRoots: [],
+      bootstrapFiles: Object.values(onecli.grants),
+      defaultCwd: workspace,
+    })
     await rig.start()
     const health = await rig.runtime.adminOk(rig.supervisor, { method: 'health' })
     expect(health.kind).toBe('health')
     providerSource = await readProviderParitySource(rig.runtime, health.data.installation_id)
     // Select a synthetic project, so providers never inherit this checkout's
     // own skills, installed dependencies, or linked-worktree Git topology.
-    workspace = path.join(rig.runtime.testRoot, 'workspace')
-    fs.mkdirSync(workspace)
-    execFileSync('git', ['init', '--quiet', workspace])
+    // Keep it under the owned synthetic HOME: installed-package ancestor
+    // discovery then stops at this boundary instead of capturing a shared
+    // /tmp/node_modules directory outside the broker's selected roots.
     // Every create selects this plugin directory relative to its cwd.
     fs.mkdirSync(path.join(workspace, 'fixture-plugin/.claude-plugin'), { recursive: true })
     fs.writeFileSync(path.join(workspace, 'fixture-plugin/.claude-plugin/plugin.json'), JSON.stringify({ name: 'fixture-plugin' }))