diff --git a/.specify/feature.json b/.specify/feature.json new file mode 100644 index 00000000..b3e8ead8 --- /dev/null +++ b/.specify/feature.json @@ -0,0 +1,3 @@ +{ + "feature_directory": "specs/001-stabilize-agent-startup" +} diff --git a/admin/slices/agent/agent/components/agent/chat/Tab.vue b/admin/slices/agent/agent/components/agent/chat/Tab.vue index ba57b109..77cd1d99 100644 --- a/admin/slices/agent/agent/components/agent/chat/Tab.vue +++ b/admin/slices/agent/agent/components/agent/chat/Tab.vue @@ -94,6 +94,7 @@ watch( :title="`Chat with ${agent.name}`" :restart-prompt="false" :agent-state="bridleAgentState" + :initial-debug-enabled="agent.debugEnabled" class="h-full w-full gap-0" /> diff --git a/admin/slices/agent/agent/components/agent/logs/Panel.vue b/admin/slices/agent/agent/components/agent/logs/Panel.vue index b5a37e54..d3ace09e 100644 --- a/admin/slices/agent/agent/components/agent/logs/Panel.vue +++ b/admin/slices/agent/agent/components/agent/logs/Panel.vue @@ -11,6 +11,10 @@ const props = defineProps<{ // (ContainerCreating 400s and the like). An overlay says what's actually // happening instead of surfacing that noise. restarting?: boolean; + // First-ever start of this agent (server-derived launchContext='initial'): + // the overlay reads "setting up" instead of "restarting", so a fresh deploy + // doesn't look like an update of something that already existed. + firstStart?: boolean; }>(); const emit = defineEmits<{ close: [] }>(); @@ -89,9 +93,15 @@ const LOG_LEVEL_TEXT: Record = { class="absolute inset-0 z-10 flex flex-col items-center justify-center gap-2 bg-background/70 backdrop-blur-[2px]" > - Agent is restarting… + + {{ firstStart ? 'Setting up agent…' : 'Agent is restarting…' }} + - Logs will resume when the new pod is up. + {{ + firstStart + ? 'First start — logs will appear once the agent’s pod is up.' + : 'Logs will resume when the new pod is up.' + }} diff --git a/admin/slices/agent/agent/components/agent/overview/RuntimeCard.vue b/admin/slices/agent/agent/components/agent/overview/RuntimeCard.vue index fdd1b8b4..1244422b 100644 --- a/admin/slices/agent/agent/components/agent/overview/RuntimeCard.vue +++ b/admin/slices/agent/agent/components/agent/overview/RuntimeCard.vue @@ -34,6 +34,12 @@ const podLabel = computed(() => podPhaseLabel(podStatus.value)); · restarts {{ podStatus.restartCount }} +

+ {{ agent.statusReason }} +

Visibility
diff --git a/admin/slices/agent/agent/composables/useAgentLifecycle.ts b/admin/slices/agent/agent/composables/useAgentLifecycle.ts index 82334d12..b03006a1 100644 --- a/admin/slices/agent/agent/composables/useAgentLifecycle.ts +++ b/admin/slices/agent/agent/composables/useAgentLifecycle.ts @@ -152,11 +152,19 @@ export function useAgentLifecycle( // Each refresh replaces the agent ref, which re-runs the computeds (fresh // Date.now() → TTL honored) and gives the server a chance to reconcile. let statusTimer: ReturnType | null = null; + // While a lifecycle mutation (restart/stop/start) is awaiting its HTTP + // response, a poll tick can resolve with the PRE-mutation status and + // overwrite the optimistic 'deploying'/'stopped' flip wholesale. Skip + // ticks for that window — the mutation handlers refresh() on completion. + const pollTick = () => { + if (restarting.value || toggling.value) return; + void refresh(); + }; watch( () => [agent.value?.status, agentStore.isRestartInFlight(agentId)] as const, ([status, inFlight]) => { if ((status && POLL_STATUSES.has(status)) || inFlight) { - if (!statusTimer) statusTimer = setInterval(refresh, 5000); + if (!statusTimer) statusTimer = setInterval(pollTick, 5000); } else if (statusTimer) { clearInterval(statusTimer); statusTimer = null; @@ -189,22 +197,25 @@ export function useAgentLifecycle( // Pod info only describes the FRESH pod; while the old one is still // being torn down it would misleadingly read "Ready". const freshPod = pod && (agentWentDown.value || !inFlight) ? pod : null; + // Server-derived launch context: a first-ever start reads "setting up", + // anything else reads "starting" — so a fresh deploy no longer looks + // like an update of something that already existed. + const firstStart = agent.value.launchContext === 'initial'; return { kind: 'starting', - title: 'Starting agent…', + title: firstStart ? 'Setting up agent…' : 'Starting agent…', detail: freshPod ? `Pod ${freshPod.podName}: ${podLabel.value ?? freshPod.phase}` - : 'Cancelling old workflow and submitting a fresh one.', + : firstStart + ? 'First start — preparing the agent’s pod.' + : 'Cancelling old workflow and submitting a fresh one.', }; } - // Strongest "agent is up" signal: chat WS is connected AND the runtime is - // registered with the hub. This bypasses DB/pod entirely — if the agent - // is actually talking to us, nothing else matters. - const chatLive = bridleStore.isConnected && bridleStore.isAgentConnected; - - if (chatLive) return null; - + // An explicit stop wins over the live-chat bypass below: the old + // runtime's WS can linger after the pod delete (indefinitely on local + // dev, where there is no pod to kill) — reading that as "all good" hid + // the stopped state from the user entirely. if (s === 'stopped') { return { kind: 'stopped', @@ -214,11 +225,22 @@ export function useAgentLifecycle( }; } + // Strongest "agent is up" signal: chat WS is connected AND the runtime is + // registered with the hub. This bypasses DB/pod entirely — if the agent + // is actually talking to us, nothing else matters. + const chatLive = bridleStore.isConnected && bridleStore.isAgentConnected; + + if (chatLive) return null; + if (s === 'failed') { return { kind: 'failed', title: 'Agent failed to start', + // Server-side statusReason is the authoritative cause (startup + // timeout, ImagePullBackOff, workflow submit error, …); live pod + // details are the fallback for failures recorded before it existed. detail: + agent.value.statusReason ?? pod?.message ?? pod?.containerWaitingReason ?? 'Pod did not come up. Check logs and restart.', diff --git a/admin/slices/agent/agent/data/agent.mapper.ts b/admin/slices/agent/agent/data/agent.mapper.ts index 58ea464e..f5750752 100644 --- a/admin/slices/agent/agent/data/agent.mapper.ts +++ b/admin/slices/agent/agent/data/agent.mapper.ts @@ -11,6 +11,7 @@ import type { IClusterCapacityData, ICreateAgentData, IUpdateAgentData, + LaunchContextTypes, } from '../domain/agent.types'; const KNOWN_STATUSES = new Set([ @@ -21,6 +22,11 @@ const KNOWN_STATUSES = new Set([ 'stopped', ]); +const KNOWN_LAUNCH_CONTEXTS = new Set([ + 'initial', + 'restart', +]); + function num(value: unknown): number { return typeof value === 'number' ? value : 0; } @@ -50,7 +56,11 @@ export class AgentMapper { llmCredentialId: typeof o.llmCredentialId === 'string' ? o.llmCredentialId : null, status: this.toStatus(o.status), + statusReason: typeof o.statusReason === 'string' ? o.statusReason : null, workflowId: typeof o.workflowId === 'string' ? o.workflowId : null, + firstDeployedAt: + typeof o.firstDeployedAt === 'string' ? o.firstDeployedAt : null, + launchContext: this.toLaunchContext(o.launchContext), config: o.config && typeof o.config === 'object' ? (o.config as Record) @@ -137,6 +147,13 @@ export class AgentMapper { : 'pending'; } + private toLaunchContext(raw: unknown): LaunchContextTypes | null { + return typeof raw === 'string' && + KNOWN_LAUNCH_CONTEXTS.has(raw as LaunchContextTypes) + ? (raw as LaunchContextTypes) + : null; + } + private toResources(raw: unknown): IAgentResources { const r = raw && typeof raw === 'object' ? (raw as Record) : {}; return { diff --git a/admin/slices/agent/agent/domain/agent.types.ts b/admin/slices/agent/agent/domain/agent.types.ts index 23d2410a..1304781a 100644 --- a/admin/slices/agent/agent/domain/agent.types.ts +++ b/admin/slices/agent/agent/domain/agent.types.ts @@ -9,6 +9,10 @@ export type AgentStatusTypes = | 'failed' | 'stopped'; +/** Why the current/last deploy ran — server-derived, so the UI can tell a + * first start from a restart even after a page reload. */ +export type LaunchContextTypes = 'initial' | 'restart'; + export interface IAgentResources { cpu: string; memory: string; @@ -49,7 +53,13 @@ export interface IAgentData { templateId: string; llmCredentialId: string | null; status: AgentStatusTypes; + /** Human-readable failure cause; non-null only while status is 'failed' + * (may still be null for failures recorded before the field existed). */ + statusReason: string | null; workflowId: string | null; + /** Null ⇒ the agent has never been deployed. */ + firstDeployedAt: string | null; + launchContext: LaunchContextTypes | null; config: Record; resources: IAgentResources; isPublic: boolean; diff --git a/admin/slices/agent/agent/utils/agentLogs.ts b/admin/slices/agent/agent/utils/agentLogs.ts index c68d935a..e3f70140 100644 --- a/admin/slices/agent/agent/utils/agentLogs.ts +++ b/admin/slices/agent/agent/utils/agentLogs.ts @@ -36,12 +36,28 @@ const ERROR_TOKEN_RE = const WARN_TOKEN_RE = /(?:^|[[("']|·\s*|\blevel[=:]\s*"?)\s*warn(?:ing)?(?=[\s:\])"',]|$)/i; +// The runtime's pretty format opens every line with an explicit level glyph +// (logger.ts GLYPH map: `·` debug/info, `✓` ok, `⚠` warn, `✗` error). When +// present it is AUTHORITATIVE — a ⚠ warn line whose body happens to contain +// `"error":"Not Found"` (a quoted JSON payload) must not be escalated to +// error by the substring heuristics above. Optional ANSI wrappers tolerated +// for FORCE_COLOR runs. +// The optional leading `HH:MM:SS.mmm` covers undated lines where the runtime's +// own time prefix wasn't stripped (no K8s timestamp on the line). +const GLYPH_LEVEL_RE = + // eslint-disable-next-line no-control-regex + /^\s*(?:\d{2}:\d{2}:\d{2}\.\d{3}\s+)?(?:\x1b\[[0-9;]*m)?([·✓⚠✗])(?:\x1b\[[0-9;]*m)?\s/u; + // The runtime prefixes its own `HH:MM:SS.mmm ` to every line. When the K8s // timestamp already gives us the time column, that prefix is pure duplication // — strip it (the column carries milliseconds, so no precision is lost). const RUNTIME_TIME_PREFIX_RE = /^\d{2}:\d{2}:\d{2}\.\d{3}\s+/; export function detectLogLevel(text: string): AgentLogLevel | null { + const glyph = text.match(GLYPH_LEVEL_RE)?.[1]; + if (glyph === '✗') return 'error'; + if (glyph === '⚠') return 'warn'; + if (glyph === '·' || glyph === '✓') return null; if (ERROR_UPPER_RE.test(text) || ERROR_TOKEN_RE.test(text)) return 'error'; if (WARN_UPPER_RE.test(text) || WARN_TOKEN_RE.test(text)) return 'warn'; return null; diff --git a/admin/slices/bridle/components/bridle/Provider.vue b/admin/slices/bridle/components/bridle/Provider.vue index 27e8e841..cf283aa3 100644 --- a/admin/slices/bridle/components/bridle/Provider.vue +++ b/admin/slices/bridle/components/bridle/Provider.vue @@ -28,12 +28,17 @@ const props = withDefaults(defineProps<{ // page a failed agent reads as "reconnecting" for 30s. Hosts that know the // real state (the admin agent page) pass it here; null = derive from WS. agentState?: 'restarting' | 'failed' | 'stopped' | null + // Host-supplied debugEnabled from an agent record the host already fetched. + // When set (non-null), the widget skips its own GET /agents/:id — the admin + // agent page otherwise loads the same agent twice on every open. + initialDebugEnabled?: boolean | null }>(), { title: 'Agent Chat', placeholder: 'Type a message...', showStatus: true, restartPrompt: true, agentState: null, + initialDebugEnabled: null, }) const store = useBridleStore() @@ -229,10 +234,17 @@ onMounted(async () => { // Clear before load so the previous agent's messages don't briefly leak // through (the store is a shared singleton across providers). store.clearMessages() - await Promise.all([ - store.loadTranscript(props.apiUrl, props.agentId, props.token), - store.loadAgentMeta(props.apiUrl, props.agentId, props.token), - ]) + // The host may already hold the agent record (admin page useAsyncData) — + // seeding from the prop avoids a duplicate GET /agents/:id on every open. + if (props.initialDebugEnabled !== null) { + store.debugEnabled = props.initialDebugEnabled + await store.loadTranscript(props.apiUrl, props.agentId, props.token) + } else { + await Promise.all([ + store.loadTranscript(props.apiUrl, props.agentId, props.token), + store.loadAgentMeta(props.apiUrl, props.agentId, props.token), + ]) + } // Re-attach debug snapshots saved in localStorage from previous sessions — // makes the inspect icon survive a page refresh. store.loadPersistedDebug(props.agentId) diff --git a/admin/slices/setup/api/data/repositories/api/schemas.gen.ts b/admin/slices/setup/api/data/repositories/api/schemas.gen.ts index 05ae19d2..2f41822d 100644 --- a/admin/slices/setup/api/data/repositories/api/schemas.gen.ts +++ b/admin/slices/setup/api/data/repositories/api/schemas.gen.ts @@ -619,6 +619,119 @@ export const AddFromArchiveResultDtoSchema = { required: ["detected", "started"], } as const; +export const AgentDtoSchema = { + type: "object", + properties: { + id: { + type: "string", + }, + name: { + type: "string", + }, + templateId: { + type: "string", + }, + llmCredentialId: { + type: "object", + nullable: true, + }, + status: { + type: "string", + enum: ["pending", "deploying", "running", "failed", "stopped"], + }, + statusReason: { + type: "string", + nullable: true, + description: `Human-readable reason accompanying status='failed' (e.g. "startup did not produce a running agent within 5 minutes", "ImagePullBackOff"). Null for all other statuses and for failures recorded before this field existed.`, + }, + workflowId: { + type: "object", + nullable: true, + }, + firstDeployedAt: { + type: "string", + nullable: true, + description: + "When this agent was first successfully deployed. Null ⇒ the agent has never been deployed.", + }, + lastDeployStartedAt: { + type: "string", + nullable: true, + description: + "When the current/last deploy was started. Anchor of the server-side deploy grace window.", + }, + launchContext: { + type: "string", + nullable: true, + enum: ["initial", "restart"], + description: + "Why the current/last deploy ran: 'initial' = first-ever start, 'restart' = any subsequent deploy (restart, start after stop, config-change redeploy). Null only for agents never deployed since this field existed.", + }, + config: { + type: "object", + }, + resources: { + type: "object", + }, + debugEnabled: { + type: "boolean", + description: + "When true, the agent runtime emits prompt-debug snapshots to admin clients via the bridle hub.", + }, + isPublic: { + type: "boolean", + description: + "When true, the agent is visible on the public landing page to unauthenticated visitors.", + }, + allowedOrigins: { + description: + "Origins (scheme + host + port) authorized to open browser WebSockets to this bot without a JWT. Only consulted when isPublic=true.", + example: ["https://bridle.cleanslice.org", "http://localhost:5173"], + type: "array", + items: { + type: "string", + }, + }, + knowledgeIds: { + type: "array", + items: { + type: "string", + }, + }, + isAdmin: { + type: "boolean", + }, + createdAt: { + format: "date-time", + type: "string", + }, + updatedAt: { + format: "date-time", + type: "string", + }, + }, + required: [ + "id", + "name", + "templateId", + "status", + "statusReason", + "workflowId", + "firstDeployedAt", + "lastDeployStartedAt", + "launchContext", + "config", + "resources", + "debugEnabled", + "isPublic", + "allowedOrigins", + "knowledgeIds", + "isAdmin", + "createdAt", + "updatedAt", + ], +} as const; + export const AgentPodStatusDtoSchema = { type: "object", properties: { @@ -685,8 +798,12 @@ export const AgentStatusDtoSchema = { type: "object", properties: { agent: { - type: "object", - description: "Agent DB record (id, name, status, etc.)", + description: "Agent DB record (id, name, status, launchContext, etc.)", + allOf: [ + { + $ref: "#/components/schemas/AgentDto", + }, + ], }, pod: { nullable: true, diff --git a/admin/slices/setup/api/data/repositories/api/sdk.gen.ts b/admin/slices/setup/api/data/repositories/api/sdk.gen.ts index 2dcb0491..c70cab81 100644 --- a/admin/slices/setup/api/data/repositories/api/sdk.gen.ts +++ b/admin/slices/setup/api/data/repositories/api/sdk.gen.ts @@ -64,8 +64,10 @@ import type { DeleteKnowledgeSourceData, DeleteKnowledgeSourceResponse, AgentControllerFindAllData, + AgentControllerFindAllResponse, AgentControllerCreateData, AgentControllerFindPublicData, + AgentControllerFindPublicResponse, AgentControllerStatusData, AgentControllerStatusResponse, AgentControllerStatusStreamData, @@ -73,6 +75,7 @@ import type { GetClusterCapacityResponse, AgentControllerRemoveData, AgentControllerFindByIdData, + AgentControllerFindByIdResponse, AgentControllerUpdateData, GetAgentMetricsData, GetAgentMetricsResponse, @@ -1250,7 +1253,7 @@ export class AgentsService { options?: Options, ) { return (options?.client ?? _heyApiClient).get< - unknown, + AgentControllerFindAllResponse, unknown, ThrowOnError >({ @@ -1286,7 +1289,7 @@ export class AgentsService { options?: Options, ) { return (options?.client ?? _heyApiClient).get< - unknown, + AgentControllerFindPublicResponse, unknown, ThrowOnError >({ @@ -1366,7 +1369,7 @@ export class AgentsService { options: Options, ) { return (options.client ?? _heyApiClient).get< - unknown, + AgentControllerFindByIdResponse, unknown, ThrowOnError >({ diff --git a/admin/slices/setup/api/data/repositories/api/types.gen.ts b/admin/slices/setup/api/data/repositories/api/types.gen.ts index 8b037b53..0006b316 100644 --- a/admin/slices/setup/api/data/repositories/api/types.gen.ts +++ b/admin/slices/setup/api/data/repositories/api/types.gen.ts @@ -249,6 +249,57 @@ export type AddFromArchiveResultDto = { started: boolean; }; +export type AgentDto = { + id: string; + name: string; + templateId: string; + llmCredentialId?: { + [key: string]: unknown; + } | null; + status: "pending" | "deploying" | "running" | "failed" | "stopped"; + /** + * Human-readable reason accompanying status='failed' (e.g. "startup did not produce a running agent within 5 minutes", "ImagePullBackOff"). Null for all other statuses and for failures recorded before this field existed. + */ + statusReason: string | null; + workflowId: { + [key: string]: unknown; + } | null; + /** + * When this agent was first successfully deployed. Null ⇒ the agent has never been deployed. + */ + firstDeployedAt: string | null; + /** + * When the current/last deploy was started. Anchor of the server-side deploy grace window. + */ + lastDeployStartedAt: string | null; + /** + * Why the current/last deploy ran: 'initial' = first-ever start, 'restart' = any subsequent deploy (restart, start after stop, config-change redeploy). Null only for agents never deployed since this field existed. + */ + launchContext: "initial" | "restart"; + config: { + [key: string]: unknown; + }; + resources: { + [key: string]: unknown; + }; + /** + * When true, the agent runtime emits prompt-debug snapshots to admin clients via the bridle hub. + */ + debugEnabled: boolean; + /** + * When true, the agent is visible on the public landing page to unauthenticated visitors. + */ + isPublic: boolean; + /** + * Origins (scheme + host + port) authorized to open browser WebSockets to this bot without a JWT. Only consulted when isPublic=true. + */ + allowedOrigins: Array; + knowledgeIds: Array; + isAdmin: boolean; + createdAt: string; + updatedAt: string; +}; + export type AgentPodStatusDto = { agentId: string; podName: string; @@ -264,11 +315,9 @@ export type AgentPodStatusDto = { export type AgentStatusDto = { /** - * Agent DB record (id, name, status, etc.) + * Agent DB record (id, name, status, launchContext, etc.) */ - agent: { - [key: string]: unknown; - }; + agent: AgentDto; /** * Live pod status; null if no pod is currently running for this agent. */ @@ -1970,9 +2019,12 @@ export type AgentControllerFindAllData = { }; export type AgentControllerFindAllResponses = { - 200: unknown; + 200: Array; }; +export type AgentControllerFindAllResponse = + AgentControllerFindAllResponses[keyof AgentControllerFindAllResponses]; + export type AgentControllerCreateData = { body: CreateAgentDto; path?: never; @@ -1992,9 +2044,12 @@ export type AgentControllerFindPublicData = { }; export type AgentControllerFindPublicResponses = { - 200: unknown; + 200: Array; }; +export type AgentControllerFindPublicResponse = + AgentControllerFindPublicResponses[keyof AgentControllerFindPublicResponses]; + export type AgentControllerStatusData = { body?: never; path?: never; @@ -2059,9 +2114,12 @@ export type AgentControllerFindByIdData = { }; export type AgentControllerFindByIdResponses = { - 200: unknown; + 200: AgentDto; }; +export type AgentControllerFindByIdResponse = + AgentControllerFindByIdResponses[keyof AgentControllerFindByIdResponses]; + export type AgentControllerUpdateData = { body: UpdateAgentDto; path: { diff --git a/api/prisma/migrations/20260730120000_agent_startup_status_fields/migration.sql b/api/prisma/migrations/20260730120000_agent_startup_status_fields/migration.sql new file mode 100644 index 00000000..b2773771 --- /dev/null +++ b/api/prisma/migrations/20260730120000_agent_startup_status_fields/migration.sql @@ -0,0 +1,5 @@ +-- Additive, nullable-only: safe on an existing database. +ALTER TABLE "Agent" ADD COLUMN "statusReason" TEXT; +ALTER TABLE "Agent" ADD COLUMN "firstDeployedAt" TIMESTAMP(3); +ALTER TABLE "Agent" ADD COLUMN "lastDeployStartedAt" TIMESTAMP(3); +ALTER TABLE "Agent" ADD COLUMN "lastLaunchContext" TEXT; diff --git a/api/prisma/migrations/20260730130000_backfill_first_deployed_at/migration.sql b/api/prisma/migrations/20260730130000_backfill_first_deployed_at/migration.sql new file mode 100644 index 00000000..6332684c --- /dev/null +++ b/api/prisma/migrations/20260730130000_backfill_first_deployed_at/migration.sql @@ -0,0 +1,9 @@ +-- Agents created before firstDeployedAt existed: any non-pending status means +-- the agent has been deployed at least once, so backfill (updatedAt is the +-- closest available approximation). Without this, their first post-migration +-- deploy would read as launchContext='initial' ("Setting up agent…") even +-- though they have been running for weeks. +UPDATE "Agent" +SET "firstDeployedAt" = "updatedAt" +WHERE "firstDeployedAt" IS NULL + AND "status" <> 'pending'; diff --git a/api/src/slices/agent/agent/agent.controller.ts b/api/src/slices/agent/agent/agent.controller.ts index 1e5ec813..09449075 100644 --- a/api/src/slices/agent/agent/agent.controller.ts +++ b/api/src/slices/agent/agent/agent.controller.ts @@ -28,6 +28,7 @@ import { AgentStatusTypes } from './domain/agent.types'; import { AgentStatusService } from './domain/agentStatus.service'; import { AgentDeployService } from './domain/agentDeploy.service'; import { + AgentDto, AgentMcpDto, AgentEnvVarDto, AgentMetricsDto, @@ -102,11 +103,25 @@ export class AgentController { agent.workflowId, ); const mapped = PHASE_TO_STATUS[phase]; - if (mapped === 'failed' && agent.status !== 'failed') { + // This can only ever be the CURRENT workflow: restart detaches the old + // workflow id before cancelling it and stop clears it, so a terminal + // phase here is a definitive failure of the in-flight deploy — not a + // stale echo of a cancelled run. + // + // Bridle-truth wins (same rule as the drift sweep): if the runtime is + // actively connected to the chat hub, the agent IS up no matter what + // the workflow record claims — writing 'failed' here would ping-pong + // the status against the reconciler's 'running' on every poll. + if ( + mapped === 'failed' && + agent.status !== 'failed' && + !this.bridleHub.isAgentConnected(agentId) + ) { await this.agentGateway.updateStatus( agentId, 'failed', agent.workflowId, + `deploy workflow ${phase.toLowerCase()}`, ); return this.agentGateway.findById(agentId); } @@ -122,6 +137,7 @@ export class AgentController { summary: 'List all agents. Public — landing/chat pages render without auth. Mutations and details still require login.', }) + @ApiOkResponse({ type: AgentDto, isArray: true }) findAll() { return this.agentGateway.findAll(); } @@ -132,6 +148,7 @@ export class AgentController { summary: 'List agents flagged as public. Used by the marketing landing page so private agents stay hidden from unauthenticated visitors.', }) + @ApiOkResponse({ type: AgentDto, isArray: true }) findPublic() { return this.agentGateway.findPublic(); } @@ -181,6 +198,7 @@ export class AgentController { summary: 'Get agent by ID. Public — chat needs agent metadata (name, status) to render.', }) + @ApiOkResponse({ type: AgentDto }) async findById(@Param('id') id: string) { const agent = await this.syncStatus(id); if (!agent) throw new NotFoundException('Agent not found'); @@ -323,13 +341,10 @@ export class AgentController { const previous = await this.agentGateway.findAdmin(); if (previous && previous.id !== agent.id) { await this.agentGateway.setAdmin(previous.id, false); - try { - await this.workflowService.cancelAgentWorkflow(previous.workflowId); - } catch (err) { - this.logger.warn( - `Cancel workflow failed for previous admin ${previous.id}: ${(err as Error).message}`, - ); - } + await this.agentDeployService.detachAndCancelWorkflow( + previous.id, + previous.workflowId, + ); await this.deploy(previous.id); } await this.agentGateway.setAdmin(agent.id, true); @@ -374,22 +389,13 @@ export class AgentController { const previous = await this.agentGateway.findAdmin(); await this.agentGateway.setAdmin(id, true); if (previous && previous.id !== id) { - try { - await this.workflowService.cancelAgentWorkflow(previous.workflowId); - } catch (err) { - this.logger.warn( - `Cancel workflow failed for previous admin ${previous.id}: ${(err as Error).message}`, - ); - } - await this.deploy(previous.id); - } - try { - await this.workflowService.cancelAgentWorkflow(agent.workflowId); - } catch (err) { - this.logger.warn( - `Cancel workflow failed for agent ${id}: ${(err as Error).message}`, + await this.agentDeployService.detachAndCancelWorkflow( + previous.id, + previous.workflowId, ); + await this.deploy(previous.id); } + await this.agentDeployService.detachAndCancelWorkflow(id, agent.workflowId); await this.deploy(id); return this.agentGateway.findById(id); } @@ -404,13 +410,7 @@ export class AgentController { const agent = await this.agentGateway.findById(id); if (!agent) throw new NotFoundException('Agent not found'); await this.agentGateway.setAdmin(id, false); - try { - await this.workflowService.cancelAgentWorkflow(agent.workflowId); - } catch (err) { - this.logger.warn( - `Cancel workflow failed for agent ${id}: ${(err as Error).message}`, - ); - } + await this.agentDeployService.detachAndCancelWorkflow(id, agent.workflowId); await this.deploy(id); return this.agentGateway.findById(id); } diff --git a/api/src/slices/agent/agent/agent.prisma b/api/src/slices/agent/agent/agent.prisma index 41226930..215f04d7 100644 --- a/api/src/slices/agent/agent/agent.prisma +++ b/api/src/slices/agent/agent/agent.prisma @@ -13,7 +13,20 @@ model Agent { llmCredentialId String? llmCredential LlmCredential? @relation(fields: [llmCredentialId], references: [id], onDelete: SetNull) status String @default("pending") + // Human-readable reason accompanying status='failed'; null for all other + // statuses (cleared on every transition out of 'failed'). + statusReason String? workflowId String? + // Set once, on the first successful workflow submit. Null ⇒ never deployed. + firstDeployedAt DateTime? + // Anchor of the deploy grace window: written together with every + // status='deploying' transition. Drift detection won't fail a pod-less + // agent while `now - lastDeployStartedAt` is inside the grace period. + lastDeployStartedAt DateTime? + // 'initial' (first-ever deploy) | 'restart' (any subsequent deploy). + // Server-derived launch context so the UI can tell a first start from a + // restart even after a page reload. + lastLaunchContext String? config Json @default("{}") resources Json @default("{\"cpu\": \"500m\", \"memory\": \"512Mi\"}") debugEnabled Boolean @default(false) diff --git a/api/src/slices/agent/agent/data/agent.gateway.ts b/api/src/slices/agent/agent/data/agent.gateway.ts index d4ada3dd..d30ba92d 100644 --- a/api/src/slices/agent/agent/data/agent.gateway.ts +++ b/api/src/slices/agent/agent/data/agent.gateway.ts @@ -7,6 +7,7 @@ import { ICreateAgentData, IUpdateAgentData, AgentStatusTypes, + LaunchContextTypes, } from '../domain/agent.types'; import { AgentMapper } from './agent.mapper'; @@ -91,11 +92,16 @@ export class AgentGateway extends IAgentGateway { id: string, status: AgentStatusTypes, workflowId?: string | null, + statusReason?: string, ): Promise { const record = await this.prisma.agent.update({ where: { id }, data: { status, + // statusReason lives and dies with 'failed': every transition to any + // other status clears it, so a reason can never outlive the failure + // it describes. + statusReason: status === 'failed' ? (statusReason ?? null) : null, // `undefined` leaves the column untouched; `null` clears it (used when // stopping an agent so the now-cancelled workflow id isn't kept around). ...(workflowId !== undefined && { workflowId }), @@ -104,7 +110,35 @@ export class AgentGateway extends IAgentGateway { return this.mapper.toEntity(record); } - async setWorkflowId(id: string, workflowId: string): Promise { + async markDeployStarted( + id: string, + launchContext: LaunchContextTypes, + ): Promise { + const record = await this.prisma.agent.update({ + where: { id }, + data: { + status: 'deploying', + statusReason: null, + lastDeployStartedAt: new Date(), + lastLaunchContext: launchContext, + }, + }); + return this.mapper.toEntity(record); + } + + async setFirstDeployedAt(id: string): Promise { + // Conditional updateMany keeps set-once semantics without a read-modify- + // write race: only the very first deploy finds firstDeployedAt IS NULL. + await this.prisma.agent.updateMany({ + where: { id, firstDeployedAt: null }, + data: { firstDeployedAt: new Date() }, + }); + } + + async setWorkflowId( + id: string, + workflowId: string | null, + ): Promise { const record = await this.prisma.agent.update({ where: { id }, data: { workflowId }, diff --git a/api/src/slices/agent/agent/data/agent.mapper.ts b/api/src/slices/agent/agent/data/agent.mapper.ts index c7fe2f4c..942be276 100644 --- a/api/src/slices/agent/agent/data/agent.mapper.ts +++ b/api/src/slices/agent/agent/data/agent.mapper.ts @@ -11,7 +11,12 @@ export class AgentMapper { templateId: record.templateId, llmCredentialId: record.llmCredentialId, status: record.status as IAgentData['status'], + statusReason: record.statusReason, workflowId: record.workflowId, + firstDeployedAt: record.firstDeployedAt, + lastDeployStartedAt: record.lastDeployStartedAt, + launchContext: + record.lastLaunchContext as IAgentData['launchContext'], config: record.config as unknown as Record, resources: record.resources as unknown as IAgentData['resources'], debugEnabled: record.debugEnabled, diff --git a/api/src/slices/agent/agent/domain/agent.gateway.ts b/api/src/slices/agent/agent/domain/agent.gateway.ts index 4ab226a4..40a26b28 100644 --- a/api/src/slices/agent/agent/domain/agent.gateway.ts +++ b/api/src/slices/agent/agent/domain/agent.gateway.ts @@ -3,6 +3,7 @@ import { ICreateAgentData, IUpdateAgentData, AgentStatusTypes, + LaunchContextTypes, } from './agent.types'; export abstract class IAgentGateway { @@ -17,8 +18,22 @@ export abstract class IAgentGateway { id: string, status: AgentStatusTypes, workflowId?: string | null, + statusReason?: string, + ): Promise; + // Atomic deploy-start write: status='deploying', statusReason cleared, + // lastDeployStartedAt=now, lastLaunchContext=. + abstract markDeployStarted( + id: string, + launchContext: LaunchContextTypes, + ): Promise; + // Sets firstDeployedAt=now only if it is still null (set-once semantics). + abstract setFirstDeployedAt(id: string): Promise; + // null detaches the current workflow (used by restart before cancelling + // the old one, so status pollers can't resolve its terminal phase). + abstract setWorkflowId( + id: string, + workflowId: string | null, ): Promise; - abstract setWorkflowId(id: string, workflowId: string): Promise; abstract setAdmin(id: string, enabled: boolean): Promise; abstract delete(id: string): Promise; } diff --git a/api/src/slices/agent/agent/domain/agent.types.ts b/api/src/slices/agent/agent/domain/agent.types.ts index 57e7375d..91acab0f 100644 --- a/api/src/slices/agent/agent/domain/agent.types.ts +++ b/api/src/slices/agent/agent/domain/agent.types.ts @@ -5,13 +5,23 @@ export type AgentStatusTypes = | 'failed' | 'stopped'; +// Why the current/last deploy ran: 'initial' = first-ever deploy of this +// agent, 'restart' = any subsequent deploy (manual restart, start after stop, +// config-change redeploy). Server-derived so the UI can distinguish a first +// start from a restart even after a page reload. +export type LaunchContextTypes = 'initial' | 'restart'; + export interface IAgentData { id: string; name: string; templateId: string; llmCredentialId: string | null; status: AgentStatusTypes; + statusReason: string | null; workflowId: string | null; + firstDeployedAt: Date | null; + lastDeployStartedAt: Date | null; + launchContext: LaunchContextTypes | null; config: Record; resources: IAgentResources; debugEnabled: boolean; diff --git a/api/src/slices/agent/agent/domain/agentDeploy.service.ts b/api/src/slices/agent/agent/domain/agentDeploy.service.ts index 090a0873..0194120f 100644 --- a/api/src/slices/agent/agent/domain/agentDeploy.service.ts +++ b/api/src/slices/agent/agent/domain/agentDeploy.service.ts @@ -53,15 +53,30 @@ export class AgentDeployService { await this.syncSkillsFromTemplate(agentId, agent.templateId); + await this.detachAndCancelWorkflow(agentId, agent.workflowId); + + await this.deploy(agentId); + } + + // Detach the workflow id from the agent row BEFORE cancelling it. A + // concurrent GET /agents/:id (syncStatus) polls the workflow referenced by + // the DB row; without this ordering it can catch the just-cancelled + // workflow in phase=Failed and write a spurious 'failed' between the + // cancel and the follow-up deploy()'s 'deploying'. Best-effort — callers + // always deploy() afterwards, which must run regardless. + async detachAndCancelWorkflow( + agentId: string, + workflowId: string | null, + ): Promise { + if (!workflowId) return; try { - await this.workflowService.cancelAgentWorkflow(agent.workflowId); + await this.agentGateway.setWorkflowId(agentId, null); + await this.workflowService.cancelAgentWorkflow(workflowId); } catch (err) { this.logger.warn( `Cancel workflow failed for agent ${agentId}: ${(err as Error).message}`, ); } - - await this.deploy(agentId); } // Stop a running agent: cancel its workflow and delete the pod so the @@ -108,17 +123,27 @@ export class AgentDeployService { this.logger.error( `Template ${agent.templateId} not found for agent ${agentId}`, ); - await this.agentGateway.updateStatus(agentId, 'failed'); + await this.agentGateway.updateStatus( + agentId, + 'failed', + undefined, + `Template ${agent.templateId} not found`, + ); return; } // Idempotent — restartAgent already marked it, but cold deploys (initial // create) call deploy() directly without going through restartAgent. this.deployTracker.mark(agentId); + // 'initial' iff this agent has never been deployed — persisted so the UI + // can tell a first start from a restart even after a page reload. + const launchContext = agent.firstDeployedAt === null ? 'initial' : 'restart'; // Mark deploying BEFORE submitting the workflow. Submit + getStatus take // seconds — long enough for the pod to come up and AgentStatusService to // flip status to 'running'. If we wrote status here after submit we'd // overwrite that 'running' with 'deploying' (last-writer-wins race). - await this.agentGateway.updateStatus(agentId, 'deploying'); + // markDeployStarted also stamps lastDeployStartedAt — the anchor of the + // drift-detection grace window — and clears any stale statusReason. + await this.agentGateway.markDeployStarted(agentId, launchContext); try { // Every agent gets a JWT scoped to its own id. Admin agents get Owner // (full Ranch control), non-admins get the Agent role (self-only @@ -135,11 +160,20 @@ export class AgentDeployService { // Only attach the new workflowId — never touch status post-submit. // Reconciler is the single source of truth for 'running'. await this.agentGateway.setWorkflowId(agentId, workflowId); + await this.agentGateway.setFirstDeployedAt(agentId); } catch (err) { this.logger.error( `Workflow submit failed for agent ${agentId}: ${(err as Error).message}`, ); - await this.agentGateway.updateStatus(agentId, 'failed'); + // Generic on purpose: statusReason is served on public agent endpoints, + // and raw submit errors can carry internal detail (Argo endpoints, + // auth specifics). The full message is in the server log above. + await this.agentGateway.updateStatus( + agentId, + 'failed', + undefined, + 'workflow submit failed', + ); } } diff --git a/api/src/slices/agent/agent/domain/agentStatus.service.ts b/api/src/slices/agent/agent/domain/agentStatus.service.ts index 183801e0..cd9f769d 100644 --- a/api/src/slices/agent/agent/domain/agentStatus.service.ts +++ b/api/src/slices/agent/agent/domain/agentStatus.service.ts @@ -19,6 +19,7 @@ import { import { IAgentGateway } from './agent.gateway'; import { IAgentData } from './agent.types'; import { AgentDeployService } from './agentDeploy.service'; +import { DEPLOY_GRACE_MS, isWithinDeployGrace } from './deployGrace'; import { DeployTracker } from './deployTracker'; import { IPodGateway } from '#/agent/pod/domain'; import { @@ -159,6 +160,11 @@ export class AgentStatusService implements OnModuleInit, OnModuleDestroy { this.deployTracker.clear(agentId); return; } + // Same 'stopped' exemption as the drift sweep: a reconnect from the + // not-yet-dead runtime of an explicitly stopped agent must not undo the + // operator's stop. A subsequent Start goes through deploy() → 'deploying' + // and re-enables this path. + if (agent.status === 'stopped') return; this.logger.log( `Reconciling agent ${agentId}: bridle runtime registered — marking running`, ); @@ -266,7 +272,15 @@ export class AgentStatusService implements OnModuleInit, OnModuleDestroy { // pod-missing/Failed state during a restart shouldn't override a // healthy runtime that's actively talking to us. if (this.bridleGateway.isAgentConnected(agent.id)) { - if (agent.status !== 'running') { + // 'stopped' is exempt from the resurrect: the operator explicitly + // stopped the agent, and the old runtime's WS can linger for a few + // seconds after the pod delete (forever on local dev, where there + // is no pod to kill). Flipping it back to 'running' here would undo + // the stop — and worse, once the WS finally dropped, the next sweep + // would find a pod-less 'running' agent and mark it FAILED. We + // still `continue` so a stopped-but-lingering runtime isn't drift + // -failed either. + if (agent.status !== 'running' && agent.status !== 'stopped') { this.logger.log( `Drift: agent ${agent.id} (${agent.name}) is ${agent.status} in DB but bridle has it registered — marking running`, ); @@ -283,13 +297,24 @@ export class AgentStatusService implements OnModuleInit, OnModuleDestroy { const pod = podByAgent.get(agent.id); if (!pod) { if (LIVE_DB_STATUSES.has(agent.status) && agent.status !== 'failed') { + // A just-submitted deploy legitimately has no pod for ~10-30s + // (Argo runs cleanup-old before run-agent). Inside the grace + // window the absence of a pod is not evidence of failure — a + // healthy stop→start must never flash 'failed'. Once the window + // expires with no pod, THAT is the definitive startup timeout. + if (isWithinDeployGrace(agent)) continue; + const reason = + agent.status === 'running' + ? 'agent pod disappeared' + : `startup did not produce a running agent within ${Math.round(DEPLOY_GRACE_MS / 60_000)} minutes`; this.logger.warn( - `Drift: agent ${agent.id} (${agent.name}) is ${agent.status} in DB but no pod exists — marking failed`, + `Drift: agent ${agent.id} (${agent.name}) is ${agent.status} in DB but no pod exists — marking failed (${reason})`, ); await this.agentGateway.updateStatus( agent.id, 'failed', agent.workflowId ?? undefined, + reason, ); driftFailedIds.push(agent.id); } @@ -378,6 +403,14 @@ export class AgentStatusService implements OnModuleInit, OnModuleDestroy { return; } if (agent.status !== 'failed') { + // Human-readable cause for the UI: prefer the waiting reason + // (CrashLoopBackOff, ImagePullBackOff, …), then the last termination + // reason (OOMKilled, …), then the bare phase. + const reason = + podStatus.containerWaitingReason ?? + podStatus.lastTerminationReason ?? + podStatus.message ?? + `pod ${podStatus.phase}`; this.logger.warn( `Reconciling agent ${agent.id}: pod ${podStatus.podName} is ${podStatus.phase}` + (podStatus.containerWaitingReason @@ -389,6 +422,7 @@ export class AgentStatusService implements OnModuleInit, OnModuleDestroy { agent.id, 'failed', agent.workflowId ?? undefined, + reason, ); } return; diff --git a/api/src/slices/agent/agent/domain/deployGrace.ts b/api/src/slices/agent/agent/domain/deployGrace.ts new file mode 100644 index 00000000..9831cec3 --- /dev/null +++ b/api/src/slices/agent/agent/domain/deployGrace.ts @@ -0,0 +1,19 @@ +import { IAgentData } from './agent.types'; + +// Single source of truth for the deploy grace window AND the definitive- +// failure safety timeout: after a deploy is submitted, the two-step Argo +// workflow (cleanup-old → run-agent) legitimately leaves the agent pod-less +// for ~10-30s. Within this window the absence of a pod is not evidence of +// failure; once it expires with no pod, the startup is definitively failed. +export const DEPLOY_GRACE_MS = 5 * 60_000; + +export function isWithinDeployGrace( + agent: Pick, + now: number = Date.now(), +): boolean { + // Legacy rows deployed before lastDeployStartedAt existed have no anchor; + // fall back to updatedAt so an agent mid-deploy during rollout still gets + // its grace window instead of an instant drift-fail. + const anchor = agent.lastDeployStartedAt ?? agent.updatedAt; + return now - anchor.getTime() <= DEPLOY_GRACE_MS; +} diff --git a/api/src/slices/agent/agent/dtos/agent.dto.ts b/api/src/slices/agent/agent/dtos/agent.dto.ts index 53625905..d80c8075 100644 --- a/api/src/slices/agent/agent/dtos/agent.dto.ts +++ b/api/src/slices/agent/agent/dtos/agent.dto.ts @@ -13,12 +13,46 @@ export class AgentDto { @ApiPropertyOptional({ nullable: true }) llmCredentialId: string | null; - @ApiProperty() + @ApiProperty({ + enum: ['pending', 'deploying', 'running', 'failed', 'stopped'], + }) status: string; + @ApiProperty({ + nullable: true, + type: String, + description: + "Human-readable reason accompanying status='failed' (e.g. \"startup did not produce a running agent within 5 minutes\", \"ImagePullBackOff\"). Null for all other statuses and for failures recorded before this field existed.", + }) + statusReason: string | null; + @ApiProperty({ nullable: true }) workflowId: string | null; + @ApiProperty({ + nullable: true, + type: String, + description: + 'When this agent was first successfully deployed. Null ⇒ the agent has never been deployed.', + }) + firstDeployedAt: Date | null; + + @ApiProperty({ + nullable: true, + type: String, + description: + 'When the current/last deploy was started. Anchor of the server-side deploy grace window.', + }) + lastDeployStartedAt: Date | null; + + @ApiProperty({ + nullable: true, + enum: ['initial', 'restart'], + description: + "Why the current/last deploy ran: 'initial' = first-ever start, 'restart' = any subsequent deploy (restart, start after stop, config-change redeploy). Null only for agents never deployed since this field existed.", + }) + launchContext: 'initial' | 'restart' | null; + @ApiProperty() config: Record; @@ -48,15 +82,8 @@ export class AgentDto { @ApiProperty({ type: [String] }) knowledgeIds: string[]; - @ApiProperty({ - description: - 'Messaging channels the runtime should connect to (telegram, ...). Each entry is { type, config }; mapped to runtime env vars at deploy time.', - isArray: true, - example: [ - { type: 'telegram', config: { botToken: 'xxx', botName: 'mybot' } }, - ], - }) - channels: unknown[]; + @ApiProperty() + isAdmin: boolean; @ApiProperty() createdAt: Date; diff --git a/api/src/slices/agent/agent/dtos/agentStatus.dto.ts b/api/src/slices/agent/agent/dtos/agentStatus.dto.ts index 07115020..5714ef97 100644 --- a/api/src/slices/agent/agent/dtos/agentStatus.dto.ts +++ b/api/src/slices/agent/agent/dtos/agentStatus.dto.ts @@ -1,4 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; +import { AgentDto } from './agent.dto'; export class AgentPodStatusDto { @ApiProperty({ example: 'agent-abc-123' }) @@ -40,8 +41,11 @@ export class AgentPodStatusDto { } export class AgentStatusDto { - @ApiProperty({ description: 'Agent DB record (id, name, status, etc.)' }) - agent: Record; + @ApiProperty({ + type: AgentDto, + description: 'Agent DB record (id, name, status, launchContext, etc.)', + }) + agent: AgentDto; @ApiProperty({ type: AgentPodStatusDto, diff --git a/api/src/slices/agent/pod/data/pod.gateway.ts b/api/src/slices/agent/pod/data/pod.gateway.ts index 3d1b8a24..1ef0e895 100644 --- a/api/src/slices/agent/pod/data/pod.gateway.ts +++ b/api/src/slices/agent/pod/data/pod.gateway.ts @@ -14,6 +14,7 @@ import { } from '@kubernetes/client-node'; import { Observable, Subject } from 'rxjs'; import { IPodGateway } from '../domain/pod.gateway'; +import { formatKubeError } from '../domain/kubeError'; import { IInfraConfigGateway } from '#/setting/domain'; import { AGENT_SLOT_CPU_MILLI, @@ -323,15 +324,7 @@ export class KubePodGateway } private extractKubeError(err: unknown): string { - if (!err || typeof err !== 'object') return String(err); - const e = err as { - body?: { message?: string }; - statusCode?: number; - message?: string; - }; - if (e.body?.message) - return `${e.statusCode ?? ''} ${e.body.message}`.trim(); - return e.message ?? JSON.stringify(e).slice(0, 200); + return formatKubeError(err); } async getMetrics(agentId: string): Promise { diff --git a/api/src/slices/agent/pod/domain/kubeError.ts b/api/src/slices/agent/pod/domain/kubeError.ts new file mode 100644 index 00000000..3a514fbe --- /dev/null +++ b/api/src/slices/agent/pod/domain/kubeError.ts @@ -0,0 +1,43 @@ +// Error-shape normalization for @kubernetes/client-node 1.x. Its ApiException +// carries the k8s Status resource in `body` — but depending on the endpoint +// the SDK delivers it either parsed (object) or as the raw JSON string +// (readNamespacedPodLog does the latter). Reading `err.body.message` as an +// object property therefore silently misses the string case, and callers fall +// back to `err.message` — a multi-line "HTTP-Code: 400\n…Body:…Headers:…" +// dump that must never reach logs or the UI. + +// The k8s Status `message` from an API error, whatever shape the body has. +export function kubeErrorMessage(err: unknown): string | null { + if (!err || typeof err !== 'object') return null; + const body = (err as { body?: unknown }).body; + if (body && typeof body === 'object') { + const msg = (body as { message?: unknown }).message; + return typeof msg === 'string' ? msg : null; + } + if (typeof body === 'string') { + try { + const parsed = JSON.parse(body) as { message?: unknown }; + return typeof parsed.message === 'string' ? parsed.message : null; + } catch { + return null; + } + } + return null; +} + +export function kubeErrorCode(err: unknown): number | null { + if (!err || typeof err !== 'object') return null; + const e = err as { statusCode?: number; code?: number }; + return e.statusCode ?? e.code ?? null; +} + +// Single-line human-readable form for logs and API responses. +export function formatKubeError(err: unknown): string { + if (!err || typeof err !== 'object') return String(err); + const code = kubeErrorCode(err); + const message = kubeErrorMessage(err); + if (message) return `${code ?? ''} ${message}`.trim(); + const raw = (err as { message?: string }).message; + if (raw) return raw.split('\n')[0]; + return JSON.stringify(err).slice(0, 200); +} diff --git a/api/src/slices/log/log.controller.ts b/api/src/slices/log/log.controller.ts index 54dfaddc..f3e90f9f 100644 --- a/api/src/slices/log/log.controller.ts +++ b/api/src/slices/log/log.controller.ts @@ -9,6 +9,11 @@ import { import { ApiTags, ApiOperation } from '@nestjs/swagger'; import { CoreV1Api, KubeConfig } from '@kubernetes/client-node'; import { IAgentGateway } from '#/agent/agent/domain'; +import { + formatKubeError, + kubeErrorCode, + kubeErrorMessage, +} from '#/agent/pod/domain/kubeError'; import { IInfraConfigGateway } from '#/setting/domain'; /** @@ -121,25 +126,21 @@ export class LogController { } private isNotFound(err: unknown): boolean { - if (!err || typeof err !== 'object') return false; - const e = err as { statusCode?: number; code?: number }; - return e.statusCode === 404 || e.code === 404; + return kubeErrorCode(err) === 404; } // K8s returns 400 BadRequest with a body like: // container "agent" in pod "…" is waiting to start: ContainerCreating // The reason after "is waiting to start: " is what we want for the UI. // Same shape covers PodInitializing, CreateContainerConfigError, etc. + // kubeErrorMessage handles both body shapes the k8s SDK produces (parsed + // object vs raw JSON string); the ApiException `message` dump is the last + // resort so this branch can never silently fall through to the raw error. private extractWaitingReason(err: unknown): string | null { - if (!err || typeof err !== 'object') return null; - const e = err as { - statusCode?: number; - code?: number; - body?: { message?: string }; - }; - const status = e.statusCode ?? e.code; - if (status !== 400) return null; - const msg = e.body?.message ?? ''; + if (kubeErrorCode(err) !== 400) return null; + const msg = + kubeErrorMessage(err) ?? + ((err as { message?: string })?.message || ''); const match = msg.match(/is waiting to start:\s*([A-Za-z0-9_]+)/); return match?.[1] ?? null; } @@ -151,14 +152,6 @@ export class LogController { } private extractKubeError(err: unknown): string { - if (!err || typeof err !== 'object') return String(err); - const e = err as { - body?: { message?: string }; - statusCode?: number; - message?: string; - }; - if (e.body?.message) - return `${e.statusCode ?? ''} ${e.body.message}`.trim(); - return e.message ?? JSON.stringify(e).slice(0, 200); + return formatKubeError(err); } } diff --git a/api/src/slices/mcpServer/domain/mcpServer.seeder.ts b/api/src/slices/mcpServer/domain/mcpServer.seeder.ts index 1448275f..a66c1e87 100644 --- a/api/src/slices/mcpServer/domain/mcpServer.seeder.ts +++ b/api/src/slices/mcpServer/domain/mcpServer.seeder.ts @@ -56,9 +56,12 @@ export class McpServerSeeder implements OnApplicationBootstrap { this.logger.log(`Seeded built-in Knowledge MCP server at ${url}`); } + // The Streamable HTTP endpoint lives at /mcp — the bare origin 404s + // ("Cannot POST /"), which used to leave every agent with 0 CleanSlice + // tools and a scary connect-failed line in its startup log. const cleansliceUrl = this.config.get('CLEANSLICE_MCP_URL') ?? - 'https://mcp.cleanslice.org/'; + 'https://mcp.cleanslice.org/mcp'; const existingCleanslice = await this.gateway.findById(CLEANSLICE_MCP_ID); if (!existingCleanslice) { @@ -77,6 +80,16 @@ export class McpServerSeeder implements OnApplicationBootstrap { this.logger.log( `Seeded built-in CleanSlice MCP server at ${cleansliceUrl}`, ); + } else if (existingCleanslice.url !== cleansliceUrl) { + // The api owns built-in rows (url is not editable via the public API), + // so converge an existing row to the configured URL on every bootstrap. + // Idempotent: no-op once the row matches. Heals deployments seeded with + // the old path-less default; agents pick the fix up on their next + // deploy (the MCP list is baked into pod env at deploy time). + await this.gateway.update(CLEANSLICE_MCP_ID, { url: cleansliceUrl }); + this.logger.log( + `Healed built-in CleanSlice MCP server url: ${existingCleanslice.url} → ${cleansliceUrl}`, + ); } } } diff --git a/api/src/slices/rancher/rancher.tool.ts b/api/src/slices/rancher/rancher.tool.ts index 0ee600fb..7acbb196 100644 --- a/api/src/slices/rancher/rancher.tool.ts +++ b/api/src/slices/rancher/rancher.tool.ts @@ -101,11 +101,18 @@ export class RancherTool { this.requireOwner(httpRequest); const agent = await this.agents.findById(id); if (!agent) return ok({ error: `Agent ${id} not found` }); - await this.agents.updateStatus(id, 'deploying'); + // Fire-and-forget the real restart flow (template resync → cancel old + // workflow → deploy). A bare status write here would leave the agent + // stuck in 'deploying' with no new pod until drift detection failed it. + void this.agentDeploy.restartAgent(id).catch((err) => { + this.logger.warn( + `restart_agent tool: restart failed for ${id}: ${(err as Error).message}`, + ); + }); return ok({ ok: true, agentId: id, - message: 'Restart queued — workflow controller will reconcile.', + message: 'Restart started — the agent pod will be replaced shortly.', }); } diff --git a/api/src/slices/workflow/data/mock-workflow.gateway.ts b/api/src/slices/workflow/data/mock-workflow.gateway.ts index 3dbd1e07..924b1a81 100644 --- a/api/src/slices/workflow/data/mock-workflow.gateway.ts +++ b/api/src/slices/workflow/data/mock-workflow.gateway.ts @@ -40,14 +40,18 @@ export class MockWorkflowGateway extends IWorkflowGateway { } async getStatus(workflowId: string): Promise { - return ( - this.workflows.get(workflowId) ?? { - name: workflowId, - phase: 'Failed', - startedAt: null, - finishedAt: null, - } - ); + const existing = this.workflows.get(workflowId); + if (!existing) { + // The store is in-memory — an API restart forgets every workflow. + // Unknown must NOT read as Failed: syncStatus would then write a + // spurious 'failed' for a perfectly healthy agent after every dev + // restart. Throwing mirrors Argo's behavior for a TTL-deleted + // workflow (fetch error → callers treat it as "no signal"). + throw new Error( + `[mock] workflow ${workflowId} not found (in-memory store reset)`, + ); + } + return existing; } async getLogs(workflowId: string): Promise { diff --git a/app/slices/setup/api/data/repositories/api/schemas.gen.ts b/app/slices/setup/api/data/repositories/api/schemas.gen.ts index 05ae19d2..2f41822d 100644 --- a/app/slices/setup/api/data/repositories/api/schemas.gen.ts +++ b/app/slices/setup/api/data/repositories/api/schemas.gen.ts @@ -619,6 +619,119 @@ export const AddFromArchiveResultDtoSchema = { required: ["detected", "started"], } as const; +export const AgentDtoSchema = { + type: "object", + properties: { + id: { + type: "string", + }, + name: { + type: "string", + }, + templateId: { + type: "string", + }, + llmCredentialId: { + type: "object", + nullable: true, + }, + status: { + type: "string", + enum: ["pending", "deploying", "running", "failed", "stopped"], + }, + statusReason: { + type: "string", + nullable: true, + description: `Human-readable reason accompanying status='failed' (e.g. "startup did not produce a running agent within 5 minutes", "ImagePullBackOff"). Null for all other statuses and for failures recorded before this field existed.`, + }, + workflowId: { + type: "object", + nullable: true, + }, + firstDeployedAt: { + type: "string", + nullable: true, + description: + "When this agent was first successfully deployed. Null ⇒ the agent has never been deployed.", + }, + lastDeployStartedAt: { + type: "string", + nullable: true, + description: + "When the current/last deploy was started. Anchor of the server-side deploy grace window.", + }, + launchContext: { + type: "string", + nullable: true, + enum: ["initial", "restart"], + description: + "Why the current/last deploy ran: 'initial' = first-ever start, 'restart' = any subsequent deploy (restart, start after stop, config-change redeploy). Null only for agents never deployed since this field existed.", + }, + config: { + type: "object", + }, + resources: { + type: "object", + }, + debugEnabled: { + type: "boolean", + description: + "When true, the agent runtime emits prompt-debug snapshots to admin clients via the bridle hub.", + }, + isPublic: { + type: "boolean", + description: + "When true, the agent is visible on the public landing page to unauthenticated visitors.", + }, + allowedOrigins: { + description: + "Origins (scheme + host + port) authorized to open browser WebSockets to this bot without a JWT. Only consulted when isPublic=true.", + example: ["https://bridle.cleanslice.org", "http://localhost:5173"], + type: "array", + items: { + type: "string", + }, + }, + knowledgeIds: { + type: "array", + items: { + type: "string", + }, + }, + isAdmin: { + type: "boolean", + }, + createdAt: { + format: "date-time", + type: "string", + }, + updatedAt: { + format: "date-time", + type: "string", + }, + }, + required: [ + "id", + "name", + "templateId", + "status", + "statusReason", + "workflowId", + "firstDeployedAt", + "lastDeployStartedAt", + "launchContext", + "config", + "resources", + "debugEnabled", + "isPublic", + "allowedOrigins", + "knowledgeIds", + "isAdmin", + "createdAt", + "updatedAt", + ], +} as const; + export const AgentPodStatusDtoSchema = { type: "object", properties: { @@ -685,8 +798,12 @@ export const AgentStatusDtoSchema = { type: "object", properties: { agent: { - type: "object", - description: "Agent DB record (id, name, status, etc.)", + description: "Agent DB record (id, name, status, launchContext, etc.)", + allOf: [ + { + $ref: "#/components/schemas/AgentDto", + }, + ], }, pod: { nullable: true, diff --git a/app/slices/setup/api/data/repositories/api/sdk.gen.ts b/app/slices/setup/api/data/repositories/api/sdk.gen.ts index 2dcb0491..c70cab81 100644 --- a/app/slices/setup/api/data/repositories/api/sdk.gen.ts +++ b/app/slices/setup/api/data/repositories/api/sdk.gen.ts @@ -64,8 +64,10 @@ import type { DeleteKnowledgeSourceData, DeleteKnowledgeSourceResponse, AgentControllerFindAllData, + AgentControllerFindAllResponse, AgentControllerCreateData, AgentControllerFindPublicData, + AgentControllerFindPublicResponse, AgentControllerStatusData, AgentControllerStatusResponse, AgentControllerStatusStreamData, @@ -73,6 +75,7 @@ import type { GetClusterCapacityResponse, AgentControllerRemoveData, AgentControllerFindByIdData, + AgentControllerFindByIdResponse, AgentControllerUpdateData, GetAgentMetricsData, GetAgentMetricsResponse, @@ -1250,7 +1253,7 @@ export class AgentsService { options?: Options, ) { return (options?.client ?? _heyApiClient).get< - unknown, + AgentControllerFindAllResponse, unknown, ThrowOnError >({ @@ -1286,7 +1289,7 @@ export class AgentsService { options?: Options, ) { return (options?.client ?? _heyApiClient).get< - unknown, + AgentControllerFindPublicResponse, unknown, ThrowOnError >({ @@ -1366,7 +1369,7 @@ export class AgentsService { options: Options, ) { return (options.client ?? _heyApiClient).get< - unknown, + AgentControllerFindByIdResponse, unknown, ThrowOnError >({ diff --git a/app/slices/setup/api/data/repositories/api/types.gen.ts b/app/slices/setup/api/data/repositories/api/types.gen.ts index 8b037b53..0006b316 100644 --- a/app/slices/setup/api/data/repositories/api/types.gen.ts +++ b/app/slices/setup/api/data/repositories/api/types.gen.ts @@ -249,6 +249,57 @@ export type AddFromArchiveResultDto = { started: boolean; }; +export type AgentDto = { + id: string; + name: string; + templateId: string; + llmCredentialId?: { + [key: string]: unknown; + } | null; + status: "pending" | "deploying" | "running" | "failed" | "stopped"; + /** + * Human-readable reason accompanying status='failed' (e.g. "startup did not produce a running agent within 5 minutes", "ImagePullBackOff"). Null for all other statuses and for failures recorded before this field existed. + */ + statusReason: string | null; + workflowId: { + [key: string]: unknown; + } | null; + /** + * When this agent was first successfully deployed. Null ⇒ the agent has never been deployed. + */ + firstDeployedAt: string | null; + /** + * When the current/last deploy was started. Anchor of the server-side deploy grace window. + */ + lastDeployStartedAt: string | null; + /** + * Why the current/last deploy ran: 'initial' = first-ever start, 'restart' = any subsequent deploy (restart, start after stop, config-change redeploy). Null only for agents never deployed since this field existed. + */ + launchContext: "initial" | "restart"; + config: { + [key: string]: unknown; + }; + resources: { + [key: string]: unknown; + }; + /** + * When true, the agent runtime emits prompt-debug snapshots to admin clients via the bridle hub. + */ + debugEnabled: boolean; + /** + * When true, the agent is visible on the public landing page to unauthenticated visitors. + */ + isPublic: boolean; + /** + * Origins (scheme + host + port) authorized to open browser WebSockets to this bot without a JWT. Only consulted when isPublic=true. + */ + allowedOrigins: Array; + knowledgeIds: Array; + isAdmin: boolean; + createdAt: string; + updatedAt: string; +}; + export type AgentPodStatusDto = { agentId: string; podName: string; @@ -264,11 +315,9 @@ export type AgentPodStatusDto = { export type AgentStatusDto = { /** - * Agent DB record (id, name, status, etc.) + * Agent DB record (id, name, status, launchContext, etc.) */ - agent: { - [key: string]: unknown; - }; + agent: AgentDto; /** * Live pod status; null if no pod is currently running for this agent. */ @@ -1970,9 +2019,12 @@ export type AgentControllerFindAllData = { }; export type AgentControllerFindAllResponses = { - 200: unknown; + 200: Array; }; +export type AgentControllerFindAllResponse = + AgentControllerFindAllResponses[keyof AgentControllerFindAllResponses]; + export type AgentControllerCreateData = { body: CreateAgentDto; path?: never; @@ -1992,9 +2044,12 @@ export type AgentControllerFindPublicData = { }; export type AgentControllerFindPublicResponses = { - 200: unknown; + 200: Array; }; +export type AgentControllerFindPublicResponse = + AgentControllerFindPublicResponses[keyof AgentControllerFindPublicResponses]; + export type AgentControllerStatusData = { body?: never; path?: never; @@ -2059,9 +2114,12 @@ export type AgentControllerFindByIdData = { }; export type AgentControllerFindByIdResponses = { - 200: unknown; + 200: AgentDto; }; +export type AgentControllerFindByIdResponse = + AgentControllerFindByIdResponses[keyof AgentControllerFindByIdResponses]; + export type AgentControllerUpdateData = { body: UpdateAgentDto; path: { diff --git a/package.json b/package.json index a31e57d8..ad8ebd9d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ranch", - "version": "0.3.27", + "version": "0.3.28", "private": true, "packageManager": "bun@1.2.12", "workspaces": [ diff --git a/specs/001-stabilize-agent-startup/checklists/requirements.md b/specs/001-stabilize-agent-startup/checklists/requirements.md new file mode 100644 index 00000000..22b84a07 --- /dev/null +++ b/specs/001-stabilize-agent-startup/checklists/requirements.md @@ -0,0 +1,36 @@ +# Specification Quality Checklist: Stabilize Agent Startup Status & Logs + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-07-30 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- Validation iteration 1: all items pass. Raw error text and status names ("deploying", "failed") appear only as descriptions of today's observed behavior and user-visible vocabulary, not as implementation prescriptions. +- The root cause of the first-launch tool-server "connect failed" error is intentionally left open (fix at source vs. re-present as non-fatal) — captured in Assumptions and constrained by FR-008. +- Items marked incomplete require spec updates before `/speckit-clarify` or `/speckit-plan` diff --git a/specs/001-stabilize-agent-startup/contracts/agent-api.md b/specs/001-stabilize-agent-startup/contracts/agent-api.md new file mode 100644 index 00000000..c898f931 --- /dev/null +++ b/specs/001-stabilize-agent-startup/contracts/agent-api.md @@ -0,0 +1,66 @@ +# API Contracts: Stabilize Agent Startup Status & Logs + +**Date**: 2026-07-30 · Applies to `api/` (NestJS, OpenAPI-generated admin SDK). All changes are **additive** — no existing field changes shape or meaning. + +## 1. Agent resource — new response fields + +Affected endpoints: `GET /agents`, `GET /agents/:id`, and any response embedding the agent record (including `AgentStatusDto.agent` on `GET /agents/status` and SSE `GET /agents/status/stream`). + +```jsonc +{ + "id": "agent-…", + "status": "deploying", // unchanged enum: pending|deploying|running|failed|stopped + "launchContext": "initial", // NEW: "initial" | "restart" | null — why the current/last deploy ran + "statusReason": null, // NEW: string | null — human-readable reason, non-null only when status="failed" + "firstDeployedAt": null, // NEW: ISO datetime | null — null ⇒ agent has never been deployed + // …all existing fields unchanged +} +``` + +Guarantees: + +- `launchContext = "initial"` on every response while (and after) an agent's **first** deploy is in flight; `"restart"` for every subsequent deploy, whatever triggered it (manual restart, start after stop, config-change redeploy, admin promote/demote). +- `statusReason` is non-null **only when** `status = "failed"`; it is cleared on every transition out of `failed` and on every new deploy. It may still be null for a `failed` recorded before the field existed (legacy rows are not backfilled) — consumers must tolerate `failed` with a null reason. +- Swagger/DTO updated so the generated admin SDK (`admin/slices/setup/api/…`) exposes the three fields with correct types (agent payload must not remain `{[key:string]: unknown}` for these fields). + +## 2. Lifecycle endpoints — status-sequence guarantees + +Endpoints unchanged in shape: `POST /agents` (create+deploy), `POST /agents/:id/start`, `POST /agents/:id/stop`, `POST /agents/:id/restart`, `POST /agents/restart-by-template/:templateId`. + +Behavioural contract (what a poller at any frequency may observe, FR-001/FR-002/FR-010): + +| Operation | Permitted status sequence | Forbidden | +|-----------|--------------------------|-----------| +| create → healthy | `pending → deploying → running` | any `failed` | +| start (from `stopped`) → healthy | `stopped → deploying → running` | any `failed`, any stale earlier status | +| restart (from `running`/`failed`) → healthy | ` → deploying → running` | `failed` appearing *after* `deploying` began | +| stop | ` → stopped` | — | +| any → genuine failure | `… → deploying → failed` (with `statusReason`) | `failed` before a definitive signal or before the 5-minute timeout | + +Definitive-failure triggers and their reasons: see [data-model.md](../data-model.md) state-machine table. `failed` MUST be written within 60 s of a definitive signal (SC-006) and MUST NOT be written inside the 5-minute grace window by the *absence-of-pod* heuristic alone. + +`POST /agents/:id/restart` additionally guarantees: the agent record never references the cancelled old workflow after the restart request is accepted (`workflowId` is cleared before cancellation — closes the stale-workflow `failed` race). + +MCP tool `restart_agent` (rancher toolset) performs the same operation as `POST /agents/:id/restart` — never a bare status write. + +## 3. Logs endpoint — marker contract + +`GET /agents/:agentId/logs?tail=N` → `{ "logs": string }`, where `logs` is either real log text or exactly one marker line: + +| Marker (exact format) | When | +|---|---| +| `[no pod yet for agent]` | Pod `agent-` does not exist (k8s 404) | +| `[container ]` | Pod exists but container is waiting; `` lowercased, e.g. `containercreating`, `podinitializing` | +| `[log fetch failed: ]` | Any other fetch error; `` MUST be a single-line, parsed error message — never a raw serialized HTTP response (no `HTTP-Code:`/`Body:`/`Headers:` dumps) | + +The `[container …]` marker MUST be returned for k8s 400 "waiting to start" responses regardless of whether the k8s client delivers the error body as an object or a string (closes RC-3). + +## 4. Status stream — unchanged, referenced + +SSE `GET /agents/status/stream` and `GET /agents/status` keep their shape (`AgentStatusDto { agent, pod | null }`; pod: `phase`, `ready`, `restartCount`, `startedAt`, `lastTerminationReason`, `containerWaitingReason`, `message`, `observedAt`). The embedded `agent` record carries the new fields from §1. `ContainerCreating`/`PodInitializing` in `containerWaitingReason` remain non-failure signals. + +## 5. MCP server bootstrap (internal contract) + +- Seeded default for the built-in CleanSlice server: `https://mcp.cleanslice.org/mcp` (env `CLEANSLICE_MCP_URL` overrides). +- On every API bootstrap the seeder converges the **existing** built-in row's `url` to the configured value (idempotent heal); built-in rows remain non-editable/non-deletable via the public API. +- Consequence for agents: `MCP_SERVERS_B64` baked at deploy time contains the healed URL from the next deploy onward; a successfully started agent registers > 0 tools from CleanSlice and its startup log contains no connect-failure line (FR-012, SC-005). diff --git a/specs/001-stabilize-agent-startup/data-model.md b/specs/001-stabilize-agent-startup/data-model.md new file mode 100644 index 00000000..5910c38d --- /dev/null +++ b/specs/001-stabilize-agent-startup/data-model.md @@ -0,0 +1,78 @@ +# Data Model: Stabilize Agent Startup Status & Logs + +**Date**: 2026-07-30 · **Plan**: [plan.md](./plan.md) · **Research**: [research.md](./research.md) + +## Agent (existing Prisma model — additive changes only) + +Source: `api/src/slices/agent/agent/agent.prisma` (composed into `api/prisma/schema.prisma`; one new additive migration). + +| Field | Type | New? | Semantics | +|-------|------|------|-----------| +| `status` | `String @default("pending")` | no | Unchanged vocabulary: `pending` \| `deploying` \| `running` \| `failed` \| `stopped` (clarification #1) | +| `workflowId` | `String?` | no | Now **cleared before** cancelling the old workflow in `restartAgent()` (D2); still cleared on stop | +| `firstDeployedAt` | `DateTime?` | **yes** | Set exactly once — on the agent's first successful workflow submit. Never updated afterwards. Null ⇒ the agent has never been deployed. A follow-up migration backfills `updatedAt` for pre-existing non-`pending` agents so their next deploy reads as `restart`, not a first launch | +| `lastDeployStartedAt` | `DateTime?` | **yes** | Set by `deploy()` together with the `deploying` status write. Anchor of the 5-minute grace/timeout window (D1). When null (legacy rows deployed before the migration), the grace predicate falls back to `updatedAt` so a mid-rollout deploy is not instantly drift-failed | +| `lastLaunchContext` | `String?` | **yes** | `'initial'` \| `'restart'`. Written by `deploy()`: `initial` iff `firstDeployedAt` is null at call time, else `restart`. Null only for legacy rows never deployed since migration | +| `statusReason` | `String?` | **yes** | Human-readable reason accompanying `failed` (FR-009). Set by every `failed` writer; cleared (null) on any transition to `deploying`, `running`, or `stopped` | + +**Write discipline**: `AgentGateway.updateStatus()` remains the single DB writer for `status` and is extended to atomically accept `statusReason` (and the deploy-time fields where relevant), preserving the existing last-writer-wins reasoning in `agentDeploy.service.ts`. + +## Status state machine (target behaviour) + +States are the existing five values; what changes is *who may write `failed` and when*. + +```text +pending ──deploy()──▶ deploying ──pod ready / bridle connect──▶ running + ▲ │ ▲ │ + │ │ └── start/restart (deploy()) │ + │ │ │ + (row created) ├──stop()──▶ stopped ──start()──▶ deploying + │ ▲ │ + │ └─────────stop()─────────┘ + │ + └──definitive failure──▶ failed ──start/restart──▶ deploying + │ + └──late pod ready / bridle connect──▶ running (self-heal, kept) +``` + +**Definitive failure** (the only permitted `deploying → failed` / `running → failed` triggers, FR-002): + +| Trigger | Latency | `statusReason` | +|---------|---------|----------------| +| Workflow submit throws (`agentDeploy.service.ts`) | immediate | generic `workflow submit failed` — raw submit errors can carry internal detail (Argo endpoints) and `statusReason` is served on public endpoints; the full message goes to the server log | +| Template missing at deploy | immediate | "template not found …" | +| Pod waiting reason ∈ `FAIL_WAITING_REASONS` (`CrashLoopBackOff`, `ImagePullBackOff`, `ErrImgPull`, `CreateContainerConfigError`, `CreateContainerError`) | ≤ pod-event latency | waiting reason + pod message | +| Pod phase `Failed` | ≤ pod-event latency | termination reason/message | +| Workflow phase `Failed`/`Error` via `syncStatus` — can only be the **current** workflow, since restart detaches the old id before cancelling and stop clears it (D2); definitive immediately, no time guard. Skipped when the runtime is live on the bridle hub (bridle-truth wins, same rule as the drift sweep) — a lying workflow record must not ping-pong a healthy agent to `failed` | ≤ next poll | `deploy workflow failed/error` | +| Drift sweep: no pod **and** `now − lastDeployStartedAt > 5 min` (D1) | ≤ 30 s after window expiry | "startup did not produce a running agent within 5 minutes" | + +**Forbidden writers removed**: drift no-pod branch inside the grace window (RC-1 primary); `syncStatus` reading the cancelled old workflow (RC-1 secondary — eliminated by clearing `workflowId` first); `restart_agent` MCP tool's bare `deploying` write (now performs a real restart); bridle-truth resurrect paths flipping an explicitly **stopped** agent back to `running` (the old runtime's WS lingers after pod delete — indefinitely on local dev — and the resurrect both undid the operator's stop and set up a later pod-less-`running` → `failed` decay); `syncStatus` writing `failed` while the runtime is live on the bridle hub. + +`ContainerCreating` / `PodInitializing` remain explicitly non-failure signals at every layer. + +## Launch context (derived, server-authoritative) + +`launchContext` on agent responses = `lastLaunchContext` (`'initial' | 'restart' | null`). Consumed by the admin UI to select copy while `status ∈ {pending, deploying}`: + +| `launchContext` | Overlay title | Log-panel placeholder | +|---|---|---| +| `initial` | "Setting up the agent…" (first start) | "First start — logs will appear when the agent is up." | +| `restart` / null | "Restarting agent…" | "Agent is restarting — logs will resume when the new pod is up." | + +(Exact wording finalized in implementation; the contract is: the two flows MUST render visibly different copy, FR-003/SC-003.) + +## Log stream markers (API → admin contract) + +`GET /agents/:agentId/logs` returns either real log text or exactly one of these single-line markers (existing contract, now reliable per D4): + +| Marker | Meaning | Admin rendering | +|--------|---------|-----------------| +| `[no pod yet for agent]` | Pod does not exist yet | "No pod yet — agent is ." placeholder | +| `[container ]` | Pod exists, container waiting (e.g. `containercreating`) | Spinner + "Container creating…" | +| `[log fetch failed: ]` | Genuinely unexpected fetch error | Italic placeholder with the *parsed one-line* message — never a raw multi-line HTTP dump | + +## Client-side state (admin, no persistence changes) + +- `IAgentData` (`admin/.../domain/agent.types.ts`) gains `launchContext`, `statusReason`, `firstDeployedAt` (nullable); mapper fills them; unknown `launchContext` strings coerce to null. +- `useAgentLifecycle`: poll ticks are skipped while a lifecycle mutation is awaiting its HTTP response (D6); restart-in-flight localStorage flag semantics unchanged. +- Log classifier (`utils/agentLogs.ts`): level determined by the line's explicit level token when present; body-substring matching only as fallback. diff --git a/specs/001-stabilize-agent-startup/plan.md b/specs/001-stabilize-agent-startup/plan.md new file mode 100644 index 00000000..2de17d10 --- /dev/null +++ b/specs/001-stabilize-agent-startup/plan.md @@ -0,0 +1,94 @@ +# Implementation Plan: Stabilize Agent Startup Status & Logs + +**Branch**: `fix/agent-deploy` | **Date**: 2026-07-30 | **Spec**: [spec.md](./spec.md) + +**Input**: Feature specification from `/specs/001-stabilize-agent-startup/spec.md` + +## Summary + +During agent start/restart the platform shows misleading signals: a transient `failed` status (~30 s) during every stop → start, a single `deploying` status that hides whether the agent is starting for the first time or restarting, a raw Kubernetes 400 error dump in the log panel while the container is being created, and a red MCP "connect failed" error on every first launch. Root causes are all identified (see [research.md](./research.md)): + +1. The 30-second drift sweep marks any pod-less `deploying` agent as `failed` with no grace period for a just-submitted deploy; a secondary race lets `GET /agents/:id` read the phase of the just-cancelled old workflow. +2. No persisted "has ever run" marker exists, so first deploy and restart are structurally indistinguishable. +3. The friendly `[container containercreating]` branch in the log controller never fires because `@kubernetes/client-node` 1.4 throws `ApiException` with a *string* body, while the code expects an object. +4. The seeded CleanSlice MCP URL lacks the `/mcp` path segment; the seeder is create-only, so existing rows keep the broken URL. + +Approach: fix status truthfulness server-side (deploy grace window backed by a persisted timestamp, 5-minute definitive-failure timeout, workflow-sync guard), add a server-provided launch context (`initial` / `restart`) plus a `statusReason` for failures, parse the k8s client's string error bodies so the friendly log markers fire, heal the CleanSlice MCP URL (default + bootstrap upsert), and update the admin UI to consume the new fields (distinct first-start vs restart copy, poll-race guard, log level classification fix). No changes to the agent runtime repo are needed. + +## Technical Context + +**Language/Version**: TypeScript 5.x on Bun 1.2 workspaces (monorepo via turbo); Node runtime in containers + +**Primary Dependencies**: `api/` — NestJS 11, Prisma 6, `@kubernetes/client-node` 1.4.0, Argo Workflows (namespace `agents`, submitted via `argo-workflow.gateway.ts`), Socket.IO (bridle chat hub), SSE status stream; `admin/` — Nuxt 3.16, Pinia, generated SDK via `openapi-ts` (`@hey-api/client-axios`), shadcn-nuxt + +**Storage**: PostgreSQL via Prisma (`Agent.status` is a plain string column, default `'pending'`); Prisma migration required for new columns + +**Testing**: `api/` — Jest (`bun run test`, currently `--passWithNoTests`); `admin/` — no test runner configured; pure `.ts` changes type-checked via `tsc` (no vue-tsc in repo) + +**Target Platform**: Kubernetes cluster (k3d locally, managed cluster in prod); admin web UI; agent pods created by a two-step Argo workflow (`cleanup-old` → `run-agent`), pod name deterministic `agent-` + +**Project Type**: Web application (NestJS API + Nuxt admin) in a monorepo + +**Performance Goals**: Status visible as `running` within 10 s of readiness (SC-002; admin polls every 5 s during launches, SSE pod stream is push); drift sweep cadence stays 30 s + +**Constraints**: Status vocabulary (`pending`/`deploying`/`running`/`failed`/`stopped`) must not change (clarification 2026-07-30); launch context must be server-derived and survive page reload (FR-003/FR-005); definitive failure = explicit runtime signals OR 5-minute safety timeout (FR-002); real failures must not be suppressed or delayed (FR-009, SC-006); built-in MCP row heal must be idempotent (seeder runs on every API boot); prod deploys happen only on `v*` tag runs + +**Scale/Scope**: Tens of agents per cluster; one admin page per agent plus a list page; 3 API slices touched (`agent`, `log`, `mcpServer` + `rancher` tool), 1 admin slice (`agent`), 1 Prisma migration, SDK regeneration + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +`.specify/memory/constitution.md` is an unfilled template — no project-specific principles or gates are ratified. No violations possible; gate **PASS** (pre-Phase-0 and re-checked post-Phase-1). General engineering defaults apply: smallest change that satisfies the spec, no status-enum breakage, idempotent bootstrap operations, migrations additive-only. + +## Project Structure + +### Documentation (this feature) + +```text +specs/001-stabilize-agent-startup/ +├── plan.md # This file +├── research.md # Phase 0 output — root causes + decisions D1–D6 +├── data-model.md # Phase 1 output — Agent columns, status state machine +├── quickstart.md # Phase 1 output — end-to-end validation scenarios +├── contracts/ +│ └── agent-api.md # Phase 1 output — DTO/endpooint/log-marker contracts +└── tasks.md # Phase 2 output (/speckit-tasks — NOT created by /speckit-plan) +``` + +### Source Code (repository root) + +```text +api/src/slices/ +├── agent/agent/ +│ ├── agent.prisma # + firstDeployedAt, lastDeployStartedAt, lastLaunchContext, statusReason +│ ├── agent.controller.ts # syncStatus: skip failure-writes inside deploy grace window +│ ├── domain/ +│ │ ├── agent.types.ts # + LaunchContext type +│ │ ├── agentDeploy.service.ts # write launch context + deploy timestamp; clear stale workflowId before cancel; statusReason on failure +│ │ └── agentStatus.service.ts # drift no-pod branch: 5-min grace from lastDeployStartedAt; statusReason on timeout/pod failure +│ ├── data/agent.gateway.ts # updateStatus extended for new fields +│ └── dtos/ # agent response fields: launchContext, statusReason, firstDeployedAt +├── log/log.controller.ts # parse ApiException string body → friendly [container …] marker fires +├── rancher/rancher.tool.ts # restart_agent: actually call restartAgent() instead of bare status write +└── mcpServer/domain/mcpServer.seeder.ts # default URL …/mcp + idempotent heal of existing built-in row + +api/prisma/ # generated schema + new migration + +admin/slices/agent/agent/ +├── composables/useAgentLifecycle.ts # pause poll while mutation in flight; failed handling aligned with definitive-failure semantics +├── components/agent/chat/Tab.vue # overlay copy: first start vs restart (launchContext-driven) +├── components/agent/logs/Panel.vue # placeholder copy per launch context +├── utils/agentFormat.ts # status labels incl. reason surfacing +├── utils/agentLogs.ts # ERROR classifier: prefer explicit level token over substring match +├── data/agent.mapper.ts # map launchContext / statusReason / firstDeployedAt +└── domain/agent.types.ts # extend IAgentData + +admin/slices/setup/api/ # regenerated SDK (openapi-ts) after API DTO changes +``` + +**Structure Decision**: Existing monorepo layout is kept; all changes land in the `api` and `admin` workspaces listed above. The agent runtime lives in a separate repository and needs no changes (research confirmed the MCP misconfiguration is platform-side). + +## Complexity Tracking + +No constitution violations — table not required. diff --git a/specs/001-stabilize-agent-startup/quickstart.md b/specs/001-stabilize-agent-startup/quickstart.md new file mode 100644 index 00000000..cbe5b47e --- /dev/null +++ b/specs/001-stabilize-agent-startup/quickstart.md @@ -0,0 +1,72 @@ +# Quickstart Validation: Stabilize Agent Startup Status & Logs + +**Date**: 2026-07-30 · **Contracts**: [contracts/agent-api.md](./contracts/agent-api.md) · **Success criteria**: [spec.md](./spec.md) §Success Criteria + +## Prerequisites + +- Local stack: `bun install`, then `cd api && bun run dev` (starts docker deps via `predev`, runs migrations — the new migration must apply cleanly to an existing dev DB) and `cd admin && bun run dev` (regenerates the SDK from swagger via `predev`; verify `launchContext`/`statusReason`/`firstDeployedAt` appear in `admin/slices/setup/api/data/repositories/api/types.gen.ts`). +- Full lifecycle scenarios need a Kubernetes + Argo environment (`k8s/local/` k3d setup) with `infrastructure.workflow_provider = argo`; the mock provider does not reproduce the pod-less window. +- Checks: `cd api && bun run test` (Jest); pure-TS admin changes via `tsc` (repo has no vue-tsc). + +## S1 — Stop → start shows no transient `failed` (P1; FR-001/002/010, SC-001) + +1. Take a `running` agent. Stop it (`POST /agents/:id/stop` or the admin Stop button) → status becomes `stopped`. +2. Start it and poll faster than the UI does: + ```bash + while true; do curl -s $API/agents/$ID -H "$AUTH" | jq -r '[.status, .launchContext, .statusReason] | @tsv'; sleep 2; done + ``` +3. **Expected**: sequence is `stopped → deploying → running` only; `failed` never appears (watch ≥ 2 drift cycles, i.e. > 60 s); `running` visible ≤ 10 s after pod ready (SC-002); `statusReason` stays null. Repeat for `POST /agents/:id/restart` from `running` — same guarantee. In the admin, the badge never flashes red and polling never stops mid-launch. +4. Page-reload check (FR-005): reload the agent page mid-deploy — overlay still shows the in-progress state. + +## S2 — Restart with the log panel open: no raw 400 dump (P2; FR-006/007, SC-004) + +1. Open the agent page with the Logs panel visible. Restart the agent. +2. **Expected**: during the pod-less window the panel shows the friendly placeholder; while the container is created it shows the spinner state ("Container creating…") — never a `[log fetch failed: HTTP-Code: 400 …]` dump. Confirm at the API layer too: + ```bash + curl -s $API/agents/$ID/logs?tail=100 -H "$AUTH" | jq -r .logs + # during ContainerCreating must print: [container containercreating] + ``` +3. Logs resume automatically once the new pod is up — no manual reload (FR-007). + +## S3 — First deploy: distinct copy + clean startup log (P2/P3; FR-003/008/012, SC-003/005) + +1. Create a brand-new agent (`POST /agents` / admin create flow). +2. **Expected**: + - Response has `launchContext: "initial"`, `firstDeployedAt` set after submit; the overlay/log placeholder uses the first-start wording, NOT "Cancelling old workflow…" / "Agent is restarting…" (SC-003 — visibly different from S1's restart copy). + - Startup log: `mcp connecting to 1 server(s): CleanSlice` followed by `total N tools registered` with **N > 0**; no `connect failed` line; no line styled as ERROR in the admin panel (SC-005). +3. Restart the same agent → `launchContext: "restart"` and restart wording. + +## S4 — Genuine failure still surfaces, with a reason (FR-009, SC-006) + +1. Force a real failure: point the agent template at a nonexistent image (or otherwise trigger `ImagePullBackOff`). +2. **Expected**: status reaches `failed` within 60 s of the signal, `statusReason` names the cause (e.g. image pull), the admin shows the failed overlay with the reason. No suppression or delay compared to today. + +## S5 — 5-minute safety timeout (FR-002 clause b) + +1. Make the pod unschedulable (e.g. impossible resource requests) and deploy. +2. **Expected**: status stays `deploying` for the full 5 minutes (no `failed` from the drift sweep during the window), then flips to `failed` with `statusReason` ≈ "startup did not produce a running agent within 5 minutes" within the next drift cycle (≤ ~30 s after expiry). + +## S6 — CleanSlice MCP row healed on boot (FR-012) + +1. With a DB whose built-in CleanSlice row still holds the bare origin URL, boot the API. +2. **Expected**: + ```bash + curl -s $API/mcp-servers -H "$AUTH" | jq '.[] | select(.builtIn) | .url' + # → "https://mcp.cleanslice.org/mcp" + curl -s -X POST https://mcp.cleanslice.org/mcp -H 'content-type: application/json' \ + -H 'accept: application/json, text/event-stream' \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"probe","version":"0"}}}' + # → 200 (serverInfo: cleanslice-mcp) + ``` + Re-boot the API → row unchanged (heal is idempotent). S3 then proves agents pick it up on deploy. + +## S7 — Regression sweep + +- `restart_agent` via the rancher MCP tool actually restarts (pod is replaced; agent does not decay to `failed` after 30 s). +- Dev-only (mock workflow provider): restart the API while an agent's runtime stays connected → the agent must NOT flip to `failed` ("deploy workflow failed") on the next `GET /agents/:id`, and must not ping-pong `failed`↔`running` (bridle-truth guard + mock getStatus throwing for forgotten workflows). +- Stop an agent whose runtime WS is still connected (always the case on local dev) → status stays `stopped` (no bridle-truth resurrect to `running`, no later decay to `failed`), and the chat shows the "Agent stopped" overlay even though the socket is technically alive. +- Opening the agent page issues exactly ONE `GET /agents/:id` (the chat widget is seeded via `initial-debug-enabled` instead of fetching its own copy). +- Restart while `deploying` (restart requested mid-deploy) → the display stays in a single coherent startup state, the grace window re-anchors, and the launch ends `running` — no oscillation, no `failed`. +- Stop while `deploying` → clean `stopped`, no later `failed` from leftover sweeps. +- Rapid stop → start → stop → start remains coherent (FR-010 edge case). +- 20× S1 cycles for SC-001/SC-004 sign-off before release (deploys ship only on `v*` tags). diff --git a/specs/001-stabilize-agent-startup/research.md b/specs/001-stabilize-agent-startup/research.md new file mode 100644 index 00000000..16d2658e --- /dev/null +++ b/specs/001-stabilize-agent-startup/research.md @@ -0,0 +1,76 @@ +# Phase 0 Research: Stabilize Agent Startup Status & Logs + +**Date**: 2026-07-30 · **Spec**: [spec.md](./spec.md) + +Three parallel codebase investigations (API lifecycle, admin UI, MCP integration) resolved every unknown. No `NEEDS CLARIFICATION` items remain. Repo state at research time: branch `fix/agent-deploy`, byte-identical to `main`. + +## Established facts (root causes) + +### RC-1 — Transient `failed` after stop → start (FR-011) + +Two independent server-side writers produce it; the first matches the ~30 s symptom exactly. + +- **Primary**: the periodic drift sweep (`api/src/slices/agent/agent/domain/agentStatus.service.ts:283-296`, every 30 s) marks any agent whose DB status is "live" (`pending`/`deploying`/`running`) but has no pod as `failed`. After a deploy is submitted, the two-step Argo workflow (`cleanup-old` → `run-agent`, `api/src/slices/workflow/data/agent-workflow.manifest.ts:102-137`) leaves a legitimate pod-less window of ~10–30 s. The no-pod branch consults neither `DeployTracker` nor any grace period. Nothing corrects `failed` back to `deploying`; only pod-ready/bridle-connect flips it to `running` — the observed ~30 s. +- **Secondary (restart only)**: `syncStatus` inside `GET /agents/:id` (`agent.controller.ts:90-117`) reads the *just-cancelled old* workflow in the 1–3 s window between `cancelAgentWorkflow` and `setWorkflowId(new)` (`agentDeploy.service.ts:57→137`); phase `Failed` → writes `failed`. Inert for stop → start because stop nulls `workflowId`. +- **Aggravator (client)**: once a 5 s poll returns `failed`, the admin clears its restart-in-flight flag and stops polling (`admin/.../useAgentLifecycle.ts:261-267`, `failed ∉ POLL_STATUSES`) — the badge sticks on `failed` until something else refetches. +- **Related dead-end**: MCP tool `restart_agent` (`api/src/slices/rancher/rancher.tool.ts:96-110`) writes `deploying` without deploying anything — such an agent is *guaranteed* to be drift-marked `failed` 30 s later. + +### RC-2 — First start indistinguishable from restart (FR-003) + +The `Agent` model (`agent.prisma:8-31`) has no deploy-history field of any kind (verified: no `deployedAt`/`firstDeploy`/counter anywhere in schema or migrations). Both flows funnel into the same `deploy()` writing the same `deploying`. The admin has no differing code path either — the overlay always says "Cancelling old workflow and submitting a fresh one." even on a first deploy (`useAgentLifecycle.ts:197`). + +### RC-3 — Raw Kubernetes 400 in the log panel (FR-006) + +`log.controller.ts` already has a friendly `[container containercreating]` branch (line 115), but it never fires: `extractWaitingReason` (133–145) reads `err.body?.message` as an object property, while `@kubernetes/client-node` 1.4.0 throws `ApiException` whose `body` is a **raw JSON string** (`getBodyAsAny()` returns `body.text()` for this endpoint). Control falls through to `extractKubeError` → returns `e.message` (the multi-line `HTTP-Code: 400 …` dump) → embedded as `[log fetch failed: …]` (line 119) → rendered near-verbatim by the admin (`useAgentLogs.ts:39-41`). The 404 branch works only because it checks `e.code`, not the body. Same string-body assumption exists in `pod.gateway.ts:325-335` (log-noise only). + +### RC-4 — MCP "CleanSlice connect failed … 404" on launch (FR-012) + +- The agent runtime (separate repo, `ghcr.io/cleanslice/runtime`) connects to exactly the URL it is given — `new StreamableHTTPClientTransport(new URL(cfg.url))`, no path appended. The server list is fully baked at deploy time into the `MCP_SERVERS_B64` env var by the ranch API (`argo-workflow.gateway.ts:89-126`), which force-attaches the built-in CleanSlice server from the DB and passes its `url` verbatim. +- The DB row is seeded with `https://mcp.cleanslice.org/` (`mcpServer.seeder.ts:61`), but the real Streamable HTTP endpoint is `https://mcp.cleanslice.org/mcp` — verified live: `POST /` → 404 `Cannot POST /` (byte-for-byte the logged error), `POST /mcp` with an MCP `initialize` → 200. +- The seeder is **create-only** (`if (!existingCleanslice)`), the URL is not editable via API for built-ins, and DELETE is forbidden — so existing deployments keep the broken URL even after the default is fixed. Agents pick up a healed URL only on their next deploy (env is baked). +- The runtime logs the failure at **warn** and continues (0 tools, no retry until next pod start). The red "ERROR" styling comes from the admin classifier `agentLogs.ts:32-48`, whose `ERROR_TOKEN_RE` matches the substring `"error":"Not Found"` *inside the JSON body* of the message. +- **Conclusion: no runtime-repo changes required.** The fix is entirely platform-side. + +## Decisions + +### D1 — Deploy grace window, persisted (fixes RC-1 primary) + +- **Decision**: Add `lastDeployStartedAt` (DB column, set by `deploy()` together with the `deploying` write). The drift sweep's no-pod branch skips agents whose `lastDeployStartedAt` is within **5 minutes** (the spec's safety timeout). When the window expires with no pod, it marks `failed` with `statusReason` = startup timeout. Explicit failure signals (`FAIL_WAITING_REASONS`, workflow-submit throw, pod phase `Failed`) keep firing immediately — the grace window applies only to the *absence-of-pod* heuristic, so real failures are not delayed (FR-009). +- **Rationale**: Persisting the timestamp makes the grace window survive API restarts (the in-memory `DeployTracker` alone would not) and gives the 5-minute definitive-failure timeout (clarification #2) a single source of truth. 5 min ≫ the legitimate 10–30 s pod-less window, and matches the spec exactly. +- **Alternatives considered**: (a) extend in-memory `DeployTracker` to cover the no-pod branch — rejected: lost on API restart, exactly when drift sweeps fire in bulk; (b) count consecutive pod-less sweeps before failing — rejected: implicit timing, harder to reason about and test than an explicit timestamp. + +### D2 — Workflow-sync guard (fixes RC-1 secondary) + +- **Decision**: Two cheap complementary fixes: (1) `restartAgent()` clears `workflowId` (null) *before* cancelling the old workflow, so a concurrent `GET /agents/:id` cannot resolve the doomed workflow; (2) `syncStatus` additionally refuses to write `failed` while the agent is inside the D1 grace window (defence in depth; also covers future callers). +- **Rationale**: Eliminates the race at its source rather than masking it; the guard reuses the same grace predicate as D1 — one concept, two writers. +- **Alternatives considered**: skip `syncStatus` entirely during `deploying` — rejected: it would also skip legitimate new-workflow failure detection later in a long deploy. + +### D3 — Launch context + first-run marker (fixes RC-2) + +- **Decision**: Add `firstDeployedAt` (set once, on first successful workflow submit) and `lastLaunchContext` (`'initial' | 'restart'`, written by `deploy()`: `initial` when `firstDeployedAt` is still null at call time, else `restart`). Expose `launchContext`, `statusReason`, and `firstDeployedAt` on agent responses; admin maps them and switches overlay/log-placeholder copy ("Setting up the agent for the first time…" vs "Restarting agent…"). Status vocabulary unchanged, per clarification #1 (option B). Config-change redeploys go through restart flows and correctly read as `restart` — the spec groups restart/update together. +- **Rationale**: Server-derived, survives reload (FR-005), no status-enum breakage, two nullable columns + DTO fields — minimal surface. `workflowId` was verified unusable as a proxy (cleared on stop; Argo workflow GC'd after 1 h). +- **Alternatives considered**: (a) new statuses `starting`/`restarting` — rejected by clarification #1; (b) client-side inference from the action the user clicked — rejected: dies on reload, violates FR-005. + +### D4 — Parse string error bodies in k8s error handling (fixes RC-3) + +- **Decision**: In `log.controller.ts`, normalize `ApiException.body`: if it is a string, `JSON.parse` it (try/catch) before reading `.message`; fall back to matching the waiting-reason regex against `e.message` too. With that, the existing `[container containercreating]` branch fires and the admin's existing spinner UI ("Container creating…") takes over — no new UI needed for this path. Apply the same normalization to `pod.gateway.ts`'s `extractKubeError`. `[log fetch failed: …]` remains only for genuinely unexpected errors, and its payload becomes the parsed one-line message, never the multi-line dump. +- **Rationale**: The friendly UX already exists on both sides of the contract; only the error-shape assumption is wrong. Smallest possible fix, no client change required for the marker path. +- **Alternatives considered**: upgrading `@kubernetes/client-node` — rejected for this feature: unrelated blast radius across every k8s call site; the string-body normalization is needed anyway for robustness. + +### D5 — CleanSlice MCP URL: fix default + heal existing rows (fixes RC-4) + +- **Decision**: (1) Change the seeded default to `https://mcp.cleanslice.org/mcp` (env override `CLEANSLICE_MCP_URL` still wins). (2) Make the seeder heal an *existing* built-in CleanSlice row on bootstrap: if its URL is the known-bad bare origin (or differs from the configured value for the built-in row), update it — idempotent, aligned with the "api owns this entry" comment in the seeder. Agents pick up the fix on their next deploy/restart, which this feature's validation covers. +- **Rationale**: A default-only fix would never repair the live deployment (create-only seeder, URL immutable via API for built-ins). Bootstrap healing fixes fresh *and* existing installs without a one-off migration script to operate. +- **Alternatives considered**: (a) one-off SQL data migration — rejected: the seeder is the declared owner of built-in rows and runs everywhere the API boots; (b) making the runtime append `/mcp` — rejected: wrong layer, runtime is a separate repo, and other MCP servers may legitimately live at other paths. + +### D6 — Admin: poll-race guard, failed handling, log level classification + +- **Decision**: (1) `useAgentLifecycle` skips its 5 s poll ticks while a lifecycle mutation (start/stop/restart request) is in flight, so a stale pre-restart status cannot overwrite the optimistic `deploying`. (2) The "clear restart-in-flight on `failed`" behaviour stays — after D1/D2 any `failed` during startup is definitive by construction. (3) `agentLogs.ts` classification prefers an explicit level token from the runtime log line over substring matches, so a warn-level line whose *body* contains `"error":"Not Found"` is no longer styled as ERROR (FR-008: non-fatal presented as non-fatal). Verify the runtime's exact line format against real pod logs during implementation before tightening the regex. +- **Rationale**: Server fixes remove the *source* of lies; these client fixes remove the remaining *amplifiers* (race, poison-pill styling). All are small, local edits to already-identified lines. +- **Alternatives considered**: request-sequence tokens on every poll — rejected as over-engineering once the mutation-in-flight guard exists; suppressing all error styling during startup — rejected: would hide real startup errors (violates FR-009). + +## Cross-cutting notes + +- **SDK regeneration**: admin consumes the API via generated `openapi-ts` SDK; after DTO changes run `bun run build:api` in `admin/` (requires the API's swagger). The agent gateway's raw-axios workaround for stop/start is outdated (SDK now has those calls) — may be cleaned up opportunistically but is not required by this feature. +- **Out of scope confirmed**: agent runtime repo (no changes needed), startup speed, Argo workflow topology, `@kubernetes/client-node` upgrade. +- **Release note**: prod deploys only happen on `v*` tag runs; the MCP heal activates on API boot, agents pick it up on next restart. diff --git a/specs/001-stabilize-agent-startup/spec.md b/specs/001-stabilize-agent-startup/spec.md new file mode 100644 index 00000000..252b290b --- /dev/null +++ b/specs/001-stabilize-agent-startup/spec.md @@ -0,0 +1,134 @@ +# Feature Specification: Stabilize Agent Startup Status & Logs + +**Feature Branch**: `001-stabilize-agent-startup` + +**Created**: 2026-07-30 + +**Status**: Draft + +**Input**: User description: "Стабилизировать запуск агента: при старте/рестарте статус ведёт себя обманчиво — после stop → start агент сначала показывает status: failed и только через ~30 секунд обновляется до running; для первого запуска и рестарта используется один статус deploying, из-за чего складывается впечатление, что агент обновляется, а не запускается впервые; во время рестарта в панели логов какое-то время отображается сырая ошибка инфраструктуры (log fetch failed: HTTP 400, container is waiting to start: ContainerCreating); при успешном первом запуске в логах видна красная ошибка (подключение к серверу инструментов: connect failed, 404), хотя всё работает." + +## Clarifications + +### Session 2026-07-30 + +- Q: Где должна храниться «правда» о том, что агент запускается впервые, а не рестартует, чтобы интерфейс различал их даже после перезагрузки страницы? → A: Вариант B — существующий набор статусов не меняется; сервер дополнительно сообщает контекст запуска (первый запуск / рестарт / обновление) отдельным признаком. Дополнительно зафиксировано: причина транзитного `failed` после stop → start должна быть диагностирована и устранена в источнике, а не скрыта на стороне интерфейса. +- Q: По какому критерию запуск агента считается «окончательно неудавшимся», после чего можно показывать статус ошибки? → A: Вариант B — явные сигналы среды выполнения (циклические падения, невозможность создать/запустить среду) **плюс** страховочный таймаут 5 минут, после которого запуск помечается как проблемный с понятным объяснением. +- Q: Что делать с ошибкой подключения к серверу инструментов («connect failed … 404») при первом запуске — чинить причину или только подачу в логах? → A: Вариант A — диагностировать и устранить причину в рамках этой фичи (инструменты реально не подключаются: «total 0 tools registered»); правило FR-008 о подаче некритичных ошибок при этом сохраняется. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Status never lies during start/restart (Priority: P1) + +An operator stops an agent and starts it again (or restarts a running agent). From the moment the action is confirmed until the agent is ready, the displayed status continuously shows that a startup is in progress. It never flashes a failure state while the agent is actually coming up, and it switches to "running" promptly once the agent is ready. Today the agent shows `failed` for roughly half a minute after a start before flipping to `running`, which makes a healthy launch look broken. + +**Why this priority**: A status that reads "failed" during every normal start destroys trust in the status display entirely — operators can no longer tell real failures from healthy launches. This is the core confusion the feature exists to remove. + +**Independent Test**: Can be fully tested by performing stop → start and restart cycles on a test agent while watching the status display, and confirming no failure state ever appears during a launch that ends in "running". + +**Acceptance Scenarios**: + +1. **Given** a stopped agent, **When** the operator starts it, **Then** the status shows a startup-in-progress state continuously until the agent is ready, and at no point shows a failure state. +2. **Given** an agent whose startup has completed, **When** the operator is viewing the agent, **Then** the displayed status changes to "running" without any manual refresh within 10 seconds of the agent becoming ready. +3. **Given** an agent that is starting, **When** the operator reloads the page mid-startup, **Then** the displayed status still shows startup-in-progress (the truthful state does not depend on staying on the page). +4. **Given** an agent whose startup genuinely fails (e.g., it can never become ready), **When** the failure is definitive, **Then** the status shows a failure state with a human-readable reason — real failures must still be visible. + +--- + +### User Story 2 - First start, restart, and update are distinguishable (Priority: P2) + +An operator deploying a brand-new agent sees messaging that clearly says the agent is being started for the first time. An operator restarting or updating an existing agent sees messaging that clearly says it is restarting/updating. Today both flows show the single status "deploying", which makes a first launch feel like an update of something that already existed. + +**Why this priority**: Removes the "it seems to be updating, not starting for the first time" confusion. Less critical than the false-failure problem, but essential for operators to understand what the system is actually doing. + +**Independent Test**: Deploy a never-before-run agent and separately restart an existing one; verify the two flows present visibly different startup messaging. + +**Acceptance Scenarios**: + +1. **Given** a newly created agent that has never run, **When** it is deployed, **Then** the status/messaging communicates a first-time start. +2. **Given** an existing agent that is running or stopped, **When** the operator restarts or starts it, **Then** the status/messaging communicates a restart — not a first-time launch and not a failure. + +--- + +### User Story 3 - Log panel stays friendly while the agent comes up (Priority: P2) + +An operator watching the log panel during a start or restart sees a clear, human-readable message that the agent is coming up and logs will resume shortly. Once the agent's runtime can serve logs, streaming resumes automatically. Today the panel briefly renders a raw infrastructure error dump (HTTP status code, JSON body, response headers) while the runtime is still being created. + +**Why this priority**: The raw error blob is the most alarming single artifact of the current experience — it looks like a crash report during every routine restart. It is presentation-only and independently fixable. + +**Independent Test**: Restart an agent with the log panel open and confirm that at no point is raw protocol/infrastructure error text rendered, and that logs resume without a manual reload. + +**Acceptance Scenarios**: + +1. **Given** an agent that is restarting and whose runtime cannot yet serve logs, **When** the log panel refreshes, **Then** the operator sees a friendly waiting message (not a raw error response) for as long as logs are unavailable. +2. **Given** the agent's runtime has become able to serve logs, **When** the next automatic refresh occurs, **Then** log streaming resumes without any manual action. + +--- + +### User Story 4 - A successful start shows no misleading errors (Priority: P3) + +An operator watching the startup logs of an agent that launches successfully sees no error-level entries that suggest the launch is broken. If an optional integration (such as a tool-server connection) fails while the agent itself starts fine, the message clearly states what is affected and that the agent is otherwise operational — or the underlying issue is fixed so the error does not occur. Today a successful first launch shows a red "connect failed" error even though the agent works. + +**Why this priority**: Contributes to the same "healthy launch looks broken" confusion, but the agent is functional, so it is less urgent than the status and log-panel fixes. + +**Independent Test**: Perform a fresh first-time launch of an agent and review every error-level entry in the visible startup log; each one must correspond to a real, user-actionable problem. + +**Acceptance Scenarios**: + +1. **Given** an agent that starts successfully, **When** the operator reviews the visible startup log, **Then** no error-level entry implies the startup failed. +2. **Given** an optional integration fails to connect while the agent itself starts fine, **When** this is shown in the log, **Then** the message identifies what is degraded (e.g., which tools are unavailable) and does not present the launch itself as failed. + +--- + +### Edge Cases + +- What happens when the operator stops an agent and immediately starts it again? No stale terminal status (previous "failed" or "stopped") may leak into the new launch's display. +- What happens when a restart is requested while a deploy is already in progress? The display must remain in a single coherent startup state, not oscillate between states. +- How does the system distinguish a slow-but-healthy startup from a genuine failure? Transient not-ready conditions during startup must not be reported as failure; only a definitive failure (agent can never become ready) may be. +- What happens when an agent hangs in startup indefinitely? After the 5-minute safety timeout the operator sees that startup exceeded the expected time (with explanation) instead of an eternal "starting" state. +- What happens when the log panel is open across the restart boundary? The panel must transition from old-run logs to the waiting message to new-run logs without showing raw errors. +- What happens when the operator opens the agent page for the first time mid-startup (no prior in-page context)? The correct startup state must still be shown. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The system MUST report a distinct startup-in-progress state from the moment a start or restart is requested until the agent is either confirmed ready or has definitively failed. +- **FR-002**: The system MUST NOT display a failure status for an agent whose startup is still in progress; a failure status may only be shown once the startup has definitively failed. A startup counts as definitively failed only when (a) the runtime emits an explicit failure signal (e.g., repeated crashes, inability to create or schedule the agent's runtime), or (b) a safety timeout of 5 minutes elapses without the agent becoming ready — in which case the display states that startup exceeded the expected time. +- **FR-003**: The system MUST visibly distinguish a first-time start of a never-run agent from a restart/update of an existing agent. The distinction MUST come from a server-provided launch context (first start / restart / update) supplied alongside the existing status vocabulary, which remains unchanged — so it survives page reloads and is not inferred client-side. +- **FR-004**: The displayed status MUST update to "running" without user action within 10 seconds of the agent becoming ready. +- **FR-005**: The displayed lifecycle state MUST be equally correct whether the operator stayed on the page, reloaded it, or opened it mid-startup. +- **FR-006**: While the agent's runtime is not yet able to serve logs, the log view MUST show a human-readable waiting message; raw infrastructure or protocol error responses MUST never be rendered to the operator. +- **FR-007**: The log view MUST resume streaming automatically once logs become available, without a manual reload. +- **FR-008**: A startup that completes successfully MUST NOT display error-level messages implying the launch failed; failures of optional integrations MUST be presented as clearly non-fatal and MUST name what is affected. +- **FR-009**: When a startup genuinely fails, the system MUST display a failure state together with a human-readable reason. +- **FR-010**: After an agent is stopped, a subsequent start MUST transition directly from the stopped state to the startup-in-progress state without transiently displaying stale states from earlier runs. +- **FR-011**: The root cause of the transient "failed" status observed after stop → start MUST be diagnosed and eliminated at its source; the fix MUST NOT merely hide or re-label the status in the user interface. +- **FR-012**: The root cause of the tool-server connection failure observed on first launch MUST be diagnosed and fixed within this feature, so that a successful launch registers its tools without any error entry. The FR-008 presentation rule still applies to any genuinely optional integration failure that remains possible. + +### Key Entities + +- **Agent**: A deployable assistant an operator manages; has a user-visible lifecycle status and a startup history (has it ever run before). +- **Lifecycle status**: The user-visible state of an agent. The existing status vocabulary (deploying, running, stopped, failed) is retained; a separate server-provided **launch context** (first start / restart / update) accompanies the in-progress state so the UI can tell the flows apart. Exactly one status applies at any moment, and the sequence of displayed states during any operation must be plausible (no failure during a healthy launch). +- **Startup log stream**: The chronological log output an operator sees while and after an agent comes up, including its availability gaps during startup. +- **Lifecycle action**: An operator-initiated start, restart, or stop that drives status transitions. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: Across 20 consecutive successful start or restart cycles, a failure status is displayed 0 times. +- **SC-002**: In at least 95% of successful starts, the displayed status reads "running" within 10 seconds of the agent becoming ready, with no manual refresh. +- **SC-003**: An operator can tell from the status display alone — without opening logs — whether an agent is starting for the first time or restarting. +- **SC-004**: Across 20 consecutive restart cycles with the log panel open, raw infrastructure/protocol error text is rendered 0 times. +- **SC-005**: A successful first-time launch produces 0 visible error-level log entries that do not correspond to a real, user-actionable problem. +- **SC-006**: When a startup genuinely fails, the operator sees a failure state with a reason within 1 minute of the failure becoming definitive. + +## Assumptions + +- The statuses operators see today are "deploying", "running", "stopped", and "failed"; both first start and restart currently surface as the single status "deploying", which is the root of the "updating vs first launch" confusion. +- The ~30-second "failed" display observed after stop → start is a status-reporting artifact (the agent ends up running), not an actual crash-and-recover; the fix targets truthful reporting, not startup mechanics. +- The raw log-fetch error appears because logs are requested before the agent's runtime has started — an expected condition during every launch, so it must be handled as a normal waiting state, not an error. +- The red "connect failed" error on first launch concerns the built-in tool-server integration: the agent chats fine but registers zero tools, so the integration is genuinely broken, not just noisy. Diagnosing and fixing its root cause is in scope (FR-012); if diagnosis shows the cause lies outside this project, the scope is revisited explicitly during planning. +- Scope is the truthfulness and clarity of status and startup-log presentation in the agent management experience; making agents start faster, and changes to how agents are scheduled or run, are out of scope. +- Real failures must remain at least as visible as they are today; this feature must not suppress or delay genuine failure reporting (FR-009, SC-006). diff --git a/specs/001-stabilize-agent-startup/tasks.md b/specs/001-stabilize-agent-startup/tasks.md new file mode 100644 index 00000000..c4a6cdf1 --- /dev/null +++ b/specs/001-stabilize-agent-startup/tasks.md @@ -0,0 +1,190 @@ +# Tasks: Stabilize Agent Startup Status & Logs + +**Input**: Design documents from `/specs/001-stabilize-agent-startup/` + +**Prerequisites**: [plan.md](./plan.md), [spec.md](./spec.md), [research.md](./research.md), [data-model.md](./data-model.md), [contracts/agent-api.md](./contracts/agent-api.md), [quickstart.md](./quickstart.md) + +**Tests**: Not explicitly requested in the spec — no dedicated test-first tasks. Each story ends with a quickstart validation task; the api Jest suite and `tsc` checks run in Polish. + +**Organization**: Tasks are grouped by user story. US1 and US2 share files (`agentDeploy.service.ts`, overlay components) and run sequentially; US3 and US4 are independent and can run in parallel with anything after Phase 2. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies on incomplete tasks) +- **[Story]**: US1–US4, mapping to spec.md user stories + +## Phase 1: Setup + +**Purpose**: Green baseline before touching lifecycle code + +- [x] T001 Verify toolchain and baseline: `bun install` at repo root; `cd api && bun run test` passes; api starts and serves swagger (`bun run dev` → `scripts/wait-for-swagger.mjs` succeeds); note current behaviour of `GET /agents/:id` during a deploy for later comparison + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Schema, types, and DTO/SDK surface that every story (except US3) builds on + +**⚠️ CRITICAL**: Complete before starting US1/US2/US4 (US3 does not depend on this phase) + +- [x] T002 Add nullable columns `firstDeployedAt DateTime?`, `lastDeployStartedAt DateTime?`, `lastLaunchContext String?`, `statusReason String?` to the Agent model in api/src/slices/agent/agent/agent.prisma per [data-model.md](./data-model.md) +- [x] T003 Regenerate composed schema and create the additive migration: `cd api && bun run generate && bun run migrate` (migration must apply cleanly to an existing dev DB; verify no destructive statements in api/prisma/migrations/) +- [x] T004 [P] Add `LaunchContext = 'initial' | 'restart'` type and extend agent domain types in api/src/slices/agent/agent/domain/agent.types.ts +- [x] T005 Extend `AgentGateway.updateStatus()` in api/src/slices/agent/agent/data/agent.gateway.ts to atomically accept `statusReason` (null = clear) and add write helpers for `firstDeployedAt`/`lastDeployStartedAt`/`lastLaunchContext`; keep it the single status writer +- [x] T006 Expose `launchContext`, `statusReason`, `firstDeployedAt` on agent responses (swagger-annotated DTO for `GET /agents`, `GET /agents/:id`, and the agent embedded in `AgentStatusDto`) in api/src/slices/agent/agent/dtos/ and api/src/slices/agent/agent/agent.controller.ts per [contracts/agent-api.md](./contracts/agent-api.md) §1 +- [x] T007 Regenerate the admin SDK: `cd admin && bun run build:api`; verify the three new fields appear typed in admin/slices/setup/api/data/repositories/api/types.gen.ts +- [x] T008 Map the new fields into the admin domain: extend `IAgentData` in admin/slices/agent/agent/domain/agent.types.ts and fill them in admin/slices/agent/agent/data/agent.mapper.ts (unknown `launchContext` strings coerce to null) + +**Checkpoint**: API serves the new fields end-to-end; UI has them typed and mapped + +--- + +## Phase 3: User Story 1 — Status never lies during start/restart (Priority: P1) 🎯 MVP + +**Goal**: Stop → start and restart never show a transient `failed`; `running` appears ≤ 10 s after readiness; genuine failures still surface within 60 s, now with a reason + +**Independent Test**: quickstart S1 (stop→start & restart status sequences, page-reload check), S4 (genuine failure + reason), S5 (5-minute timeout) + +### Implementation for User Story 1 + +- [x] T009 [US1] In `deploy()` (api/src/slices/agent/agent/domain/agentDeploy.service.ts): write `lastDeployStartedAt = now` together with the `deploying` status write; clear `statusReason`; set `statusReason` on the template-missing and submit-throw `failed` paths (per [data-model.md](./data-model.md) definitive-failure table) +- [x] T010 [US1] In api/src/slices/agent/agent/domain/agentStatus.service.ts: (a) drift no-pod branch skips agents with `now − lastDeployStartedAt ≤ 5 min`; (b) on expiry marks `failed` with `statusReason` "startup did not produce a running agent within 5 minutes"; (c) pod-event reconciler failure writes set `statusReason` from waiting reason / termination message; (d) transitions to `running` clear `statusReason` +- [x] T011 [US1] In `restartAgent()` (api/src/slices/agent/agent/domain/agentDeploy.service.ts): clear `workflowId` (null) BEFORE `cancelAgentWorkflow`, so no reader can resolve the doomed workflow (contracts §2 restart guarantee) +- [x] T012 [US1] In `syncStatus` (api/src/slices/agent/agent/agent.controller.ts): with T011 in place the referenced workflow can only ever be the CURRENT one (restart detaches the old id before cancelling; stop clears it), so its `Failed`/`Error` phase is a definitive signal — write `failed` immediately WITH a `statusReason`, no time-based guard (narrowed from the original blanket grace-window formulation per analysis finding I1, which would have delayed real workflow-level failures and violated SC-006); additionally skip the write while the runtime is live on the bridle hub (bridle-truth wins — found in live testing: a lying/stale workflow record ping-ponged a healthy agent between `failed` and `running`), and MockWorkflowGateway.getStatus now throws for unknown workflows instead of fabricating phase `Failed` (in-memory store resets on every dev API restart); keep existing `stopped` early-return +- [x] T013 [US1] Fix `restart_agent` in api/src/slices/rancher/rancher.tool.ts to invoke the real `restartAgent()` flow instead of a bare `deploying` status write +- [x] T014 [US1] In admin/slices/agent/agent/composables/useAgentLifecycle.ts: skip poll ticks while a lifecycle mutation request is in flight (restart/start/stop awaiting HTTP response), so a stale pre-restart status can't overwrite the optimistic `deploying` +- [x] T015 [US1] Surface `statusReason` in the admin failed states: failed chat overlay detail in useAgentLifecycle.ts `chatOverlay` + admin/slices/agent/agent/components/agent/chat/Tab.vue, and as tooltip/subtext on the status badge via admin/slices/agent/agent/utils/agentFormat.ts and components that render it +- [ ] T016 [US1] Validate per specs/001-stabilize-agent-startup/quickstart.md S1, S4, S5 against the local k3d + Argo environment; record observed status sequences in the PR description + +**Checkpoint**: SC-001/SC-002/SC-006 verifiable; badge never flashes `failed` during a healthy launch + +--- + +## Phase 4: User Story 2 — First start, restart, and update are distinguishable (Priority: P2) + +**Goal**: Server-provided `launchContext` drives visibly different first-start vs restart messaging; survives page reload + +**Independent Test**: quickstart S3 (fresh agent shows first-start wording and `launchContext: "initial"`; subsequent restart shows restart wording and `"restart"`) + +### Implementation for User Story 2 + +- [x] T017 [US2] In `deploy()` (api/src/slices/agent/agent/domain/agentDeploy.service.ts): compute `lastLaunchContext` = `'initial'` iff `firstDeployedAt` is null at call time else `'restart'`, persist it with the `deploying` write; set `firstDeployedAt = now` once, immediately after the first successful workflow submit (depends on T009 — same file) +- [x] T018 [US2] Drive the chat overlay copy from `launchContext` in admin/slices/agent/agent/composables/useAgentLifecycle.ts (`chatOverlay`): first start → "Setting up the agent…"-style title/detail; restart → existing "Restarting…" wording; never show "Cancelling old workflow…" for `initial` (depends on T014/T015 — same file) +- [x] T019 [US2] Pass launch context into the logs panel and switch its placeholder copy in admin/slices/agent/agent/components/agent/chat/Tab.vue and admin/slices/agent/agent/components/agent/logs/Panel.vue (first start vs "Agent is restarting — logs will resume…") +- [ ] T020 [US2] Validate per specs/001-stabilize-agent-startup/quickstart.md S3 including the reload check (FR-005) and SC-003 (distinguishable from status display alone) + +**Checkpoint**: US1 + US2 shippable together; first launch no longer looks like an update + +--- + +## Phase 5: User Story 3 — Log panel stays friendly while the agent comes up (Priority: P2) + +**Goal**: The friendly `[container …]` marker actually fires; raw multi-line k8s error dumps never reach the UI + +**Independent Test**: quickstart S2 (restart with log panel open; `curl` logs endpoint during ContainerCreating returns `[container containercreating]`) + +**Note**: Independent of Phase 2 — can start any time after Setup, in parallel with US1/US2 + +### Implementation for User Story 3 + +- [x] T021 [P] [US3] In api/src/slices/log/log.controller.ts: normalize `ApiException.body` (JSON.parse when string, try/catch) in `extractWaitingReason` and `extractKubeError`; also match the waiting-reason regex against `e.message` as fallback; ensure `[log fetch failed: …]` payload is a single parsed line (contracts §3) +- [x] T022 [P] [US3] Apply the same string-body normalization to `extractKubeError` in api/src/slices/agent/agent/data/pod.gateway.ts (server-log noise reduction, same error shape) +- [ ] T023 [US3] Validate per specs/001-stabilize-agent-startup/quickstart.md S2: no raw `HTTP-Code: 400` text in the panel across a restart, spinner state shown, logs auto-resume (SC-004, FR-007) + +**Checkpoint**: SC-004 verifiable independently of all other stories + +--- + +## Phase 6: User Story 4 — A successful start shows no misleading errors (Priority: P3) + +**Goal**: CleanSlice MCP connects (tools > 0) so the startup log has no connect-failure line; warn-level lines are not styled as errors + +**Independent Test**: quickstart S6 (seeder heal + live `/mcp` probe) and the log assertions of S3 (N > 0 tools, no ERROR-styled lines on a clean start) + +**Note**: Independent of US1–US3; T024–T025 touch disjoint files + +### Implementation for User Story 4 + +- [x] T024 [P] [US4] In api/src/slices/mcpServer/domain/mcpServer.seeder.ts: change the default CleanSlice URL to `https://mcp.cleanslice.org/mcp` (env `CLEANSLICE_MCP_URL` still wins) and make the seeder idempotently converge an EXISTING built-in row's `url` to the configured value on every bootstrap (contracts §5) +- [x] T025 [P] [US4] In admin/slices/agent/agent/utils/agentLogs.ts: classify severity by the runtime's explicit level token when present, falling back to substring heuristics only without one — first verify the real pod-log line format, then adjust `ERROR_TOKEN_RE`/classifier so `"error":"Not Found"` inside a JSON body no longer styles a warn line as ERROR (FR-008) +- [ ] T026 [US4] Validate per specs/001-stabilize-agent-startup/quickstart.md S6 (row healed, probe returns 200, re-boot idempotent) and re-run S3's log assertions after a fresh agent deploy (SC-005, FR-012) + +**Checkpoint**: A clean first launch shows zero error-level entries end-to-end + +--- + +## Phase 7: Polish & Cross-Cutting Concerns + +- [ ] T027 Run the full regression sweep per specs/001-stabilize-agent-startup/quickstart.md S7: rancher-tool restart, stop-while-deploying, rapid stop/start cycles, 20× S1 + S2 cycles for SC-001/SC-004 sign-off +- [x] T028 [P] Verify suites green: `cd api && bun run test`; type-check pure-TS admin changes with `tsc` (repo has no vue-tsc); `cd admin && bun run build` compiles +- [ ] T029 [P] Opportunistic cleanup (optional): replace the outdated raw-axios stop/start workaround in admin/slices/agent/agent/data/agent.gateway.ts with the now-existing generated SDK calls (`agentControllerStop`/`agentControllerStart`), noted in research.md cross-cutting notes + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: none +- **Foundational (Phase 2)**: after Setup — blocks US1, US2, US4-validation; internal order T002 → T003 → T005 → T006 → T007 → T008, with T004 parallel after T002 +- **US1 (Phase 3)**: after Phase 2 +- **US2 (Phase 4)**: after US1 (T017 extends T009's code in agentDeploy.service.ts; T018 extends T014/T015's code in useAgentLifecycle.ts) +- **US3 (Phase 5)**: after Setup only — fully parallel with Phases 2–4 +- **US4 (Phase 6)**: T024/T025 after Setup; T026 needs a deployable stack (and re-uses S3's assertions, so best after US2) +- **Polish (Phase 7)**: after all selected stories + +### User Story Dependencies + +- **US1 (P1)**: Foundational only — the MVP +- **US2 (P2)**: builds on US1's edits (same files); independently *testable* via S3 once merged +- **US3 (P2)**: no dependencies on other stories +- **US4 (P3)**: no code dependencies on other stories; final validation piggybacks on S3 + +### Parallel Opportunities + +```bash +# After Setup, three tracks can run concurrently (different files): +Track A (api lifecycle): T002→T003→…→T008 then T009–T016 then T017–T020 +Track B (log handling): T021, T022 in parallel, then T023 +Track C (MCP + classifier): T024, T025 in parallel + +# Within Phase 2: T004 alongside T003 +# Within US3: T021 ∥ T022 +# Within US4: T024 ∥ T025 +# Polish: T028 ∥ T029 +``` + +--- + +## Implementation Strategy + +### MVP First (US1 only) + +1. Phase 1 → Phase 2 → Phase 3 (T009–T016) +2. **STOP and VALIDATE**: quickstart S1/S4/S5 — the false-`failed` symptom is gone; real failures keep a reason +3. Ship (release goes out on the next `v*` tag) + +### Incremental Delivery + +1. + US3 (can even land before US1 — zero coupling): the raw 400 dump disappears +2. + US2: first start vs restart become distinguishable +3. + US4: clean first-launch log, MCP tools actually register +4. Polish: regression sweep + suites + optional gateway cleanup + +Each story leaves all previous behaviour intact; no story changes the status vocabulary or breaks the SDK contract. + +--- + +## Notes + +- Code-review follow-ups (self-review of the working diff, implemented): + - Stale-workflow race closed in the remaining admin paths (create-with-isAdmin, promote-admin, demote-admin): extracted `AgentDeployService.detachAndCancelWorkflow()` — detaches `workflowId` before cancelling — and reused it in `restartAgent` + all controller call sites (agent delete excluded: the row is gone before the cancel). + - `statusReason` for workflow-submit failures is now the generic `workflow submit failed` (raw submit errors can leak internal detail on the public agent endpoints; the full message stays in the server log). + - Backfill migration `20260730130000_backfill_first_deployed_at` sets `firstDeployedAt = updatedAt` for pre-existing non-`pending` agents, so legacy agents don't show the first-start copy on their next deploy. +- Live-testing follow-ups (found during dev validation, implemented alongside US1): + - Bridle-truth resurrect paths (drift sweep + `markRunningFromBridle`) now exempt `stopped` — the old runtime's WS lingers after stop (indefinitely on local dev) and the resurrect undid the operator's stop, then decayed to `failed` once the WS dropped. UI: the "Agent stopped" overlay now wins over the live-chat bypass (admin/.../useAgentLifecycle.ts). + - Duplicate `GET /agents/:id` on page open removed: BridleProvider accepts `initial-debug-enabled` from the host instead of fetching the agent itself (admin/slices/bridle/components/bridle/Provider.vue + chat/Tab.vue). +- Total: **29 tasks** (Setup 1, Foundational 7, US1 8, US2 4, US3 3, US4 3, Polish 3) +- US1+US2 intentionally share files — do not parallelize those two stories across people +- Full lifecycle validation requires k3d + Argo (`infrastructure.workflow_provider = argo`); the mock provider hides the pod-less window +- Commit after each task or logical group; stop at any checkpoint to validate independently