diff --git a/src/codex/account-lifecycle.ts b/src/codex/account-lifecycle.ts index 65e35aa5f10..ffb7a5be540 100644 --- a/src/codex/account-lifecycle.ts +++ b/src/codex/account-lifecycle.ts @@ -14,6 +14,7 @@ import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account"; import { clearAccountQuota } from "./quota"; import { forgetManualPreference } from "./routing/active-account"; import { deleteAllHealthForAccount } from "./routing/health-store"; +import { forgetCodexFailureWindow } from "./routing/failure-window"; import { clearThreadAccountMapForAccount } from "./routing/thread-affinity"; import { clearCodexPoolRefreshFailure } from "./pool-refresh-backoff"; @@ -49,11 +50,15 @@ export function purgeCodexAccountRuntimeState(accountId: string): void { clearAccountNeedsReauth(accountId); clearAccountQuota(accountId); clearThreadAccountMapForAccount(accountId); - // Deliberately not via `./routing`: the account-delete path only needs the health rows and - // the manual preference gone, and importing the router here closed a cycle through - // routing/health-store and routing/active-account. + // Deliberately not via `./routing`: the account-delete path only needs the health rows + // and their sibling routing state gone, and importing the router barrel here closed a + // cycle through routing/health-store and routing/active-account. The leaf imports below + // carry every per-account forget that barrel helper used to perform — including the + // failure window, whose degrade and pin-warning state must not leak into a replacement + // account or a new physical login behind __main__. deleteAllHealthForAccount(accountId); forgetManualPreference(accountId); + forgetCodexFailureWindow(accountId); clearCodexPoolRefreshFailure(accountId); if (accountId === MAIN_CODEX_ACCOUNT_ID) { clearMainAccountInfoCache(); diff --git a/src/codex/account-store.ts b/src/codex/account-store.ts index bfe677c2d8e..b384ab5588b 100644 --- a/src/codex/account-store.ts +++ b/src/codex/account-store.ts @@ -136,25 +136,37 @@ export function peekCodexAccountRecordStore(): Record boolean) + & { readonly snapshotStatus: "ok" | "failed" }; + /** * The same verdict as {@link isCodexAccountGenerationLive}, over ONE store load. * @@ -409,12 +425,15 @@ export function isCodexAccountGenerationLive(id: string, generation: number): bo * several accounts in one synchronous pass on the request path, so it opens one checker and * closes over the snapshot instead. */ -export function beginCodexAccountGenerationLiveCheck(): (id: string, generation: number) => boolean { - const snapshot = loadCodexAccountRecordSnapshot(); - return (id, generation) => { +export function beginCodexAccountGenerationLiveCheck(): CodexAccountGenerationLiveCheck { + let result: CodexAccountRecordStoreResult; + try { result = loadCodexAccountRecordStoreResult(); } + catch { result = { status: "failed" }; } + const snapshot = result.status === "ok" ? result.records : {}; + return Object.assign((id: string, generation: number) => { const record = snapshot[id]; return !!record?.credential && record.deletedAt == null && record.generation === generation; - }; + }, { snapshotStatus: result.status }); } export function saveCodexAccountCredentialIfGeneration( diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index c4cd30b325d..7e1cd3548e7 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -324,6 +324,8 @@ export type CodexAuthContext = writerGeneration: number; /** Captured before async credential work; never reconstructed after the upstream response. */ mainQuotaWriter?: MainQuotaWriter; + /** Immutable credential proof captured while materializing this physical attempt's headers. */ + mainQuotaDispatch?: MainQuotaDispatch; reserveAuthorization?: MainReserveAuthorization; accessToken: string; chatgptAccountId: string; @@ -1766,6 +1768,8 @@ export function materializeCodexUpstreamAuth( if (ctx.kind === "main-pool") { ctx.mainQuotaWriter = observeSelectedMainCredential(ctx, ctx.mainQuotaWriter); assertMainAccountPolicy(options.config); + ctx.mainQuotaDispatch = ctx.mainQuotaWriter && isMainQuotaWriterLive(ctx.mainQuotaWriter) + ? selectedMainQuotaDispatch(selected) : undefined; } assertMaterializedReserve(selected, ctx, options); return selected; diff --git a/src/codex/routing.ts b/src/codex/routing.ts index 7eefb87e4e2..57bf246ec45 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -23,7 +23,7 @@ import { failureWindowSteersNewThreads, forgetCodexFailureWindow, forgetCodexFailureWindows, - noteCodexFailureWindowSample, + noteCodexFailureWindowOutcome, settleCodexSuccessStreak, } from "./routing/failure-window"; @@ -1305,7 +1305,7 @@ export function recordCodexUpstreamOutcome( */ dropSpentCredentialFailure(accountId); if (outcomeClass === "success") { - noteCodexFailureWindowSample(config, accountId, false, now); + noteCodexFailureWindowOutcome(config, accountId, false, now, meta); // The operator's one-shot is spent by a dispatch that actually worked, and only by that. // A failed lookup leaves it unspent so the intent survives the failure. consumeManualPreference(accountId, codexPoolKeyForScope(quotaScope)); @@ -1555,7 +1555,7 @@ export function recordCodexUpstreamOutcome( // A transient failure concludes an owning probe; an unrelated 5xx must not // consume someone else's live lease or drop hard-cooldown bookkeeping (#433). const transientBase = ownsProbeLease(current, meta) ? withProbeLeaseReleased(current!, now) : current; - noteCodexFailureWindowSample(config, accountId, true, now); + noteCodexFailureWindowOutcome(config, accountId, true, now, meta); const stale = current?.lastFailureAt ? now - current.lastFailureAt > CODEX_FAILURE_WINDOW_MS : false; const hardCooldownUntil = getCodexAccountCooldownUntil(accountId, now) ?? undefined; // Soft avoid + affinity clears are part of failover. When threshold is 0, leave diff --git a/src/codex/routing/cooldown-math.ts b/src/codex/routing/cooldown-math.ts index 3e03e029aae..515b4d9eb22 100644 --- a/src/codex/routing/cooldown-math.ts +++ b/src/codex/routing/cooldown-math.ts @@ -6,6 +6,7 @@ import { isThirtyDayOnlyCodexPlan } from "../plan"; import { hasSpendableCodexCredits, isTerminalShortWindow, type CodexSpendableCredits } from "../quota-types"; import type { CodexQuotaScope } from "./health-store"; import type { TransientProbeGrant } from "./thread-affinity"; +import type { MainQuotaDispatch } from "../main-account-cache"; export { TERMINAL_SHORT_WINDOW_FRESHNESS_MS } from "../quota-types"; @@ -99,6 +100,10 @@ export type CodexUpstreamOutcomeMeta = { * means the caller cannot supply lineage and the historical unfenced handling stands. */ credentialGeneration?: number; + /** Main credential proof captured before dispatch; null means no owned proof was available. */ + mainFailureWindowDispatch?: MainQuotaDispatch | null; + /** Pool sample provenance retained by a terminal callback even if its context is reused. */ + failureWindowCredentialGeneration?: number; }; export function computeCodexUsageScore(quota: { diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index 71aa43ebee0..61ae76149be 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -1,5 +1,9 @@ import { pinnedCodexAccountId } from "../account-priority"; import type { OcxConfig } from "../../types"; +import { beginCodexAccountGenerationLiveCheck, isCodexAccountGenerationLive } from "../account-store"; +import { isMainQuotaDispatchLive } from "../main-account-cache"; +import type { CodexAuthContext } from "../auth-context"; +import type { CodexUpstreamOutcomeMeta } from "./cooldown-math"; import { deleteAccountHealth, setAccountHealth, @@ -14,12 +18,31 @@ export const CODEX_SLIDING_FAILURE_RECOVER_RATIO = 0.1; export const CODEX_SLIDING_FAILURE_RECOVER_SUSTAIN_MS = 30_000; const MAX_SAMPLES = 512; -type Sample = { at: number; failed: boolean }; -type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number }; +type Sample = { at: number; failed: boolean; credentialGeneration?: number }; +type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number; credentialGeneration?: number }; const windows = new Map(); const pinnedDegradeWarned = new Set(); +/** Read only dispatch-time provenance retained by auth materialization, never recapture at arrival. */ +export function codexFailureWindowOutcomeMeta( + context: CodexAuthContext, +): Pick { + if (context.kind === "pool") return { failureWindowCredentialGeneration: context.generation }; + return context.kind === "main-pool" ? { mainFailureWindowDispatch: context.mainQuotaDispatch ?? null } : {}; +} + +/** Fence window samples only; other health and recovery-lease semantics stay with their owners. */ +export function noteCodexFailureWindowOutcome( + config: OcxConfig, accountId: string, failed: boolean, now: number, meta: CodexUpstreamOutcomeMeta, +): void { + if (meta.mainFailureWindowDispatch !== undefined + && (!meta.mainFailureWindowDispatch || !isMainQuotaDispatchLive(meta.mainFailureWindowDispatch))) return; + const generation = meta.failureWindowCredentialGeneration ?? meta.credentialGeneration; + if (generation !== undefined && !isCodexAccountGenerationLive(accountId, generation)) return; + appendSample(config, accountId, failed, now, generation); +} + export function codexFailureWindowEnabled(config: OcxConfig): boolean { return config.codexFailureWindow !== false; } @@ -33,11 +56,27 @@ function stateFor(accountId: string): WindowState { return state; } -function recompute(state: WindowState, now: number): void { - const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; - if (state.samples.length > 0 && state.samples[0]!.at <= cutoff) { - state.samples = state.samples.filter(sample => sample.at > cutoff); +function recompute( + accountId: string, state: WindowState, now: number, + generationLive?: ReturnType, +): void { + // One store snapshot per insertion/read, never one file load per retained sample. The + // unlocked insertion precheck can race another process; provenance must survive it. + const tagged = state.credentialGeneration !== undefined + || state.samples.some(sample => sample.credentialGeneration !== undefined); + const live = generationLive ?? (tagged ? beginCodexAccountGenerationLiveCheck() : undefined); + // Unknown evidence is not a credential removal: preserve samples and all derived state. + if (live?.snapshotStatus === "failed") return; + if (state.credentialGeneration !== undefined && !live?.(accountId, state.credentialGeneration)) { + state.degraded = false; + state.healthySince = undefined; + state.credentialGeneration = undefined; + pinnedDegradeWarned.delete(accountId); } + const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; + state.samples = state.samples.filter(sample => sample.at > cutoff + && (sample.credentialGeneration === undefined || live?.(accountId, sample.credentialGeneration))); + state.credentialGeneration ??= state.samples.find(sample => sample.credentialGeneration !== undefined)?.credentialGeneration; const samples = state.samples.length; const failed = state.samples.reduce((count, sample) => count + (sample.failed ? 1 : 0), 0); const degraded = samples >= CODEX_SLIDING_FAILURE_MIN_SAMPLES @@ -71,19 +110,32 @@ export function noteCodexFailureWindowSample( accountId: string, failed: boolean, now: number, +): void { + appendSample(config, accountId, failed, now); +} + +function appendSample( + config: OcxConfig, accountId: string, failed: boolean, now: number, credentialGeneration?: number, ): void { if (!codexFailureWindowEnabled(config)) return; + const existing = windows.get(accountId); + const tagged = credentialGeneration !== undefined || existing?.credentialGeneration !== undefined + || existing?.samples.some(sample => sample.credentialGeneration !== undefined); + const live = tagged ? beginCodexAccountGenerationLiveCheck() : undefined; + if (live?.snapshotStatus === "failed") return; + // A stale arrival is not an ownership transition and must not delete a live replacement's state. + if (credentialGeneration !== undefined && !live?.(accountId, credentialGeneration)) return; const state = stateFor(accountId); - state.samples.push({ at: now, failed }); + state.samples.push({ at: now, failed, ...(credentialGeneration === undefined ? {} : { credentialGeneration }) }); if (state.samples.length > MAX_SAMPLES) state.samples.splice(0, state.samples.length - MAX_SAMPLES); - recompute(state, now); + recompute(accountId, state, now, live); if (!state.degraded) pinnedDegradeWarned.delete(accountId); } export function isCodexFailureWindowDegraded(accountId: string, now = Date.now()): boolean { const state = windows.get(accountId); if (!state) return false; - recompute(state, now); + recompute(accountId, state, now); if (!state.degraded) pinnedDegradeWarned.delete(accountId); return state.degraded; } diff --git a/src/providers/openai-sidecar.ts b/src/providers/openai-sidecar.ts index c68c0c73e7b..93e3398ae18 100644 --- a/src/providers/openai-sidecar.ts +++ b/src/providers/openai-sidecar.ts @@ -12,6 +12,7 @@ import { type CodexAuthPolicyConfig, } from "../codex/auth-context"; import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome } from "../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../codex/routing/failure-window"; import { inspectChatGptDomainClaim } from "../oauth/chatgpt"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential, type DataPlaneAdmission } from "../server/auth-cors"; import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "../types"; @@ -205,6 +206,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); throw new CodexPoolAuthenticationError("Selected Codex account is unavailable"); } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -220,6 +222,7 @@ export async function resolveFirstUsableOpenAiSidecar( probeLeaseId: authContext.probeLeaseId, probeQuotaScope: authContext.probeQuotaScope, writerGeneration: authContext.writerGeneration, + ...windowMeta, // 401/403 here is evidence about this exact stored credential; without the generation a // replacement inherits the quarantine (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), @@ -256,6 +259,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); continue; } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -271,6 +275,7 @@ export async function resolveFirstUsableOpenAiSidecar( threadId: authContext.affinityKey, probeLeaseId: authContext.probeLeaseId, writerGeneration: authContext.writerGeneration, + ...windowMeta, // Same fence as the exact-account recorder above (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), }, diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index cb93a79f280..32263104a72 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -88,6 +88,7 @@ import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome, } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountChangeConversationStateScrub, conversationStateBindingFromAuth, @@ -1062,6 +1063,7 @@ export async function handleResponsesCompact( if ((outcome === 401 || outcome === 403) && upstream && wasPermanentAccountRefusal(upstream)) return; recordCodexUpstreamOutcome(config, ctx.accountId, outcome, { ...meta, + ...codexFailureWindowOutcomeMeta(ctx), threadId: ctx.kind === "pool" || ctx.kind === "main-pool" ? ctx.affinityKey : undefined, fixedAccount: ctx.fixedAccount, modelId: selectedModelId, diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index a2b35f7da73..b51b7ce32c1 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -11,6 +11,7 @@ import { claimMainQuotaDispatchForWs, isMainQuotaDispatchLive, renewMainQuotaDis import type { CodexWsQuotaObserver } from "./codex-ws-metadata"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isCodexAccountGenerationLive } from "../../codex/account-store"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountQuotaFromUpstreamHeaders as applyCapturedCodexQuota } from "../../codex/quota"; import type { RouteResult } from "../../router"; import { @@ -93,20 +94,19 @@ export function sidecarOutcomeRecorder( config: OcxConfig, authCtx: CodexAuthContext, ): ((outcome: CodexUpstreamOutcome) => void) | undefined { - return authCtx.kind === "pool" || authCtx.kind === "main-pool" - ? outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { - threadId: authCtx.affinityKey, - fixedAccount: authCtx.fixedAccount, - probeLeaseId: authCtx.probeLeaseId, - probeQuotaScope: authCtx.probeQuotaScope, - writerGeneration: authCtx.writerGeneration, - // A vision or web-search sidecar can return 401/403, and that is evidence about the exact - // stored credential it used. Without the generation it becomes an account-wide quarantine - // that a replacement inherits (#2892 gap 4). `main-pool` has no stored-record generation, so - // it keeps the unfenced account-wide semantics. - ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), - }) - : undefined; + if (authCtx.kind !== "pool" && authCtx.kind !== "main-pool") return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); + return outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, + threadId: authCtx.affinityKey, + fixedAccount: authCtx.fixedAccount, + probeLeaseId: authCtx.probeLeaseId, + probeQuotaScope: authCtx.probeQuotaScope, + writerGeneration: authCtx.writerGeneration, + // Quarantine retains its stored-credential fence. main-pool has no stored-record + // generation; its separate dispatch proof above fences only failure-window samples. + ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), + }); } @@ -148,6 +148,7 @@ export function recordCodexPreOutputRefusalOutcome( if (!usesCodexForwardPoolAuth(authCtx, provider) || wasPermanentAccountRefusal(response)) return; noteCodexWsPreOutputRefusalOutcome(response); recordCodexUpstreamOutcome(config, authCtx.accountId, response.status, { + ...codexFailureWindowOutcomeMeta(authCtx), threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -676,6 +677,7 @@ export async function retryCodexPoolOnAlternateAccount( if (codexWsPreOutputRefusalOutcomeRecorded(firstResponse)) return; noteCodexWsPreOutputRefusalOutcome(firstResponse); recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, fixedAccount: firstAuthCtx.fixedAccount, modelId: route.modelId, @@ -693,6 +695,7 @@ export async function retryCodexPoolOnAlternateAccount( if (outcomeStatus === firstResponse.status || (outcomeStatus !== 429 && outcomeStatus !== 402)) return; recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...codexQuotaOutcomeMeta(firstResponse), + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -862,6 +865,7 @@ export async function retryCodexPoolOnAlternateAccount( } recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...quotaMeta, + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -1149,6 +1153,7 @@ export function codexForwardTerminalOutcomeRecorder( logCtx?: RequestLogContext, ): ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined { if (!usesCodexForwardPoolAuth(authCtx, provider)) return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); return (status, httpStatusOverride) => { const quotaStatus = [httpStatusOverride, logCtx?.terminalHttpStatus] .find(value => value === 429 || value === 402); @@ -1157,6 +1162,7 @@ export function codexForwardTerminalOutcomeRecorder( // request. Don't penalize account health; record success to clear any // prior soft-avoid so a healthy account isn't stuck avoided. recordCodexUpstreamOutcome(config, authCtx.accountId, 200, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -1180,6 +1186,7 @@ export function codexForwardTerminalOutcomeRecorder( ? 200 : (quotaStatus ?? httpStatusOverride ?? logCtx?.terminalHttpStatus ?? 502); recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 3d286dec672..cac782e0e28 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -37,6 +37,7 @@ import type { ResponsesTerminalStatus } from "../../bridge"; import { isCodexWsQuotaObservedResponse, isCodexWsUpstreamResponse, isCodexWsPreludeProjection } from "./ws-upstream"; import { recordSubagentQuotaFailureForThreadSpawn } from "../../codex/subagent-model-fallback"; import { recordCodexUpstreamOutcome } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { codexProbeLeaseId, codexProbeQuotaScope, codexTransientProbeGrant, releaseCodexAuthContextProbeLease } from "../../codex/auth-context"; import { consumeComboFailure } from "./core-combo-failure"; import { clientCancelledResponse, readDisplaySafeErrorText } from "./core-errors"; @@ -535,6 +536,7 @@ export async function deliverPassthroughResponse( // request. The sibling recorders on this path already decline: the terminal recorder // needs an ok streaming body, and the quota-header snapshot finds no quota headers. recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, upstreamResponse.status, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), ...quotaMeta, threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 849001a6798..c63bf32697b 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -18,6 +18,7 @@ import type { ResponsesEffects } from "./response-effects"; import type { ResponsesSendBudget } from "./request-send-budget"; import { transientSendCapFor } from "./request-send-budget"; import { renewMainQuotaDispatchForAttempt } from "../../codex/main-account-cache"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isLocalUpstream } from "../../lib/local-upstream"; import { codexSafetyBufferingFilterOptions, terminalStatusFromParsed } from "../relay"; @@ -990,6 +991,7 @@ export async function preparePassthroughExchange( } if (usesCodexForwardPoolAuth(admissionState.authCtx, route.provider)) { recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, outcome, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, modelId: route.modelId, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 809727ad810..09c86dd0f16 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. A snapshot read or parse failure is explicitly unknown and preserves the samples, degraded flag, recovery timer, retained generation and pin warning; a genuinely absent or valid empty store still retires its old generations. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-account-delete-atomicity.test.ts b/tests/codex-integration/codex-account-delete-atomicity.test.ts index b1df61a9400..9ec8ae90591 100644 --- a/tests/codex-integration/codex-account-delete-atomicity.test.ts +++ b/tests/codex-integration/codex-account-delete-atomicity.test.ts @@ -32,6 +32,12 @@ import { getConfigPath, loadConfig, saveConfig } from "../../src/config"; import * as configModule from "../../src/config"; import { setCodexAccountAutoSwitchThresholdOverride } from "../../src/codex/account-auto-switch"; import { clearAllManualPreferences, manualPreferenceBlocks, resetCodexRoutingForManualSelection } from "../../src/codex/routing/active-account"; +import { + failureWindowSteersNewThreads, + forgetCodexFailureWindow, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { POOL_KEY_CODEX } from "../../src/oauth/pool-kernel"; import { prepareConfigObjectChildDeletionRebase } from "../../src/config/rebase-provenance"; import type { OcxConfig } from "../../src/types"; @@ -429,4 +435,56 @@ describe("Codex account delete persistence ordering", () => { clearAllManualPreferences(); } }); + + test("deleting the account forgets its failure-window degradation but keeps sibling state", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (accountId: string): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, accountId, true, now - index); + }; + degrade(ACCOUNT_ID); + degrade("sibling-account"); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(true); + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + try { + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(false); + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + // A different account's window is untouched by the deletion. + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + } finally { + forgetCodexFailureWindow(ACCOUNT_ID); + forgetCodexFailureWindow("sibling-account"); + } + }); + + test("deleting the account forgets its pinned-degrade warning", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, ACCOUNT_ID, true, now - index); + }; + forgetCodexFailureWindow(ACCOUNT_ID); + degrade(); + const warnSpy = spyOn(console, "warn").mockImplementation(() => {}); + try { + // The seeded config pins ACCOUNT_ID under the default "hold" policy, so steering + // logs the pin-degraded warning exactly once per process-lifetime of the state. + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + expect(warnSpy).toHaveBeenCalledTimes(1); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + // Deletion clears the pin as well; re-pin to model a recreated same-id account the + // operator selects again. If the warned set survived the purge, a re-degraded + // same-id pinned account would stay silent forever; the forget lets it fire again. + config.activeCodexAccountPinned = ACCOUNT_ID; + degrade(); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(2); + } finally { + warnSpy.mockRestore(); + forgetCodexFailureWindow(ACCOUNT_ID); + } + }); }); diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 55e5e773b24..ee16af342df 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,8 +1,16 @@ -import { afterEach, beforeEach, expect, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import * as fs from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import * as accountStore from "../../src/codex/account-store"; +import { headersForCodexAuthContext, type CodexAuthContext } from "../../src/codex/auth-context"; +import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { captureMainQuotaWriter, clearMainAccountInfoCache } from "../../src/codex/main-account-cache"; +import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/account-id"; +import { captureConfigGeneration } from "../../src/lib/state-store-sweeper"; +import { codexForwardTerminalOutcomeRecorder, sidecarOutcomeRecorder } from "../../src/server/responses/core-codex-account"; import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../../src/codex/auth-api"; import { clearCodexUpstreamHealth, @@ -12,11 +20,12 @@ import { } from "../../src/codex/routing"; import { forgetCodexFailureWindows, + failureWindowSteersNewThreads, isCodexFailureWindowDegraded, noteCodexFailureWindowSample, } from "../../src/codex/routing/failure-window"; import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; -import type { OcxConfig } from "../../src/types"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const NOW = 1_700_000_000_000; @@ -47,13 +56,19 @@ function fill(accountId: string, failures: number, successes: number, now = NOW) let home = ""; let previousHome: string | undefined; +let previousCodexHome: string | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-window-")); setIcaclsRunnerForTests(() => ICACLS_OK); setAsyncIcaclsRunnerForTests(async () => ICACLS_OK); process.env.OPENCODEX_HOME = home; + process.env.CODEX_HOME = join(home, "codex"); + mkdirSync(process.env.CODEX_HOME, { recursive: true }); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); @@ -75,11 +90,383 @@ afterEach(async () => { clearThreadAccountMap(); clearAccountQuota(); forgetCodexFailureWindows(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; await removeTreeWithRetry(home); }); +const forwardProvider = { adapter: "openai-responses", authMode: "forward" } as OcxProviderConfig; + +function terminal(config: OcxConfig, context: CodexAuthContext) { + headersForCodexAuthContext(new Headers(), context, config); + return codexForwardTerminalOutcomeRecorder(config, context, forwardProvider)!; +} + +function mainTerminal(config: OcxConfig, identity: string, accessToken = `access-${identity}`) { + writeFileSync(join(process.env.CODEX_HOME!, "auth.json"), JSON.stringify({ tokens: { access_token: accessToken, account_id: identity } })); + reconcileMainCodexAccountRuntimeState(); + return terminal(config, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + mainQuotaWriter: captureMainQuotaWriter(identity), accessToken, chatgptAccountId: identity, fixedAccount: true, + }); +} + +function poolTerminal(config: OcxConfig) { + const record = readCodexAccountRecord("a")!; + return terminal(config, { + kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), generation: record.generation, + accessToken: record.credential!.accessToken, chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true, + }); +} + +test("late terminal outcomes cannot refill a replacement main failure window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const replacement = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) replacement("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + replacement("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a returned main identity does not revive the first login's terminal writer", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const first = mainTerminal(cfg, "physical-a"); + mainTerminal(cfg, "physical-b"); + const returned = mainTerminal(cfg, "physical-a"); + for (let index = 0; index < 20; index++) first("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) returned("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("late successful terminals cannot count toward the replacement main window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const current = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + old("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("sidecar recorder retains the old main dispatch even if its context is reused", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const context: CodexAuthContext = { kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, + writerGeneration: captureConfigGeneration(), mainQuotaWriter: captureMainQuotaWriter("physical-a"), + accessToken: "access-physical-a", chatgptAccountId: "physical-a" }; + headersForCodexAuthContext(new Headers(), context, cfg); + const old = sidecarOutcomeRecorder(cfg, context)!; + const current = mainTerminal(cfg, "physical-b"); + context.mainQuotaWriter = captureMainQuotaWriter("physical-b"); + context.accessToken = "access-physical-b"; + context.chatgptAccountId = "physical-b"; + headersForCodexAuthContext(new Headers(), context, cfg); + for (let index = 0; index < 20; index++) old(500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a replaced main credential retires its terminal samples without an identity change", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a", "old-access"); + const current = mainTerminal(cfg, "physical-a", "new-access"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + current("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("same-id pool recreation rejects the deleted credential's late terminal samples", () => { + const cfg = config(); + const old = poolTerminal(cfg); + deleteCodexAccount(cfg, "a"); + saveCodexAccountCredential("a", { accessToken: "replacement-access", refreshToken: "replacement-refresh", chatgptAccountId: "replacement-account", expiresAt: Date.now() + 60_000 }); + cfg.codexAccounts!.push({ id: "a", email: "replacement@test", isMain: false }); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("terminal pool samples keep the dispatched generation if the context is reused", () => { + const cfg = config(); + const record = readCodexAccountRecord("a")!; + const context: CodexAuthContext = { kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), + generation: record.generation, accessToken: record.credential!.accessToken, + chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true }; + const old = terminal(cfg, context); + context.generation = saveCodexAccountCredential("a", { ...record.credential!, accessToken: "new-access" }); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a main terminal without an owned dispatch proof cannot seed the window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const record = codexForwardTerminalOutcomeRecorder(cfg, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + accessToken: "unobserved-access", chatgptAccountId: "physical-a", + }, forwardProvider)!; + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); +}); + +test("a cross-process replacement after the unlocked live check cannot donate a terminal sample", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.isCodexAccountGenerationLive; + let swapped = false; + const liveCheck = spyOn(accountStore, "isCodexAccountGenerationLive").mockImplementation((id, generation) => { + const live = original(id, generation); + if (id === "a" && live && !swapped) { + swapped = true; + const credential = readCodexAccountRecord(id)!.credential!; + saveCodexAccountCredential(id, { ...credential, accessToken: "replacement-after-read" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + liveCheck.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a replacement does not inherit degraded state or the prior pinned warning", () => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const old = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-access" }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(2); + // Rejecting a stale writer must not erase the replacement's healthy or degraded window. + old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + } finally { + warn.mockRestore(); + } +}); + +test("a replacement after the insertion snapshot is filtered at window read time", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.beginCodexAccountGenerationLiveCheck; + let swapped = false; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(() => { + const live = original(); + if (!swapped) { + swapped = true; + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-snapshot" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + snapshot.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a full tagged window loads one current credential snapshot per read", () => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 512; index++) { + recordCodexUpstreamOutcome(cfg, "a", 200, { failureWindowCredentialGeneration: generation, now: NOW }); + } + const original = accountStore.beginCodexAccountGenerationLiveCheck; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(original); + try { + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "next-full-window" }); + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(2); + } finally { + snapshot.mockRestore(); + } +}); + +function withFailedStoreSnapshot(kind: "read" | "parse", run: () => void): void { + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const original = fs.readFileSync; + const read = kind === "read" ? spyOn(fs, "readFileSync").mockImplementation((...args: Parameters) => { + if (String(args[0]) === path) throw Object.assign(new Error("fixture read unavailable"), { code: "EACCES" }); + return original(...args); + }) : undefined; + if (kind === "parse") writeFileSync(path, "{incomplete-json"); + try { + run(); + } finally { + read?.mockRestore(); + writeFileSync(path, bytes); + } +} + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves degraded samples and the pinned warning", kind => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + withFailedStoreSnapshot(kind, () => { + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + } +}); + +test.each(["read", "parse"] as const)("a temporary %s failure does not erase or append to nineteen retained samples", kind => { + const cfg = config(); + const record = poolTerminal(cfg); + for (let index = 0; index < 19; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + withFailedStoreSnapshot(kind, () => { + noteCodexFailureWindowSample(cfg, "a", true, Date.now()); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves the recovery timer and retained generation", kind => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + const record = (failed: boolean, now: number) => recordCodexUpstreamOutcome(cfg, "a", failed ? 500 : 200, + { failureWindowCredentialGeneration: generation, now }); + for (let index = 0; index < 20; index++) record(true, NOW); + for (let index = 0; index < 20; index++) record(false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + withFailedStoreSnapshot(kind, () => expect(isCodexFailureWindowDegraded("a", NOW + 75_000)).toBe(true)); + expect(isCodexFailureWindowDegraded("a", NOW + 90_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 91_000)).toBe(false); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-failure" }); + const current = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 19; index++) recordCodexUpstreamOutcome(cfg, "a", 500, + { failureWindowCredentialGeneration: current, now: NOW + 92_000 }); + expect(isCodexFailureWindowDegraded("a", NOW + 92_000)).toBe(false); +}); + +test.each(["absent", "empty"] as const)("a valid %s snapshot still retires the previous generation", kind => { + const cfg = config(); + const old = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + const path = join(home, "codex-accounts.json"); + if (kind === "absent") unlinkSync(path); + else writeFileSync(path, "{}"); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("ok"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); +}); + +test.each(["[]", "false", "0", "null", "\"\"", "\"text\""])("a wrong-shaped top-level snapshot %s preserves the failure window", contents => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, contents); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, bytes); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + writeFileSync(path, bytes); + } +}); + +test("a replacement does not inherit the previous generation's recovery timer", () => { + const cfg = config(); + const firstGeneration = readCodexAccountRecord("a")!.generation; + const record = (outcome: number, generation: number, now: number) => recordCodexUpstreamOutcome(cfg, "a", outcome, + { failureWindowCredentialGeneration: generation, now }); + const fillGeneration = (generation: number, failed: boolean, now: number) => { + for (let index = 0; index < 20; index++) record(failed ? 500 : 200, generation, now); + }; + fillGeneration(firstGeneration, true, NOW); + fillGeneration(firstGeneration, false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + const credential = readCodexAccountRecord("a")!.credential!; + const nextGeneration = saveCodexAccountCredential("a", { ...credential, accessToken: "next-access" }); + fillGeneration(nextGeneration, false, NOW + 62_000); + expect(isCodexFailureWindowDegraded("a", NOW + 62_000)).toBe(false); + fillGeneration(nextGeneration, true, NOW + 63_000); + expect(isCodexFailureWindowDegraded("a", NOW + 63_000)).toBe(true); + // Only G2 successes remain in-window; its own 30-second recovery starts here. + fillGeneration(nextGeneration, false, NOW + 124_000); + expect(isCodexFailureWindowDegraded("a", NOW + 153_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 154_000)).toBe(false); +}); + test("fewer than 20 samples do not degrade an account", () => { fill("a", 19, 0); expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); diff --git a/tests/codex-integration/codex-main-rotation.test.ts b/tests/codex-integration/codex-main-rotation.test.ts index ffa98019c1e..bfc114c935d 100644 --- a/tests/codex-integration/codex-main-rotation.test.ts +++ b/tests/codex-integration/codex-main-rotation.test.ts @@ -21,6 +21,11 @@ import { reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests, } from "../../src/codex/account-lifecycle"; +import { + forgetCodexFailureWindows, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "../../src/codex/main-account"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { @@ -243,6 +248,29 @@ describe("main account rotation (Option A)", () => { } }); + test("a physical main account switch forgets its failure-window degradation", () => { + const config = makeConfig(); + const now = Date.now(); + for (let index = 0; index < 24; index += 1) { + noteCodexFailureWindowSample(config, MAIN_CODEX_ACCOUNT_ID, true, now - index); + } + try { + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(true); + // First reconcile observes the fixture identity; the physical switch happens on the + // replacement write below and must drop the alias-keyed window like every other + // runtime row the purge owns. + expect(reconcileMainCodexAccountRuntimeState()).toBe(false); + writeFileSync( + join(CODEX_DIR, "auth.json"), + JSON.stringify({ tokens: { access_token: "replacement_access", account_id: "replacement_acct" } }), + ); + expect(reconcileMainCodexAccountRuntimeState()).toBe(true); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(false); + } finally { + forgetCodexFailureWindows(); + } + }); + test("does not treat a transient missing auth file as an account switch", () => { expect(reconcileMainCodexAccountRuntimeState()).toBe(false); recordCodexUpstreamOutcome(makeConfig(), MAIN_CODEX_ACCOUNT_ID, 429, { retryAfter: "3600" });