From db36abc9e33dbe20b7581baaf8d0efaf954809fe Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 01:01:32 -0700 Subject: [PATCH 1/5] fix(codex): clear failure windows when account identities are removed Forget account-scoped failure samples and pinned degradation warnings during deletion and physical main-account replacement. Preserve sibling account windows and cover both lifecycle paths. --- src/codex/account-lifecycle.ts | 11 +++- structure/providers/openai-accounts.md | 2 +- .../codex-account-delete-atomicity.test.ts | 58 +++++++++++++++++++ .../codex-main-rotation.test.ts | 28 +++++++++ 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/src/codex/account-lifecycle.ts b/src/codex/account-lifecycle.ts index 65e35aa5f10..ffb7a5be540 100644 --- a/src/codex/account-lifecycle.ts +++ b/src/codex/account-lifecycle.ts @@ -14,6 +14,7 @@ import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account"; import { clearAccountQuota } from "./quota"; import { forgetManualPreference } from "./routing/active-account"; import { deleteAllHealthForAccount } from "./routing/health-store"; +import { forgetCodexFailureWindow } from "./routing/failure-window"; import { clearThreadAccountMapForAccount } from "./routing/thread-affinity"; import { clearCodexPoolRefreshFailure } from "./pool-refresh-backoff"; @@ -49,11 +50,15 @@ export function purgeCodexAccountRuntimeState(accountId: string): void { clearAccountNeedsReauth(accountId); clearAccountQuota(accountId); clearThreadAccountMapForAccount(accountId); - // Deliberately not via `./routing`: the account-delete path only needs the health rows and - // the manual preference gone, and importing the router here closed a cycle through - // routing/health-store and routing/active-account. + // Deliberately not via `./routing`: the account-delete path only needs the health rows + // and their sibling routing state gone, and importing the router barrel here closed a + // cycle through routing/health-store and routing/active-account. The leaf imports below + // carry every per-account forget that barrel helper used to perform — including the + // failure window, whose degrade and pin-warning state must not leak into a replacement + // account or a new physical login behind __main__. deleteAllHealthForAccount(accountId); forgetManualPreference(accountId); + forgetCodexFailureWindow(accountId); clearCodexPoolRefreshFailure(accountId); if (accountId === MAIN_CODEX_ACCOUNT_ID) { clearMainAccountInfoCache(); diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 809727ad810..93e330de4c8 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-account-delete-atomicity.test.ts b/tests/codex-integration/codex-account-delete-atomicity.test.ts index b1df61a9400..9ec8ae90591 100644 --- a/tests/codex-integration/codex-account-delete-atomicity.test.ts +++ b/tests/codex-integration/codex-account-delete-atomicity.test.ts @@ -32,6 +32,12 @@ import { getConfigPath, loadConfig, saveConfig } from "../../src/config"; import * as configModule from "../../src/config"; import { setCodexAccountAutoSwitchThresholdOverride } from "../../src/codex/account-auto-switch"; import { clearAllManualPreferences, manualPreferenceBlocks, resetCodexRoutingForManualSelection } from "../../src/codex/routing/active-account"; +import { + failureWindowSteersNewThreads, + forgetCodexFailureWindow, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { POOL_KEY_CODEX } from "../../src/oauth/pool-kernel"; import { prepareConfigObjectChildDeletionRebase } from "../../src/config/rebase-provenance"; import type { OcxConfig } from "../../src/types"; @@ -429,4 +435,56 @@ describe("Codex account delete persistence ordering", () => { clearAllManualPreferences(); } }); + + test("deleting the account forgets its failure-window degradation but keeps sibling state", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (accountId: string): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, accountId, true, now - index); + }; + degrade(ACCOUNT_ID); + degrade("sibling-account"); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(true); + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + try { + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(false); + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + // A different account's window is untouched by the deletion. + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + } finally { + forgetCodexFailureWindow(ACCOUNT_ID); + forgetCodexFailureWindow("sibling-account"); + } + }); + + test("deleting the account forgets its pinned-degrade warning", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, ACCOUNT_ID, true, now - index); + }; + forgetCodexFailureWindow(ACCOUNT_ID); + degrade(); + const warnSpy = spyOn(console, "warn").mockImplementation(() => {}); + try { + // The seeded config pins ACCOUNT_ID under the default "hold" policy, so steering + // logs the pin-degraded warning exactly once per process-lifetime of the state. + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + expect(warnSpy).toHaveBeenCalledTimes(1); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + // Deletion clears the pin as well; re-pin to model a recreated same-id account the + // operator selects again. If the warned set survived the purge, a re-degraded + // same-id pinned account would stay silent forever; the forget lets it fire again. + config.activeCodexAccountPinned = ACCOUNT_ID; + degrade(); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(2); + } finally { + warnSpy.mockRestore(); + forgetCodexFailureWindow(ACCOUNT_ID); + } + }); }); diff --git a/tests/codex-integration/codex-main-rotation.test.ts b/tests/codex-integration/codex-main-rotation.test.ts index ffa98019c1e..bfc114c935d 100644 --- a/tests/codex-integration/codex-main-rotation.test.ts +++ b/tests/codex-integration/codex-main-rotation.test.ts @@ -21,6 +21,11 @@ import { reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests, } from "../../src/codex/account-lifecycle"; +import { + forgetCodexFailureWindows, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "../../src/codex/main-account"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { @@ -243,6 +248,29 @@ describe("main account rotation (Option A)", () => { } }); + test("a physical main account switch forgets its failure-window degradation", () => { + const config = makeConfig(); + const now = Date.now(); + for (let index = 0; index < 24; index += 1) { + noteCodexFailureWindowSample(config, MAIN_CODEX_ACCOUNT_ID, true, now - index); + } + try { + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(true); + // First reconcile observes the fixture identity; the physical switch happens on the + // replacement write below and must drop the alias-keyed window like every other + // runtime row the purge owns. + expect(reconcileMainCodexAccountRuntimeState()).toBe(false); + writeFileSync( + join(CODEX_DIR, "auth.json"), + JSON.stringify({ tokens: { access_token: "replacement_access", account_id: "replacement_acct" } }), + ); + expect(reconcileMainCodexAccountRuntimeState()).toBe(true); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(false); + } finally { + forgetCodexFailureWindows(); + } + }); + test("does not treat a transient missing auth file as an account switch", () => { expect(reconcileMainCodexAccountRuntimeState()).toBe(false); recordCodexUpstreamOutcome(makeConfig(), MAIN_CODEX_ACCOUNT_ID, 429, { retryAfter: "3600" }); From d9547bff48b8168c2cc0bd24a863d79eeb72d920 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 02:18:11 -0700 Subject: [PATCH 2/5] fix(codex): fence late failure-window samples by dispatch identity Carry the main dispatch proof and stored Pool credential generation into terminal outcomes. Reject stale success and transient samples after identity or credential replacement, while preserving existing health and probe handling. Cover main identity ABA, same-ID Pool recreation, context reuse, and valid-current controls. --- src/codex/auth-context.ts | 4 + src/codex/routing.ts | 6 +- src/codex/routing/cooldown-math.ts | 5 + src/codex/routing/failure-window.ts | 23 +++ src/providers/openai-sidecar.ts | 5 + src/server/responses/compact.ts | 2 + src/server/responses/core-codex-account.ts | 35 ++-- src/server/responses/passthrough-delivery.ts | 2 + src/server/responses/passthrough-dispatch.ts | 2 + structure/providers/openai-accounts.md | 2 +- .../codex-failure-window.test.ts | 154 +++++++++++++++++- 11 files changed, 219 insertions(+), 21 deletions(-) diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index c4cd30b325d..7e1cd3548e7 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -324,6 +324,8 @@ export type CodexAuthContext = writerGeneration: number; /** Captured before async credential work; never reconstructed after the upstream response. */ mainQuotaWriter?: MainQuotaWriter; + /** Immutable credential proof captured while materializing this physical attempt's headers. */ + mainQuotaDispatch?: MainQuotaDispatch; reserveAuthorization?: MainReserveAuthorization; accessToken: string; chatgptAccountId: string; @@ -1766,6 +1768,8 @@ export function materializeCodexUpstreamAuth( if (ctx.kind === "main-pool") { ctx.mainQuotaWriter = observeSelectedMainCredential(ctx, ctx.mainQuotaWriter); assertMainAccountPolicy(options.config); + ctx.mainQuotaDispatch = ctx.mainQuotaWriter && isMainQuotaWriterLive(ctx.mainQuotaWriter) + ? selectedMainQuotaDispatch(selected) : undefined; } assertMaterializedReserve(selected, ctx, options); return selected; diff --git a/src/codex/routing.ts b/src/codex/routing.ts index 7eefb87e4e2..57bf246ec45 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -23,7 +23,7 @@ import { failureWindowSteersNewThreads, forgetCodexFailureWindow, forgetCodexFailureWindows, - noteCodexFailureWindowSample, + noteCodexFailureWindowOutcome, settleCodexSuccessStreak, } from "./routing/failure-window"; @@ -1305,7 +1305,7 @@ export function recordCodexUpstreamOutcome( */ dropSpentCredentialFailure(accountId); if (outcomeClass === "success") { - noteCodexFailureWindowSample(config, accountId, false, now); + noteCodexFailureWindowOutcome(config, accountId, false, now, meta); // The operator's one-shot is spent by a dispatch that actually worked, and only by that. // A failed lookup leaves it unspent so the intent survives the failure. consumeManualPreference(accountId, codexPoolKeyForScope(quotaScope)); @@ -1555,7 +1555,7 @@ export function recordCodexUpstreamOutcome( // A transient failure concludes an owning probe; an unrelated 5xx must not // consume someone else's live lease or drop hard-cooldown bookkeeping (#433). const transientBase = ownsProbeLease(current, meta) ? withProbeLeaseReleased(current!, now) : current; - noteCodexFailureWindowSample(config, accountId, true, now); + noteCodexFailureWindowOutcome(config, accountId, true, now, meta); const stale = current?.lastFailureAt ? now - current.lastFailureAt > CODEX_FAILURE_WINDOW_MS : false; const hardCooldownUntil = getCodexAccountCooldownUntil(accountId, now) ?? undefined; // Soft avoid + affinity clears are part of failover. When threshold is 0, leave diff --git a/src/codex/routing/cooldown-math.ts b/src/codex/routing/cooldown-math.ts index 3e03e029aae..515b4d9eb22 100644 --- a/src/codex/routing/cooldown-math.ts +++ b/src/codex/routing/cooldown-math.ts @@ -6,6 +6,7 @@ import { isThirtyDayOnlyCodexPlan } from "../plan"; import { hasSpendableCodexCredits, isTerminalShortWindow, type CodexSpendableCredits } from "../quota-types"; import type { CodexQuotaScope } from "./health-store"; import type { TransientProbeGrant } from "./thread-affinity"; +import type { MainQuotaDispatch } from "../main-account-cache"; export { TERMINAL_SHORT_WINDOW_FRESHNESS_MS } from "../quota-types"; @@ -99,6 +100,10 @@ export type CodexUpstreamOutcomeMeta = { * means the caller cannot supply lineage and the historical unfenced handling stands. */ credentialGeneration?: number; + /** Main credential proof captured before dispatch; null means no owned proof was available. */ + mainFailureWindowDispatch?: MainQuotaDispatch | null; + /** Pool sample provenance retained by a terminal callback even if its context is reused. */ + failureWindowCredentialGeneration?: number; }; export function computeCodexUsageScore(quota: { diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index 71aa43ebee0..f0cb2c4967f 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -1,5 +1,9 @@ import { pinnedCodexAccountId } from "../account-priority"; import type { OcxConfig } from "../../types"; +import { isCodexAccountGenerationLive } from "../account-store"; +import { isMainQuotaDispatchLive } from "../main-account-cache"; +import type { CodexAuthContext } from "../auth-context"; +import type { CodexUpstreamOutcomeMeta } from "./cooldown-math"; import { deleteAccountHealth, setAccountHealth, @@ -20,6 +24,25 @@ type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number const windows = new Map(); const pinnedDegradeWarned = new Set(); +/** Read only dispatch-time provenance retained by auth materialization, never recapture at arrival. */ +export function codexFailureWindowOutcomeMeta( + context: CodexAuthContext, +): Pick { + if (context.kind === "pool") return { failureWindowCredentialGeneration: context.generation }; + return context.kind === "main-pool" ? { mainFailureWindowDispatch: context.mainQuotaDispatch ?? null } : {}; +} + +/** Fence window samples only; other health and recovery-lease semantics stay with their owners. */ +export function noteCodexFailureWindowOutcome( + config: OcxConfig, accountId: string, failed: boolean, now: number, meta: CodexUpstreamOutcomeMeta, +): void { + if (meta.mainFailureWindowDispatch !== undefined + && (!meta.mainFailureWindowDispatch || !isMainQuotaDispatchLive(meta.mainFailureWindowDispatch))) return; + const generation = meta.failureWindowCredentialGeneration ?? meta.credentialGeneration; + if (generation !== undefined && !isCodexAccountGenerationLive(accountId, generation)) return; + noteCodexFailureWindowSample(config, accountId, failed, now); +} + export function codexFailureWindowEnabled(config: OcxConfig): boolean { return config.codexFailureWindow !== false; } diff --git a/src/providers/openai-sidecar.ts b/src/providers/openai-sidecar.ts index c68c0c73e7b..93e3398ae18 100644 --- a/src/providers/openai-sidecar.ts +++ b/src/providers/openai-sidecar.ts @@ -12,6 +12,7 @@ import { type CodexAuthPolicyConfig, } from "../codex/auth-context"; import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome } from "../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../codex/routing/failure-window"; import { inspectChatGptDomainClaim } from "../oauth/chatgpt"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential, type DataPlaneAdmission } from "../server/auth-cors"; import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "../types"; @@ -205,6 +206,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); throw new CodexPoolAuthenticationError("Selected Codex account is unavailable"); } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -220,6 +222,7 @@ export async function resolveFirstUsableOpenAiSidecar( probeLeaseId: authContext.probeLeaseId, probeQuotaScope: authContext.probeQuotaScope, writerGeneration: authContext.writerGeneration, + ...windowMeta, // 401/403 here is evidence about this exact stored credential; without the generation a // replacement inherits the quarantine (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), @@ -256,6 +259,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); continue; } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -271,6 +275,7 @@ export async function resolveFirstUsableOpenAiSidecar( threadId: authContext.affinityKey, probeLeaseId: authContext.probeLeaseId, writerGeneration: authContext.writerGeneration, + ...windowMeta, // Same fence as the exact-account recorder above (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), }, diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index cb93a79f280..32263104a72 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -88,6 +88,7 @@ import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome, } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountChangeConversationStateScrub, conversationStateBindingFromAuth, @@ -1062,6 +1063,7 @@ export async function handleResponsesCompact( if ((outcome === 401 || outcome === 403) && upstream && wasPermanentAccountRefusal(upstream)) return; recordCodexUpstreamOutcome(config, ctx.accountId, outcome, { ...meta, + ...codexFailureWindowOutcomeMeta(ctx), threadId: ctx.kind === "pool" || ctx.kind === "main-pool" ? ctx.affinityKey : undefined, fixedAccount: ctx.fixedAccount, modelId: selectedModelId, diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index a2b35f7da73..b51b7ce32c1 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -11,6 +11,7 @@ import { claimMainQuotaDispatchForWs, isMainQuotaDispatchLive, renewMainQuotaDis import type { CodexWsQuotaObserver } from "./codex-ws-metadata"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isCodexAccountGenerationLive } from "../../codex/account-store"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountQuotaFromUpstreamHeaders as applyCapturedCodexQuota } from "../../codex/quota"; import type { RouteResult } from "../../router"; import { @@ -93,20 +94,19 @@ export function sidecarOutcomeRecorder( config: OcxConfig, authCtx: CodexAuthContext, ): ((outcome: CodexUpstreamOutcome) => void) | undefined { - return authCtx.kind === "pool" || authCtx.kind === "main-pool" - ? outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { - threadId: authCtx.affinityKey, - fixedAccount: authCtx.fixedAccount, - probeLeaseId: authCtx.probeLeaseId, - probeQuotaScope: authCtx.probeQuotaScope, - writerGeneration: authCtx.writerGeneration, - // A vision or web-search sidecar can return 401/403, and that is evidence about the exact - // stored credential it used. Without the generation it becomes an account-wide quarantine - // that a replacement inherits (#2892 gap 4). `main-pool` has no stored-record generation, so - // it keeps the unfenced account-wide semantics. - ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), - }) - : undefined; + if (authCtx.kind !== "pool" && authCtx.kind !== "main-pool") return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); + return outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, + threadId: authCtx.affinityKey, + fixedAccount: authCtx.fixedAccount, + probeLeaseId: authCtx.probeLeaseId, + probeQuotaScope: authCtx.probeQuotaScope, + writerGeneration: authCtx.writerGeneration, + // Quarantine retains its stored-credential fence. main-pool has no stored-record + // generation; its separate dispatch proof above fences only failure-window samples. + ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), + }); } @@ -148,6 +148,7 @@ export function recordCodexPreOutputRefusalOutcome( if (!usesCodexForwardPoolAuth(authCtx, provider) || wasPermanentAccountRefusal(response)) return; noteCodexWsPreOutputRefusalOutcome(response); recordCodexUpstreamOutcome(config, authCtx.accountId, response.status, { + ...codexFailureWindowOutcomeMeta(authCtx), threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -676,6 +677,7 @@ export async function retryCodexPoolOnAlternateAccount( if (codexWsPreOutputRefusalOutcomeRecorded(firstResponse)) return; noteCodexWsPreOutputRefusalOutcome(firstResponse); recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, fixedAccount: firstAuthCtx.fixedAccount, modelId: route.modelId, @@ -693,6 +695,7 @@ export async function retryCodexPoolOnAlternateAccount( if (outcomeStatus === firstResponse.status || (outcomeStatus !== 429 && outcomeStatus !== 402)) return; recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...codexQuotaOutcomeMeta(firstResponse), + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -862,6 +865,7 @@ export async function retryCodexPoolOnAlternateAccount( } recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...quotaMeta, + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -1149,6 +1153,7 @@ export function codexForwardTerminalOutcomeRecorder( logCtx?: RequestLogContext, ): ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined { if (!usesCodexForwardPoolAuth(authCtx, provider)) return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); return (status, httpStatusOverride) => { const quotaStatus = [httpStatusOverride, logCtx?.terminalHttpStatus] .find(value => value === 429 || value === 402); @@ -1157,6 +1162,7 @@ export function codexForwardTerminalOutcomeRecorder( // request. Don't penalize account health; record success to clear any // prior soft-avoid so a healthy account isn't stuck avoided. recordCodexUpstreamOutcome(config, authCtx.accountId, 200, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -1180,6 +1186,7 @@ export function codexForwardTerminalOutcomeRecorder( ? 200 : (quotaStatus ?? httpStatusOverride ?? logCtx?.terminalHttpStatus ?? 502); recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 3d286dec672..cac782e0e28 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -37,6 +37,7 @@ import type { ResponsesTerminalStatus } from "../../bridge"; import { isCodexWsQuotaObservedResponse, isCodexWsUpstreamResponse, isCodexWsPreludeProjection } from "./ws-upstream"; import { recordSubagentQuotaFailureForThreadSpawn } from "../../codex/subagent-model-fallback"; import { recordCodexUpstreamOutcome } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { codexProbeLeaseId, codexProbeQuotaScope, codexTransientProbeGrant, releaseCodexAuthContextProbeLease } from "../../codex/auth-context"; import { consumeComboFailure } from "./core-combo-failure"; import { clientCancelledResponse, readDisplaySafeErrorText } from "./core-errors"; @@ -535,6 +536,7 @@ export async function deliverPassthroughResponse( // request. The sibling recorders on this path already decline: the terminal recorder // needs an ok streaming body, and the quota-header snapshot finds no quota headers. recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, upstreamResponse.status, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), ...quotaMeta, threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 849001a6798..c63bf32697b 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -18,6 +18,7 @@ import type { ResponsesEffects } from "./response-effects"; import type { ResponsesSendBudget } from "./request-send-budget"; import { transientSendCapFor } from "./request-send-budget"; import { renewMainQuotaDispatchForAttempt } from "../../codex/main-account-cache"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isLocalUpstream } from "../../lib/local-upstream"; import { codexSafetyBufferingFilterOptions, terminalStatusFromParsed } from "../relay"; @@ -990,6 +991,7 @@ export async function preparePassthroughExchange( } if (usesCodexForwardPoolAuth(admissionState.authCtx, route.provider)) { recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, outcome, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, modelId: route.modelId, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 93e330de4c8..a35985819b5 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 55e5e773b24..3ab8478150b 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,8 +1,14 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import { headersForCodexAuthContext, type CodexAuthContext } from "../../src/codex/auth-context"; +import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { captureMainQuotaWriter, clearMainAccountInfoCache } from "../../src/codex/main-account-cache"; +import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/account-id"; +import { captureConfigGeneration } from "../../src/lib/state-store-sweeper"; +import { codexForwardTerminalOutcomeRecorder, sidecarOutcomeRecorder } from "../../src/server/responses/core-codex-account"; import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../../src/codex/auth-api"; import { clearCodexUpstreamHealth, @@ -16,7 +22,7 @@ import { noteCodexFailureWindowSample, } from "../../src/codex/routing/failure-window"; import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; -import type { OcxConfig } from "../../src/types"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const NOW = 1_700_000_000_000; @@ -47,13 +53,19 @@ function fill(accountId: string, failures: number, successes: number, now = NOW) let home = ""; let previousHome: string | undefined; +let previousCodexHome: string | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-window-")); setIcaclsRunnerForTests(() => ICACLS_OK); setAsyncIcaclsRunnerForTests(async () => ICACLS_OK); process.env.OPENCODEX_HOME = home; + process.env.CODEX_HOME = join(home, "codex"); + mkdirSync(process.env.CODEX_HOME, { recursive: true }); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); @@ -75,11 +87,147 @@ afterEach(async () => { clearThreadAccountMap(); clearAccountQuota(); forgetCodexFailureWindows(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; await removeTreeWithRetry(home); }); +const forwardProvider = { adapter: "openai-responses", authMode: "forward" } as OcxProviderConfig; + +function terminal(config: OcxConfig, context: CodexAuthContext) { + headersForCodexAuthContext(new Headers(), context, config); + return codexForwardTerminalOutcomeRecorder(config, context, forwardProvider)!; +} + +function mainTerminal(config: OcxConfig, identity: string, accessToken = `access-${identity}`) { + writeFileSync(join(process.env.CODEX_HOME!, "auth.json"), JSON.stringify({ tokens: { access_token: accessToken, account_id: identity } })); + reconcileMainCodexAccountRuntimeState(); + return terminal(config, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + mainQuotaWriter: captureMainQuotaWriter(identity), accessToken, chatgptAccountId: identity, fixedAccount: true, + }); +} + +function poolTerminal(config: OcxConfig) { + const record = readCodexAccountRecord("a")!; + return terminal(config, { + kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), generation: record.generation, + accessToken: record.credential!.accessToken, chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true, + }); +} + +test("late terminal outcomes cannot refill a replacement main failure window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const replacement = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) replacement("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + replacement("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a returned main identity does not revive the first login's terminal writer", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const first = mainTerminal(cfg, "physical-a"); + mainTerminal(cfg, "physical-b"); + const returned = mainTerminal(cfg, "physical-a"); + for (let index = 0; index < 20; index++) first("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) returned("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("late successful terminals cannot count toward the replacement main window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const current = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + old("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("sidecar recorder retains the old main dispatch even if its context is reused", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const context: CodexAuthContext = { kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, + writerGeneration: captureConfigGeneration(), mainQuotaWriter: captureMainQuotaWriter("physical-a"), + accessToken: "access-physical-a", chatgptAccountId: "physical-a" }; + headersForCodexAuthContext(new Headers(), context, cfg); + const old = sidecarOutcomeRecorder(cfg, context)!; + const current = mainTerminal(cfg, "physical-b"); + context.mainQuotaWriter = captureMainQuotaWriter("physical-b"); + context.accessToken = "access-physical-b"; + context.chatgptAccountId = "physical-b"; + headersForCodexAuthContext(new Headers(), context, cfg); + for (let index = 0; index < 20; index++) old(500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a replaced main credential retires its terminal samples without an identity change", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a", "old-access"); + const current = mainTerminal(cfg, "physical-a", "new-access"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + current("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("same-id pool recreation rejects the deleted credential's late terminal samples", () => { + const cfg = config(); + const old = poolTerminal(cfg); + deleteCodexAccount(cfg, "a"); + saveCodexAccountCredential("a", { accessToken: "replacement-access", refreshToken: "replacement-refresh", chatgptAccountId: "replacement-account", expiresAt: Date.now() + 60_000 }); + cfg.codexAccounts!.push({ id: "a", email: "replacement@test", isMain: false }); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("terminal pool samples keep the dispatched generation if the context is reused", () => { + const cfg = config(); + const record = readCodexAccountRecord("a")!; + const context: CodexAuthContext = { kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), + generation: record.generation, accessToken: record.credential!.accessToken, + chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true }; + const old = terminal(cfg, context); + context.generation = saveCodexAccountCredential("a", { ...record.credential!, accessToken: "new-access" }); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a main terminal without an owned dispatch proof cannot seed the window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const record = codexForwardTerminalOutcomeRecorder(cfg, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + accessToken: "unobserved-access", chatgptAccountId: "physical-a", + }, forwardProvider)!; + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); +}); + test("fewer than 20 samples do not degrade an account", () => { fill("a", 19, 0); expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); From 8cd4a5ba46ef4b58a19368afd7b2c90b0ead45fd Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 02:59:34 -0700 Subject: [PATCH 3/5] fix(codex): revalidate tagged failure samples after credential replacement Retain each stored Pool sample's dispatch credential generation and filter it against one current store snapshot per insertion or lookup. Reset retired-generation degradation, recovery timers and pin warnings without clearing a replacement on stale arrival. Cover both unlocked-read boundaries, derived state and the 512-sample snapshot bound. --- src/codex/routing/failure-window.ts | 45 ++++-- structure/providers/openai-accounts.md | 2 +- .../codex-failure-window.test.ts | 129 +++++++++++++++++- 3 files changed, 163 insertions(+), 13 deletions(-) diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index f0cb2c4967f..4610867e4b7 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -1,6 +1,6 @@ import { pinnedCodexAccountId } from "../account-priority"; import type { OcxConfig } from "../../types"; -import { isCodexAccountGenerationLive } from "../account-store"; +import { beginCodexAccountGenerationLiveCheck, isCodexAccountGenerationLive } from "../account-store"; import { isMainQuotaDispatchLive } from "../main-account-cache"; import type { CodexAuthContext } from "../auth-context"; import type { CodexUpstreamOutcomeMeta } from "./cooldown-math"; @@ -18,8 +18,8 @@ export const CODEX_SLIDING_FAILURE_RECOVER_RATIO = 0.1; export const CODEX_SLIDING_FAILURE_RECOVER_SUSTAIN_MS = 30_000; const MAX_SAMPLES = 512; -type Sample = { at: number; failed: boolean }; -type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number }; +type Sample = { at: number; failed: boolean; credentialGeneration?: number }; +type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number; credentialGeneration?: number }; const windows = new Map(); const pinnedDegradeWarned = new Set(); @@ -40,7 +40,7 @@ export function noteCodexFailureWindowOutcome( && (!meta.mainFailureWindowDispatch || !isMainQuotaDispatchLive(meta.mainFailureWindowDispatch))) return; const generation = meta.failureWindowCredentialGeneration ?? meta.credentialGeneration; if (generation !== undefined && !isCodexAccountGenerationLive(accountId, generation)) return; - noteCodexFailureWindowSample(config, accountId, failed, now); + appendSample(config, accountId, failed, now, generation); } export function codexFailureWindowEnabled(config: OcxConfig): boolean { @@ -56,11 +56,25 @@ function stateFor(accountId: string): WindowState { return state; } -function recompute(state: WindowState, now: number): void { - const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; - if (state.samples.length > 0 && state.samples[0]!.at <= cutoff) { - state.samples = state.samples.filter(sample => sample.at > cutoff); +function recompute( + accountId: string, state: WindowState, now: number, + generationLive?: ReturnType, +): void { + // One store snapshot per insertion/read, never one file load per retained sample. The + // unlocked insertion precheck can race another process; provenance must survive it. + const tagged = state.credentialGeneration !== undefined + || state.samples.some(sample => sample.credentialGeneration !== undefined); + const live = generationLive ?? (tagged ? beginCodexAccountGenerationLiveCheck() : undefined); + if (state.credentialGeneration !== undefined && !live?.(accountId, state.credentialGeneration)) { + state.degraded = false; + state.healthySince = undefined; + state.credentialGeneration = undefined; + pinnedDegradeWarned.delete(accountId); } + const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; + state.samples = state.samples.filter(sample => sample.at > cutoff + && (sample.credentialGeneration === undefined || live?.(accountId, sample.credentialGeneration))); + state.credentialGeneration ??= state.samples.find(sample => sample.credentialGeneration !== undefined)?.credentialGeneration; const samples = state.samples.length; const failed = state.samples.reduce((count, sample) => count + (sample.failed ? 1 : 0), 0); const degraded = samples >= CODEX_SLIDING_FAILURE_MIN_SAMPLES @@ -94,19 +108,28 @@ export function noteCodexFailureWindowSample( accountId: string, failed: boolean, now: number, +): void { + appendSample(config, accountId, failed, now); +} + +function appendSample( + config: OcxConfig, accountId: string, failed: boolean, now: number, credentialGeneration?: number, ): void { if (!codexFailureWindowEnabled(config)) return; + const live = credentialGeneration === undefined ? undefined : beginCodexAccountGenerationLiveCheck(); + // A stale arrival is not an ownership transition and must not delete a live replacement's state. + if (credentialGeneration !== undefined && !live?.(accountId, credentialGeneration)) return; const state = stateFor(accountId); - state.samples.push({ at: now, failed }); + state.samples.push({ at: now, failed, ...(credentialGeneration === undefined ? {} : { credentialGeneration }) }); if (state.samples.length > MAX_SAMPLES) state.samples.splice(0, state.samples.length - MAX_SAMPLES); - recompute(state, now); + recompute(accountId, state, now, live); if (!state.degraded) pinnedDegradeWarned.delete(accountId); } export function isCodexFailureWindowDegraded(accountId: string, now = Date.now()): boolean { const state = windows.get(accountId); if (!state) return false; - recompute(state, now); + recompute(accountId, state, now); if (!state.degraded) pinnedDegradeWarned.delete(accountId); return state.degraded; } diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index a35985819b5..1926e916f6d 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 3ab8478150b..c48eeea7a9d 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,8 +1,9 @@ -import { afterEach, beforeEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import * as accountStore from "../../src/codex/account-store"; import { headersForCodexAuthContext, type CodexAuthContext } from "../../src/codex/auth-context"; import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; import { captureMainQuotaWriter, clearMainAccountInfoCache } from "../../src/codex/main-account-cache"; @@ -18,6 +19,7 @@ import { } from "../../src/codex/routing"; import { forgetCodexFailureWindows, + failureWindowSteersNewThreads, isCodexFailureWindowDegraded, noteCodexFailureWindowSample, } from "../../src/codex/routing/failure-window"; @@ -228,6 +230,131 @@ test("a main terminal without an owned dispatch proof cannot seed the window", ( expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); }); +test("a cross-process replacement after the unlocked live check cannot donate a terminal sample", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.isCodexAccountGenerationLive; + let swapped = false; + const liveCheck = spyOn(accountStore, "isCodexAccountGenerationLive").mockImplementation((id, generation) => { + const live = original(id, generation); + if (id === "a" && live && !swapped) { + swapped = true; + const credential = readCodexAccountRecord(id)!.credential!; + saveCodexAccountCredential(id, { ...credential, accessToken: "replacement-after-read" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + liveCheck.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a replacement does not inherit degraded state or the prior pinned warning", () => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const old = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-access" }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(2); + // Rejecting a stale writer must not erase the replacement's healthy or degraded window. + old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + } finally { + warn.mockRestore(); + } +}); + +test("a replacement after the insertion snapshot is filtered at window read time", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.beginCodexAccountGenerationLiveCheck; + let swapped = false; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(() => { + const live = original(); + if (!swapped) { + swapped = true; + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-snapshot" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + snapshot.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a full tagged window loads one current credential snapshot per read", () => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 512; index++) { + recordCodexUpstreamOutcome(cfg, "a", 200, { failureWindowCredentialGeneration: generation, now: NOW }); + } + const original = accountStore.beginCodexAccountGenerationLiveCheck; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(original); + try { + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "next-full-window" }); + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(2); + } finally { + snapshot.mockRestore(); + } +}); + +test("a replacement does not inherit the previous generation's recovery timer", () => { + const cfg = config(); + const firstGeneration = readCodexAccountRecord("a")!.generation; + const record = (outcome: number, generation: number, now: number) => recordCodexUpstreamOutcome(cfg, "a", outcome, + { failureWindowCredentialGeneration: generation, now }); + const fillGeneration = (generation: number, failed: boolean, now: number) => { + for (let index = 0; index < 20; index++) record(failed ? 500 : 200, generation, now); + }; + fillGeneration(firstGeneration, true, NOW); + fillGeneration(firstGeneration, false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + const credential = readCodexAccountRecord("a")!.credential!; + const nextGeneration = saveCodexAccountCredential("a", { ...credential, accessToken: "next-access" }); + fillGeneration(nextGeneration, false, NOW + 62_000); + expect(isCodexFailureWindowDegraded("a", NOW + 62_000)).toBe(false); + fillGeneration(nextGeneration, true, NOW + 63_000); + expect(isCodexFailureWindowDegraded("a", NOW + 63_000)).toBe(true); + // Only G2 successes remain in-window; its own 30-second recovery starts here. + fillGeneration(nextGeneration, false, NOW + 124_000); + expect(isCodexFailureWindowDegraded("a", NOW + 153_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 154_000)).toBe(false); +}); + test("fewer than 20 samples do not degrade an account", () => { fill("a", 19, 0); expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); From 230e8040bc7eb0fffe8d16e1a0d727dfb6f27647 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 04:00:41 -0700 Subject: [PATCH 4/5] fix(codex): preserve failure windows when credential snapshots fail Distinguish failed reads or parsing from valid absent/empty credential stores. Preserve retained samples and derived failure-window state while evidence is unknown, keeping existing boolean liveness callers compatible. Cover fixture read/parse failures, recovery and empty-store retirement without adding snapshot reuse optimization. --- src/codex/account-store.ts | 34 +++++-- src/codex/routing/failure-window.ts | 8 +- structure/providers/openai-accounts.md | 2 +- .../codex-failure-window.test.ts | 88 ++++++++++++++++++- 4 files changed, 121 insertions(+), 11 deletions(-) diff --git a/src/codex/account-store.ts b/src/codex/account-store.ts index bfe677c2d8e..b89047c8919 100644 --- a/src/codex/account-store.ts +++ b/src/codex/account-store.ts @@ -136,10 +136,17 @@ export function peekCodexAccountRecordStore(): Record boolean) + & { readonly snapshotStatus: "ok" | "failed" }; + /** * The same verdict as {@link isCodexAccountGenerationLive}, over ONE store load. * @@ -409,12 +424,15 @@ export function isCodexAccountGenerationLive(id: string, generation: number): bo * several accounts in one synchronous pass on the request path, so it opens one checker and * closes over the snapshot instead. */ -export function beginCodexAccountGenerationLiveCheck(): (id: string, generation: number) => boolean { - const snapshot = loadCodexAccountRecordSnapshot(); - return (id, generation) => { +export function beginCodexAccountGenerationLiveCheck(): CodexAccountGenerationLiveCheck { + let result: CodexAccountRecordStoreResult; + try { result = loadCodexAccountRecordStoreResult(); } + catch { result = { status: "failed" }; } + const snapshot = result.status === "ok" ? result.records : {}; + return Object.assign((id: string, generation: number) => { const record = snapshot[id]; return !!record?.credential && record.deletedAt == null && record.generation === generation; - }; + }, { snapshotStatus: result.status }); } export function saveCodexAccountCredentialIfGeneration( diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index 4610867e4b7..61ae76149be 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -65,6 +65,8 @@ function recompute( const tagged = state.credentialGeneration !== undefined || state.samples.some(sample => sample.credentialGeneration !== undefined); const live = generationLive ?? (tagged ? beginCodexAccountGenerationLiveCheck() : undefined); + // Unknown evidence is not a credential removal: preserve samples and all derived state. + if (live?.snapshotStatus === "failed") return; if (state.credentialGeneration !== undefined && !live?.(accountId, state.credentialGeneration)) { state.degraded = false; state.healthySince = undefined; @@ -116,7 +118,11 @@ function appendSample( config: OcxConfig, accountId: string, failed: boolean, now: number, credentialGeneration?: number, ): void { if (!codexFailureWindowEnabled(config)) return; - const live = credentialGeneration === undefined ? undefined : beginCodexAccountGenerationLiveCheck(); + const existing = windows.get(accountId); + const tagged = credentialGeneration !== undefined || existing?.credentialGeneration !== undefined + || existing?.samples.some(sample => sample.credentialGeneration !== undefined); + const live = tagged ? beginCodexAccountGenerationLiveCheck() : undefined; + if (live?.snapshotStatus === "failed") return; // A stale arrival is not an ownership transition and must not delete a live replacement's state. if (credentialGeneration !== undefined && !live?.(accountId, credentialGeneration)) return; const state = stateFor(accountId); diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 1926e916f6d..09c86dd0f16 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. A snapshot read or parse failure is explicitly unknown and preserves the samples, degraded flag, recovery timer, retained generation and pin warning; a genuinely absent or valid empty store still retires its old generations. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index c48eeea7a9d..6a3a123ae7e 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import * as fs from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; @@ -332,6 +333,91 @@ test("a full tagged window loads one current credential snapshot per read", () = } }); +function withFailedStoreSnapshot(kind: "read" | "parse", run: () => void): void { + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const original = fs.readFileSync; + const read = kind === "read" ? spyOn(fs, "readFileSync").mockImplementation((...args: Parameters) => { + if (String(args[0]) === path) throw Object.assign(new Error("fixture read unavailable"), { code: "EACCES" }); + return original(...args); + }) : undefined; + if (kind === "parse") writeFileSync(path, "{incomplete-json"); + try { + run(); + } finally { + read?.mockRestore(); + writeFileSync(path, bytes); + } +} + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves degraded samples and the pinned warning", kind => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + withFailedStoreSnapshot(kind, () => { + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + } +}); + +test.each(["read", "parse"] as const)("a temporary %s failure does not erase or append to nineteen retained samples", kind => { + const cfg = config(); + const record = poolTerminal(cfg); + for (let index = 0; index < 19; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + withFailedStoreSnapshot(kind, () => { + noteCodexFailureWindowSample(cfg, "a", true, Date.now()); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves the recovery timer and retained generation", kind => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + const record = (failed: boolean, now: number) => recordCodexUpstreamOutcome(cfg, "a", failed ? 500 : 200, + { failureWindowCredentialGeneration: generation, now }); + for (let index = 0; index < 20; index++) record(true, NOW); + for (let index = 0; index < 20; index++) record(false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + withFailedStoreSnapshot(kind, () => expect(isCodexFailureWindowDegraded("a", NOW + 75_000)).toBe(true)); + expect(isCodexFailureWindowDegraded("a", NOW + 90_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 91_000)).toBe(false); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-failure" }); + const current = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 19; index++) recordCodexUpstreamOutcome(cfg, "a", 500, + { failureWindowCredentialGeneration: current, now: NOW + 92_000 }); + expect(isCodexFailureWindowDegraded("a", NOW + 92_000)).toBe(false); +}); + +test.each(["absent", "empty"] as const)("a valid %s snapshot still retires the previous generation", kind => { + const cfg = config(); + const old = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + const path = join(home, "codex-accounts.json"); + if (kind === "absent") unlinkSync(path); + else writeFileSync(path, "{}"); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("ok"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); +}); + test("a replacement does not inherit the previous generation's recovery timer", () => { const cfg = config(); const firstGeneration = readCodexAccountRecord("a")!.generation; From 49309523d283719760d6c498b55f8fa4dd2fff0a Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 04:31:21 -0700 Subject: [PATCH 5/5] fix(codex): reject non-object credential snapshot shapes Require a non-null, non-array object before normalizing store rows. Preserve failure-window evidence for invalid top-level arrays or scalars while keeping missing files and empty objects valid. Cover six top-level shapes without changing row normalization. --- src/codex/account-store.ts | 1 + .../codex-failure-window.test.ts | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/codex/account-store.ts b/src/codex/account-store.ts index b89047c8919..b384ab5588b 100644 --- a/src/codex/account-store.ts +++ b/src/codex/account-store.ts @@ -150,6 +150,7 @@ function loadCodexAccountRecordStoreResult(): CodexAccountRecordStoreResult { const normalized: CodexAccountStore = {}; try { const raw = JSON.parse(readFileSync(path, "utf-8")) as RawCodexAccountStore; + if (!isObject(raw)) throw new Error("invalid Codex account store shape"); for (const [id, value] of Object.entries(raw)) { const record = normalizeRecord(value); if (record) normalized[id] = record; diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 6a3a123ae7e..ee16af342df 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -418,6 +418,32 @@ test.each(["absent", "empty"] as const)("a valid %s snapshot still retires the p expect(isCodexFailureWindowDegraded("a")).toBe(false); }); +test.each(["[]", "false", "0", "null", "\"\"", "\"text\""])("a wrong-shaped top-level snapshot %s preserves the failure window", contents => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, contents); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, bytes); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + writeFileSync(path, bytes); + } +}); + test("a replacement does not inherit the previous generation's recovery timer", () => { const cfg = config(); const firstGeneration = readCodexAccountRecord("a")!.generation;