From f9e7b9358594654007e84633fc0236f5f5e605f4 Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 19 Oct 2026 13:12:40 +0000 Subject: [PATCH 01/17] fix(codex): renew opted-in pool credit evidence on the existing sweep --- .../docs/reference/configuration/providers.md | 3 + src/codex/auth-api/pool-mode-gate.ts | 25 +++++- structure/INDEX.md | 2 + .../decisions/ADR-7377-pool-credit-renewal.md | 15 ++++ structure/manifest.json | 7 ++ structure/providers/openai-accounts.md | 2 +- structure/providers/openai-tiers.md | 2 +- structure/providers/pool-credit-renewal.md | 24 ++++++ .../codex-quota-prime.test.ts | 82 +++++++++++++++++++ 9 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 structure/decisions/ADR-7377-pool-credit-renewal.md create mode 100644 structure/providers/pool-credit-renewal.md diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 62bb5f93c0f..7065600ad25 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -106,6 +106,9 @@ separate. Full request URLs such as `/api/v1/responses` are not provider base UR | `cacheRetention?` | `"none" \| "short" \| "long"` | first-party Anthropic: `"long"`; other Anthropic-compatible endpoints: `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. When unset, requests to `api.anthropic.com` (API key or the OAuth account pools) use the 1-hour TTL, because agentic turns often pause longer than five minutes and a 1-hour write (2x base input instead of 1.25x) pays for itself on the first reuse; Anthropic-compatible gateways and vendor endpoints keep the 5-minute TTL because their 1-hour support is undocumented. On native `/v1/messages` passthrough, caller breakpoints without a `ttl` get `"1h"` unless the request pins any breakpoint to `"5m"`, in which case it is forwarded unchanged. On a translated `/v1/messages` route, caller breakpoints that all ask for `ttl: "1h"` mean `"long"` and any explicit `"5m"` means `"short"`. An explicit value always wins. Changing the TTL makes each open conversation write its cache once more, since 5-minute and 1-hour entries are separate. Cost estimates price writes reported as 1-hour at the 1-hour rate. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | +Opted-in pool accounts with a full included usage window refresh their credit evidence through the existing background sweep from three minutes of age. Ordinary response headers do not refresh credit balances. A failed metadata read never extends the five-minute spending-evidence lifetime. + + Across these strategies, each account's effective threshold is its `codexAccountAutoSwitchThresholds` entry when present, otherwise global `autoSwitchThreshold`. A zero override disables only proactive usage switching; startup binding, hard-lock, cooldown, entitlement checks, and failure recovery remain enforced. Setting an account's switching threshold to `0` does not disable main-account hard-lock, startup diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index 92b02e0ed2f..fc29bd2410e 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -78,6 +78,21 @@ export async function runCodexCooldownRecoveryProbes(config: OcxConfig, now = Da const MAIN_CREDIT_RENEWAL_AGE_MS = 3 * 60_000; +/** Traffic headers renew usage, never the independent credit observation (#7377). */ +function poolCreditRenewalDue(config: OcxConfig, accountId: string, now = Date.now()): boolean { + const account = configuredPoolAccount(config, accountId); + if (!account || !codexAccountUsesCreditsAfterLimit(config, accountId) + || isCodexAccountPaused(config, accountId) || isAccountNeedsReauth(accountId)) return false; + const quota = getAccountQuota(accountId); + if (codexUsageLimitResetAt(quota, account.plan, now) === undefined) return false; + const observedAt = quota?.credits?.observedAt; + if (observedAt !== undefined && Number.isFinite(observedAt) + && now - observedAt >= 0 && now - observedAt < MAIN_CREDIT_RENEWAL_AGE_MS) return false; + const attempt = poolQuotaPrimeAttemptedAt.get(accountId); + return !attempt || attempt.generation !== readCodexAccountRecord(accountId)?.generation + || now - attempt.at >= MAIN_CREDIT_RENEWAL_AGE_MS; +} + let mainHardLockRecoveryInFlight: Promise | null = null; let mainHardLockRecoveryAttempt: { identity: number; credential: number; after: number; delay: number } | undefined; @@ -186,6 +201,7 @@ export function registerCodexCooldownRecoveryProbeWorker(config: OcxConfig): voi afterTick: () => { void runCodexCooldownRecoveryProbes(config); void runMainAccountHardLockRecovery(config); + void primeCodexPoolQuotas(config, "credit-renewal"); }, }); } @@ -265,6 +281,7 @@ export async function primeCodexPoolQuotas( primeInFlight = trackBackgroundWork(`codex-pool-quota-prime:${reason}`, (async () => { const pool = (runtimeConfig.codexAccounts ?? []).filter(isSelectableCodexPoolAccount); const stale = pool.filter(a => { + if (reason === "credit-renewal") return poolCreditRenewalDue(runtimeConfig, a.id); const q = getAccountQuota(a.id); const observationStale = reason === "priority-failback" && q && !codexQuotaHasFreshUsage(q, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS); @@ -278,7 +295,10 @@ export async function primeCodexPoolQuotas( if (lastAttempt.generation !== readCodexAccountRecord(a.id)?.generation) return true; return Date.now() - lastAttempt.at >= POOL_CACHE_TTL; }); + if (reason === "credit-renewal" && stale.length === 0) return; const primeMain = async () => { + // Main renewal owns its physical-identity claims and passive probe separately. + if (reason === "credit-renewal") return; const mainLease = tryAcquireNativeMainPrimeLease(); if (!mainLease) return; try { @@ -311,11 +331,12 @@ export async function primeCodexPoolQuotas( primeMain(), mapWithConcurrency(stale, POOL_QUOTA_REFRESH_CONCURRENCY, async a => { if (!getCodexAccountCredential(a.id)) return; + if (reason === "credit-renewal" && !poolCreditRenewalDue(getRuntimeConfig(config), a.id)) return; let result: PoolQuotaResult; try { const quota = getAccountQuota(a.id); - const bypassCachedQuota = reason === "priority-failback" && quota !== null - && !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS); + const bypassCachedQuota = reason === "credit-renewal" || (reason === "priority-failback" && quota !== null + && !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS)); result = await fetchPoolAccountQuota(a.id, bypassCachedQuota, a.plan, getValidPoolTokenForPrime); } catch (error) { // Local quota-flight saturation proves no WHAM request existed for this account. diff --git a/structure/INDEX.md b/structure/INDEX.md index e6c9e5e80bb..ea555bd6e75 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -84,6 +84,7 @@ Per-vendor contracts and the adapter authority that constructs them. | [`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md) | Anthropic OAuth instances (primary and Pool 2), account pause, model routes, quota labels, and pool-bound helpers, quota, reset grants and management surfaces. | | [`providers/oauth-code-display-login.md`](providers/oauth-code-display-login.md) | Headless code-display OAuth login: flow ownership, expiry, unsupported providers, single-operator store. | | [`providers/openai-tiers.md`](providers/openai-tiers.md) | Pool/Direct account modes, API-key separation, and the public provider and quota contract. | +| [`providers/pool-credit-renewal.md`](providers/pool-credit-renewal.md) | Independent credit-evidence renewal for opted-in exhausted Codex pool accounts. | | [`providers/openai-accounts.md`](providers/openai-accounts.md) | Migration and restore, wire identity, store concurrency, pool ordering and exclusions, quota observations, and account-bound retention. | | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | Opt-in effective priority bands that spend other pool accounts before the signed-in identity, its identity observation and management API. | | [`providers/cursor.md`](providers/cursor.md) | Cursor native exec, parameterized models, checkpoints, and active-context usage. | @@ -185,6 +186,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/codex/account-preservation.ts` | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | | `src/codex/app-server-processes.ts` | [`codex-cli-update.md`](codex-cli-update.md) | | `src/codex/auth-api/account-preservation.ts` | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | +| `src/codex/auth-api/pool-mode-gate.ts` | [`providers/pool-credit-renewal.md`](providers/pool-credit-renewal.md) | | `src/codex/catalog/effort.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md)
[`catalog-effort-clamp.md`](catalog-effort-clamp.md) | | `src/codex/catalog/model-hints.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md) | | `src/codex/catalog/ownership.ts` | [`catalog-routed-ownership.md`](catalog-routed-ownership.md) | diff --git a/structure/decisions/ADR-7377-pool-credit-renewal.md b/structure/decisions/ADR-7377-pool-credit-renewal.md new file mode 100644 index 00000000000..fbea59f7e83 --- /dev/null +++ b/structure/decisions/ADR-7377-pool-credit-renewal.md @@ -0,0 +1,15 @@ +# ADR-7377: decision recorded under "Pool credit renewal" + +- Purpose: keep an explicitly opted-in credited pool account observable after its included usage + fills, without requiring a dashboard/manual metadata read every five minutes (#7377). +- Constraint: response headers update aggregate usage but do not observe credit balance. Reusing + aggregate freshness can therefore starve credit refresh while normal requests keep arriving. +- Alternatives: extend credit validity from traffic, add another polling loop, or reuse the existing + sweep and bounded pool quota primer. Extending validity invents spendable balance evidence; + another loop duplicates lifecycle/concurrency ownership. +- Decision: a pool-only prime reason selects exhausted, opted-in eligible accounts by the independent + credit clock, bypasses aggregate cache freshness, and reuses generation-aware probe/backoff owners. +- Consequences: metadata work starts from three minutes of evidence age with a matching minimum + per-credential retry gap. Native main keeps its separate passive/physical-identity path. Failed + reads do not make an account selectable or authorize any inference. A network outage can still + let credits expire; admission correctly remains closed until fresh evidence returns. diff --git a/structure/manifest.json b/structure/manifest.json index 43e7ed5815b..64dc945629b 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -532,6 +532,13 @@ "src/codex/" ] }, + { + "path": "providers/pool-credit-renewal.md", + "tier": 4, + "title": "Pool credit renewal", + "scope": "Independent credit-evidence renewal for opted-in exhausted Codex pool accounts.", + "documents": ["src/codex/auth-api/pool-mode-gate.ts"] + }, { "path": "providers/openai-accounts.md", "tier": 4, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 809727ad810..ed6675f64ae 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -535,7 +535,7 @@ including failures, while preserving main-owner claims and per-credential WHAM d No requests means no new polling. For this reason only, stale observation proof bypasses aggregate quota-cache freshness after the existing attempt backoff. Main refresh keeps its owned lease and passive intent: cache bypass does not clear an inference reauthentication mark. Other prime reasons -retain their existing cache rules. The split config schema degrades malformed optional values to +retain their existing cache rules except the independently scheduled [pool credit renewal](pool-credit-renewal.md). The split config schema degrades malformed optional values to false. Exact-account and Direct routes are unchanged. ## Main-account policy observations diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 9631ceab68e..854c502fc6c 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -334,7 +334,7 @@ identity retries: WHAM success and terminal 401/403 responses never set or clear this mode. Native token preparation also preserves the traffic quarantine. Other callers retain the existing auth behavior. Failed or incomplete observations never renew the credit clock; refreshing usage does not redeem reset credits or validate pending accounts through inference. -Coverage: `tests/codex-integration/main-account-credit-renewal.test.ts`. +Coverage: `tests/codex-integration/main-account-credit-renewal.test.ts`. Pool accounts use the separate [pool credit renewal](pool-credit-renewal.md) contract; their fresh spendable-evidence requirement is identical. Credit parsing, expiry, partial updates and reset-ticket separation are covered in `tests/codex-integration/codex-quota-parser-parity.test.ts`; selection and bulk-pause behavior diff --git a/structure/providers/pool-credit-renewal.md b/structure/providers/pool-credit-renewal.md new file mode 100644 index 00000000000..ad47b4ebef0 --- /dev/null +++ b/structure/providers/pool-credit-renewal.md @@ -0,0 +1,24 @@ +# Pool credit renewal + +`src/codex/auth-api/pool-mode-gate.ts` renews independently aged pool credit evidence on the +existing `codex-cooldown-recovery` after-tick hook, using `primeCodexPoolQuotas` with reason +`credit-renewal`. No extra timer or inference request is added. + +Only configured pool accounts opted into `creditCodexAccountIds` with a currently exhausted +included window qualify. Paused, reauthentication-required, below-limit and non-opted-in accounts +are skipped. A credit observation younger than three minutes is retained; due or missing evidence +uses the existing bounded-concurrency quota reader with cache bypass, because traffic headers can +keep aggregate usage fresh while the independent credit clock expires. + +The existing per-credential attempt map bounds retries to at least three minutes. Query backoff, +request admission and generation/configuration publication fences remain owned by the quota reader. +Only actual WHAM evidence advances the credit clock. A failed read leaves stale evidence unusable; +no credit freshness limit or spending consent is relaxed. Negative evidence remains discoverable. +The eligibility check runs again before each queued account read. The pool-only reason does not +claim, reconcile or read native main; its separate renewal owner is unchanged. + +Regression coverage: `tests/codex-integration/codex-quota-prime.test.ts` exercises pre-expiry and +expired evidence, admission after a fresh read, the sweep entry, no native-main work, exclusions +and failed-read backoff. Tests use fake metadata and private homes, not paid inference. + +> Decision record: [ADR-7377](../decisions/ADR-7377-pool-credit-renewal.md) diff --git a/tests/codex-integration/codex-quota-prime.test.ts b/tests/codex-integration/codex-quota-prime.test.ts index 75bd5fbfd12..fd508e7149d 100644 --- a/tests/codex-integration/codex-quota-prime.test.ts +++ b/tests/codex-integration/codex-quota-prime.test.ts @@ -22,6 +22,9 @@ import { saveCodexAccountCredential, } from "../../src/codex/account-store"; import { resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { isCodexAccountHeldForCredits } from "../../src/codex/account-credit-use"; +import * as sweeper from "../../src/lib/state-store-sweeper"; +import { registerCodexCooldownRecoveryProbeWorker } from "../../src/codex/auth-api/pool-mode-gate"; import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/main-account"; import { completeNativeMainRecovery, @@ -142,6 +145,85 @@ describe("primeCodexPoolQuotas", () => { if (existsSync(TEST_DIR)) removeTreeWithRetry(TEST_DIR); }); + test.each([180_000, 360_000])("credit renewal at %d ms reads WHAM despite fresh traffic headers and never touches native main (#7377)", async age => { + const config = makeConfig({ creditCodexAccountIds: ["p1"] }); + seedPoolAccount(config, "p1", "plus"); + const now = Date.now(); + setAccountQuotaFromParsed("p1", { shortPercent: 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - age } }); + expect(isCodexAccountHeldForCredits(config, "p1", getAccountQuota("p1"), "plus", now)).toBe(age > 300_000); + // A real response header updates usage, not the older credit observation. + updateAccountQuota("p1", 10); + const originalFetch = globalThis.fetch; + const probed: string[] = []; + try { + globalThis.fetch = (async (input, init) => { + expect(String(input)).toContain("/backend-api/wham/usage"); + probed.push(new Headers(init?.headers).get("chatgpt-account-id")!); + return Response.json({ plan_type: "plus", rate_limit: { + primary_window: { used_percent: 100, reset_at: Math.floor((now + 3_600_000) / 1000), limit_window_seconds: 18_000 }, + }, credits: { has_credits: true, balance: 5, overage_limit_reached: false }, spend_control: { reached: false } }); + }) as typeof fetch; + const noMain = () => { throw new Error("credit renewal must not access native main"); }; + await primeCodexPoolQuotas(config, "credit-renewal", { reconcileMainAccount: noMain, readMainTokens: noMain, fetchMainInfo: noMain }); + expect(probed).toEqual(["acct-p1"]); + expect(getAccountQuota("p1")?.credits?.observedAt).toBeGreaterThanOrEqual(now); + expect(isCodexAccountHeldForCredits(config, "p1", getAccountQuota("p1"), "plus", Date.now())).toBe(false); + let afterTick: (() => void) | undefined; + const registration = spyOn(sweeper, "registerStateSweepAfterTick").mockImplementation(entry => { + afterTick = entry.afterTick; return () => {}; + }); + try { + // Stale credit evidence is independently renewable on the existing sweep. + clearCodexQuotaPrimeState(); + setAccountQuotaFromParsed("p1", { credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - age } }); + registerCodexCooldownRecoveryProbeWorker(config); + expect(afterTick).toBeDefined(); + afterTick!(); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(probed).toHaveLength(2); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(probed).toHaveLength(2); + } finally { registration.mockRestore(); } + } finally { globalThis.fetch = originalFetch; } + }); + + test("credit renewal excludes non-opted-in, paused, reauth, below-limit and fresh-credit accounts", async () => { + const config = makeConfig({ creditCodexAccountIds: ["paused", "reauth", "below", "fresh"], pausedCodexAccountIds: ["paused"] }); + const now = Date.now(); + for (const id of ["off", "paused", "reauth", "below", "fresh"]) { + seedPoolAccount(config, id, "plus"); + setAccountQuotaFromParsed(id, { shortPercent: id === "below" ? 50 : 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - (id === "fresh" ? 0 : 180_000) } }); + } + markAccountNeedsReauth("reauth"); + const originalFetch = globalThis.fetch; + let calls = 0; + try { + globalThis.fetch = (async () => { calls++; throw new Error("no eligible credit renewal"); }) as typeof fetch; + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(calls).toBe(0); + expect(isAccountNeedsReauth("reauth")).toBe(true); + } finally { globalThis.fetch = originalFetch; clearAccountNeedsReauth("reauth"); } + }); + + test("failed credit renewal backs off without making stale credits spendable", async () => { + const config = makeConfig({ creditCodexAccountIds: ["p1"] }); + seedPoolAccount(config, "p1", "plus"); + const now = Date.now(); + setAccountQuotaFromParsed("p1", { shortPercent: 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - 360_000 } }); + const originalFetch = globalThis.fetch; + let calls = 0; + try { + globalThis.fetch = (async () => { calls++; return new Response(null, { status: 503 }); }) as typeof fetch; + await primeCodexPoolQuotas(config, "credit-renewal"); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(calls).toBe(1); + expect(getAccountQuota("p1")?.credits?.observedAt).toBe(now - 360_000); + } finally { globalThis.fetch = originalFetch; } + }); + test("startup may observe pending account quota without inference or validation", async () => { const config = makeConfig(); seedPoolAccount(config, "pending", "pro"); From af500bfc6854b45244e03bf165cc6487305dacc1 Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Mon, 19 Oct 2026 13:19:50 +0000 Subject: [PATCH 02/17] test(codex): assert renewal performs no native-main access --- tests/codex-integration/codex-quota-prime.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/codex-integration/codex-quota-prime.test.ts b/tests/codex-integration/codex-quota-prime.test.ts index fd508e7149d..3ccd538b678 100644 --- a/tests/codex-integration/codex-quota-prime.test.ts +++ b/tests/codex-integration/codex-quota-prime.test.ts @@ -164,8 +164,10 @@ describe("primeCodexPoolQuotas", () => { primary_window: { used_percent: 100, reset_at: Math.floor((now + 3_600_000) / 1000), limit_window_seconds: 18_000 }, }, credits: { has_credits: true, balance: 5, overage_limit_reached: false }, spend_control: { reached: false } }); }) as typeof fetch; - const noMain = () => { throw new Error("credit renewal must not access native main"); }; + let mainAccesses = 0; + const noMain = () => { mainAccesses++; throw new Error("credit renewal must not access native main"); }; await primeCodexPoolQuotas(config, "credit-renewal", { reconcileMainAccount: noMain, readMainTokens: noMain, fetchMainInfo: noMain }); + expect(mainAccesses).toBe(0); expect(probed).toEqual(["acct-p1"]); expect(getAccountQuota("p1")?.credits?.observedAt).toBeGreaterThanOrEqual(now); expect(isCodexAccountHeldForCredits(config, "p1", getAccountQuota("p1"), "plus", Date.now())).toBe(false); From b0d8fbafb56fde6ba16ca6189517aaf5d95c107a Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 09:52:50 -0700 Subject: [PATCH 03/17] fix(codex): limit Direct hard-lock fallback to the locked account Use the original route's credential ownership and observed-main caller check before applying Direct hard-lock fallback. Preserve binding, missing-bearer and disabled-route behavior; document and cover the boundary. --- src/codex/auth-context.ts | 16 +- .../responses/main-hard-lock-routing.ts | 44 ++++- src/server/responses/request-prepare.ts | 13 +- structure/codex-account-controls.md | 14 +- ...account-hard-lock-fallback-routing.test.ts | 179 ++++++++++++++---- .../main-account-policy-binding-wait.test.ts | 91 +++++++++ 6 files changed, 304 insertions(+), 53 deletions(-) diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index c4cd30b325d..577ee8ebad5 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1080,6 +1080,19 @@ async function awaitMainAccountPolicyBindingSettled( if (!settled) throw new CodexMainProfileDrainingError(); } +/** Memory-only ownership check shared by Direct admission and pre-auth hard-lock routing. */ +export async function mainAccountHardLockAppliesToCaller( + headers: Headers, + policy: CodexAuthPolicyConfig, + options: Pick = {}, +): Promise { + if (options.substituteMainCredentialForDirect === true) return true; + if (!hasCallerCodexBearer(headers)) return false; + if (isMainAccountHardLockEnabled(policy)) await awaitMainAccountPolicyBindingSettled(options); + return callerMatchesObservedMain(headers); +} + /** * Resolve the credential for one Codex turn. A main hard-lock reached through unpinned pool * selection is annotated with the earliest known pool recovery, so the refusal can name a retry @@ -1172,8 +1185,7 @@ async function resolveCodexAuthContextUnannotated( // Trusted substitution still has to claim and validate stored main below. if (!substituteStoredMain && !hasCallerCodexBearer(headers)) throw new CodexDirectAuthenticationError(); if (!substituteStoredMain) { - if (isMainAccountHardLockEnabled(policy)) await awaitMainAccountPolicyBindingSettled(options); - if (callerMatchesObservedMain(headers)) assertMainAccountPolicy(policy); + if (await mainAccountHardLockAppliesToCaller(headers, policy, options)) assertMainAccountPolicy(policy); assertCallerOwnedMainPoolNotCooled(); if (reserve) { const selected = materializeCodexUpstreamAuth(headers, { kind: "main", accountId: null }, { config: policy }); diff --git a/src/server/responses/main-hard-lock-routing.ts b/src/server/responses/main-hard-lock-routing.ts index e24081f5d8b..931bbc5c2e4 100644 --- a/src/server/responses/main-hard-lock-routing.ts +++ b/src/server/responses/main-hard-lock-routing.ts @@ -1,6 +1,6 @@ import type { OcxConfig } from "../../types"; import { resolveComboId } from "../../combos"; -import { isBareOpenAiFamilyModel } from "../../router"; +import { isBareOpenAiFamilyModel, routeConcreteModel } from "../../router"; import { NATIVE_RESERVE_MODEL } from "../../codex/catalog/native-models"; import { getMainAccountHardLockStatus, @@ -10,6 +10,29 @@ import { codexQuotaScopeForModel } from "../../codex/routing/health-store"; import { getEligiblePoolAccounts } from "../../codex/routing/selection"; import { OPENAI_CODEX_PROVIDER_ID, isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { providerCodexAccountMode } from "../../providers/registry"; +import { mainAccountHardLockAppliesToCaller } from "../../codex/auth-context"; +import { codexRouteCredentialDomainHeaders, codexRouteCredentialOwnership } from "./core-auth"; +import type { HandleResponsesOptions } from "./core-options"; + +export type MainHardLockCallerCheck = (sourceModel: string) => Promise; + +/** Inspect the original credential domain, before fallback changes the route or drops headers. */ +export function mainHardLockFallbackCallerCheck( + req: Request, config: OcxConfig, options: HandleResponsesOptions, +): MainHardLockCallerCheck { + return async sourceModel => { + let sourceRoute: ReturnType; + try { sourceRoute = routeConcreteModel(config, sourceModel); } + // This preview cannot replace the normal routing phase's existing error response. + catch { return false; } + const headers = codexRouteCredentialDomainHeaders(req, sourceRoute, options, false, config); + const { substituteMainCredential } = codexRouteCredentialOwnership(headers, config, sourceRoute, options); + return mainAccountHardLockAppliesToCaller(headers, options.codexAuthPolicy ?? config, { + substituteMainCredentialForDirect: substituteMainCredential, + signal: options.abortSignal ?? req.signal, + }); + }; +} function isConfiguredComboSelector(config: OcxConfig, modelId: string): boolean { const comboId = resolveComboId(config, modelId); @@ -72,11 +95,12 @@ function rawModelOf(body: unknown): { raw: Record; model: strin * - Luna Reserve keeps its own admission policy (`assertReserveAdmission`); its helper * traffic is not operator chat and must not silently change models. */ -export function applyMainHardLockFallbackOverride( +export async function applyMainHardLockFallbackOverride( body: unknown, config: OcxConfig, - now = Date.now(), -): MainHardLockFallbackOverride | null { + now?: number, + callerCheck?: MainHardLockCallerCheck, +): Promise { const parsed = rawModelOf(body); if (!parsed) return null; const fallback = resolveMainAccountHardLockFallback(config); @@ -85,14 +109,18 @@ export function applyMainHardLockFallbackOverride( if (!isBareOpenAiFamilyModel(parsed.model) || parsed.model === NATIVE_RESERVE_MODEL || isConfiguredComboSelector(config, parsed.model)) return null; - if (getMainAccountHardLockStatus(config, now).state !== "blocked") return null; const accountMode = providerCodexAccountMode( OPENAI_CODEX_PROVIDER_ID, config.providers?.[OPENAI_CODEX_PROVIDER_ID], ); - if (accountMode !== "direct" - && getEligiblePoolAccounts(config, undefined, now, codexQuotaScopeForModel(parsed.model)).length > 0) { - return null; + if (accountMode === "direct") { + // Direct may forward a different caller's healthy credential. Only stored-main substitution + // or an observed-main match owes this lock. Recheck usage after the startup binding wait. + if (!callerCheck || !await callerCheck(parsed.model)) return null; + if (getMainAccountHardLockStatus(config, now ?? Date.now()).state !== "blocked") return null; + } else { + if (getMainAccountHardLockStatus(config, now ?? Date.now()).state !== "blocked") return null; + if (getEligiblePoolAccounts(config, undefined, now ?? Date.now(), codexQuotaScopeForModel(parsed.model)).length > 0) return null; } parsed.raw.model = fallback.model; diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index e7ebf8eedf8..f64ed70fdab 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -158,7 +158,7 @@ import { } from "./core-codex-account"; import { acquireUpstreamHostAdmission } from "../../codex/upstream-host-health"; import { applyCompactionRoutingOverride, compactionRoutingKeepsProviderIdentity } from "./compaction-routing"; -import { applyMainHardLockFallbackOverride } from "./main-hard-lock-routing"; +import { applyMainHardLockFallbackOverride, mainHardLockFallbackCallerCheck } from "./main-hard-lock-routing"; import { codexAuthContextLogLabel } from "../../codex/account-label"; import { conversationStateBindingFromAuth, @@ -364,7 +364,16 @@ export async function prepareResponsesRequest( // must run before comboIdFromRawBody reads `model`. if (!concreteSelection && !options.compactionRoutingOverride && !isCompactionTrigger && !options.internalDecisionCall && !shadowCallIntercepted && !shadowCallMatches && !memoryModelApplies && !threadSpawn && inboundWire === "responses") { - options.mainHardLockFallbackOverride = applyMainHardLockFallbackOverride(body, config); + try { + options.mainHardLockFallbackOverride = await applyMainHardLockFallbackOverride( + body, config, undefined, mainHardLockFallbackCallerCheck(req, config, options), + ); + } catch (error) { + if (options.abortSignal?.aborted || req.signal.aborted) return clientCancelledResponse(); + const mapped = mapCodexAuthContextErrorToResponse(error, { now: Date.now() }); + if (mapped) return mapped; + throw error; + } } const comboId = !concreteSelection ? comboIdFromRawBody(body, config) : null; // A decision model may be a plain model or a non-JEV combo. A JEV combo here would recurse diff --git a/structure/codex-account-controls.md b/structure/codex-account-controls.md index 6b186fd74c9..6c6523b30de 100644 --- a/structure/codex-account-controls.md +++ b/structure/codex-account-controls.md @@ -84,9 +84,14 @@ protects: - `getEligiblePoolAccounts` is consulted under the request model's quota scope whenever canonical `openai` is in Pool mode, so "the Pool can serve it" shares the dispatch's own eligibility and - scoped-cooldown rules. An empty eligible list (or Direct mode, which never consults stored Pool - accounts) is what makes the fallback the request's only way through; the lock excludes the main - account there itself. + scoped-cooldown rules. An empty eligible list permits fallback; the lock excludes the main + account from that list itself. +- Direct does not consult stored Pool accounts. Fallback applies only when the original source + route's credential ownership selects stored-main substitution, or the caller's bearer and + effective workspace match the currently observed main credential. Caller-owned checks wait + boundedly for pending startup policy binding, then recheck identity and the blocked state before + rewriting. Other callers retain their native route; a missing caller bearer keeps its normal + authentication refusal. - Only bare Codex-native selectors qualify. Provider-qualified rows, combos, account namespaces (including `nativeAlias` combos), compaction triggers, shadow-intercepted helper calls and policy routes name their own destination on purpose, and Luna Reserve keeps its own admission. A @@ -105,7 +110,8 @@ model — rather than clearing it. Settings projection and writes live beside th `src/server/management/config-routes.ts`; `null` clears the section. Coverage: `tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts` and the fallback cases in `tests/config/settings-main-account-hard-lock.test.ts` plus -`gui/tests/main-account-hard-lock-setting.test.tsx`. +`gui/tests/main-account-hard-lock-setting.test.tsx`. Startup-binding rechecks are covered by +`tests/codex-integration/main-account-policy-binding-wait.test.ts`. ## Stored-account authentication policy diff --git a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts index 14b91b46535..c807325eb77 100644 --- a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts +++ b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { applyMainHardLockFallbackOverride } from "../../src/server/responses/main-hard-lock-routing"; import { resolveMainAccountHardLockFallback } from "../../src/codex/main-account-hard-lock"; -import { captureMainQuotaWriter, clearMainAccountInfoCache, observeMainQuotaIdentity } from "../../src/codex/main-account-cache"; +import { captureMainQuotaWriter, clearMainAccountInfoCache, observeMainQuotaCredential, observeMainQuotaIdentity } from "../../src/codex/main-account-cache"; import { clearAccountQuota, setAccountQuotaFromParsed, type StoredAccountQuota } from "../../src/codex/quota"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { NATIVE_RESERVE_MODEL } from "../../src/codex/catalog/native-models"; @@ -88,64 +88,164 @@ describe("main-account hard-lock fallback resolution", () => { }); }); -describe("applyMainHardLockFallbackOverride", () => { - test("a blocked lock with no eligible account rewrites the model and the effort", () => { +describe("Direct hard-lock fallback caller identity", () => { + const mainAccountId = "fallback-account-a"; + function tokenFor(accountId: string): string { + return `header.${Buffer.from(JSON.stringify({ + exp: Math.floor(Date.now() / 1000) + 3600, + "https://api.openai.com/auth": { chatgpt_account_id: accountId }, + })).toString("base64url")}.signature`; + } + const mainToken = tokenFor(mainAccountId); + const otherToken = tokenFor("fallback-account-b"); + function directConfig(): OcxConfig { + return config({ + defaultProvider: "openai", + autoSwitchThreshold: 0, + providers: { + openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", + baseUrl: "https://chatgpt.com/backend-api/codex" }, + rescue: { adapter: "openai-responses", authMode: "key", apiKey: "fixture-fallback-key", + baseUrl: "https://fallback.example/v1" }, + }, + codexMainAccountHardLockFallback: { model: "rescue/rescue-model" }, + } as Partial); + } + async function dispatch(callerToken: string, accountId: string, source?: "bearer" | "dedicated" | "x-api-key") { + const seen: Array<{ url: string; auth: string | null; model: string }> = []; + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")); + seen.push({ url: String(input), auth: new Headers(init?.headers).get("authorization"), model: body.model }); + return Response.json({ id: "resp_caller_fixture", status: "completed", model: body.model, + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "ok" }] }], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }); + }) as typeof fetch; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${callerToken}`, "chatgpt-account-id": accountId }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), directConfig(), { model: "", provider: "" }, source ? { + admission: { kind: "configured", keyId: "fixture-admission", source }, + } : {}); + expect(response.status).toBe(200); + await response.text(); + expect(seen).toHaveLength(1); + return seen[0]!; + } + beforeEach(() => { + observeMainQuotaCredential(mainToken, mainAccountId); + observe({ weeklyPercent: 100 }); + }); + test("another caller stays native while the observed main is blocked", async () => { + expect(await dispatch(otherToken, "fallback-account-b")).toEqual({ + url: "https://chatgpt.com/backend-api/codex/responses", auth: `Bearer ${otherToken}`, model: "gpt-5.3", + }); + }); + test("the same workspace with an unobserved bearer stays native", async () => { + const unobservedToken = mainToken.replace(".signature", ".other-signature"); + const seen = await dispatch(unobservedToken, mainAccountId); + expect(seen.url).toBe("https://chatgpt.com/backend-api/codex/responses"); + expect(seen.auth).toBe(`Bearer ${unobservedToken}`); + }); + test("a caller matching observed main uses the configured fallback", async () => { + expect(await dispatch(mainToken, mainAccountId)).toEqual({ + url: "https://fallback.example/v1/responses", auth: "Bearer fixture-fallback-key", model: "rescue-model", + }); + }); + test("proxy bearer admission preserves stored-main fallback without matching the proxy secret", async () => { + const seen = await dispatch("fixture-proxy-secret", "", "bearer"); + expect(seen.url).toBe("https://fallback.example/v1/responses"); + expect(seen.auth).toBe("Bearer fixture-fallback-key"); + }); + for (const source of ["dedicated", "x-api-key"] as const) { + test(`${source} admission preserves the separate caller's native credential`, async () => { + const seen = await dispatch(otherToken, "fallback-account-b", source); + expect(seen.url).toBe("https://chatgpt.com/backend-api/codex/responses"); + expect(seen.auth).toBe(`Bearer ${otherToken}`); + }); + } + test("an unblocked matching main stays native", async () => { + observe({ weeklyPercent: 10 }); + const seen = await dispatch(mainToken, mainAccountId); + expect(seen.url).toBe("https://chatgpt.com/backend-api/codex/responses"); + expect(seen.auth).toBe(`Bearer ${mainToken}`); + }); + test("a disabled original route keeps its normal 404 with fallback configured and hard lock off", async () => { + const cfg = directConfig(); + cfg.providers.openai!.disabled = true; + cfg.codexMainAccountHardLock = false; + let sends = 0; + globalThis.fetch = (async () => { sends++; throw new Error("disabled route must not send"); }) as typeof fetch; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${mainToken}`, + "chatgpt-account-id": mainAccountId }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), cfg, { model: "", provider: "" }); + expect(response.status).toBe(404); + const body = await response.json() as { error: { type: string } }; + expect(body.error.type).toBe("invalid_request_error"); + expect(sends).toBe(0); + }); +}); + +describe("applyMainHardLockFallbackOverride", async () => { + test("a blocked lock with no eligible account rewrites the model and the effort", async () => { observe({ weeklyPercent: 98 }); const body: Record = { model: "gpt-5.3", reasoning: { effort: "high" } }; - expect(applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))) + expect(await applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))) .toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); expect(body.model).toBe("deepseek/deepseek-chat"); expect(body.reasoning).toEqual({ effort: "low" }); }); - test("each blocking window governs on its own", () => { + test("each blocking window governs on its own", async () => { const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); observe({ shortPercent: 90 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ shortPercent: 0 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); observe({ weeklyPercent: 98 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ weeklyPercent: 0 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); }); - test("a monthly-only account governs on monthly; supplementary monthly data never blocks", () => { + test("a monthly-only account governs on monthly; supplementary monthly data never blocks", async () => { const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); // No short or weekly evidence at all: monthly is the account's only window. observe({ monthlyPercent: 99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ monthlyPercent: 50 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); // Supplementary monthly data on a 5h/weekly account never becomes a lock. observe({ shortPercent: 10, weeklyPercent: 10, monthlyPercent: 99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); }); - test("release is the same observation that admits: dropping below the threshold stops the rewrite", () => { + test("release is the same observation that admits: dropping below the threshold stops the rewrite", async () => { const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); observe({ weeklyPercent: 99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ weeklyPercent: 97.99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); }); - test("unknown usage, the opt-out, and a missing section all keep the refusal behavior", () => { - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); + test("unknown usage, the opt-out, and a missing section all keep the refusal behavior", async () => { + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); observe({ weeklyPercent: 100 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLock: false, codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config())).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLock: false, codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config())).toBeNull(); }); - test("an eligible pool account keeps the request on Codex", () => { + test("an eligible pool account keeps the request on Codex", async () => { observe({ weeklyPercent: 100 }); const cfg = withPoolAccount(config({ codexMainAccountHardLockFallback: FALLBACK } as Partial)); const body: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); expect(body.model).toBe("gpt-5.3"); }); - test("a pool account in shared reset-derived cooldown or unused under Direct mode does not suppress the fallback", () => { + test("a pool account in shared reset-derived cooldown or unused under Direct mode does not suppress the fallback", async () => { observe({ weeklyPercent: 100 }); const cfg = withPoolAccount(config({ codexMainAccountHardLockFallback: FALLBACK } as Partial)); setScopedHealth("pool-a", "shared", { @@ -158,7 +258,7 @@ describe("applyMainHardLockFallbackOverride", () => { cooldownGeneration: 1, }); const cooledBody: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(cooledBody, cfg, now)) + expect(await applyMainHardLockFallbackOverride(cooledBody, cfg, now)) .toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); expect(cooledBody.model).toBe("deepseek/deepseek-chat"); @@ -175,11 +275,14 @@ describe("applyMainHardLockFallbackOverride", () => { codexMainAccountHardLockFallback: FALLBACK, } as Partial)); const directBody: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(directBody, directCfg, now)) + // Direct fails closed without the caller ownership proof supplied by request preparation. + expect(await applyMainHardLockFallbackOverride(directBody, directCfg, now)).toBeNull(); + expect(directBody.model).toBe("gpt-5.3"); + expect(await applyMainHardLockFallbackOverride(directBody, directCfg, now, async () => true)) .toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); }); - test("only bare Codex-native selectors qualify", () => { + test("only bare Codex-native selectors qualify", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ codexMainAccountHardLockFallback: FALLBACK, @@ -194,40 +297,42 @@ describe("applyMainHardLockFallbackOverride", () => { } as Partial); for (const model of ["deepseek/deepseek-chat", "combo/x", "openai-codex/gpt-5.3", NATIVE_RESERVE_MODEL, "gpt-5.4"]) { const body: Record = { model }; - expect(applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); expect(body.model).toBe(model); } }); - test("a Codex-native fallback target never applies: the refusal is the honest answer", () => { + test("a Codex-native fallback target never applies: the refusal is the honest answer", async () => { observe({ weeklyPercent: 100 }); for (const target of ["gpt-5.4", "openai/gpt-5.4"]) { const body: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: { model: target } } as Partial))).toBeNull(); + expect(await applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: { model: target } } as Partial))).toBeNull(); expect(body.model).toBe("gpt-5.3"); } }); - test("a fallback without an effort leaves the request's own reasoning untouched", () => { + test("a fallback without an effort leaves the request's own reasoning untouched", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ codexMainAccountHardLockFallback: { model: "deepseek/deepseek-chat" } } as Partial); const body: Record = { model: "gpt-5.3", reasoning: { effort: "high" } }; - expect(applyMainHardLockFallbackOverride(body, cfg)).toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); + expect(await applyMainHardLockFallbackOverride(body, cfg)).toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); expect(body.model).toBe("deepseek/deepseek-chat"); expect(body.reasoning).toEqual({ effort: "high" }); }); - test("a body without a usable model selector is never touched", () => { + test("a body without a usable model selector is never touched", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); - expect(applyMainHardLockFallbackOverride(null, cfg)).toBeNull(); - expect(applyMainHardLockFallbackOverride([], cfg)).toBeNull(); - expect(applyMainHardLockFallbackOverride({ model: "" }, cfg)).toBeNull(); - expect(applyMainHardLockFallbackOverride({ model: 42 }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride(null, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride([], cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: 42 }, cfg)).toBeNull(); }); test("handleResponses strips caller credentials, preserves requestedModel on combo fallback, and yields to shadow/compaction turns", async () => { observe({ weeklyPercent: 100 }); + // This caller is the observed main credential that actually owes the hard lock. + observeMainQuotaCredential("caller-chatgpt-token", "fallback-account-a"); const cfg = config({ defaultProvider: "gateway", providers: { @@ -284,7 +389,7 @@ describe("applyMainHardLockFallbackOverride", () => { "content-type": "application/json", "session_id": "lane-hard-lock-fallback", "authorization": "Bearer caller-chatgpt-token", - "chatgpt-account-id": "acct-caller", + "chatgpt-account-id": "fallback-account-a", }); const lane = sessionLaneIdFromRequest(headers); rememberComboForLane(lane, "prior", { provider: "gateway", model: "prior-model" }, "prior-model", captureConfigGeneration()); diff --git a/tests/codex-integration/main-account-policy-binding-wait.test.ts b/tests/codex-integration/main-account-policy-binding-wait.test.ts index 3e5c0d61fa5..67d8c7a8f89 100644 --- a/tests/codex-integration/main-account-policy-binding-wait.test.ts +++ b/tests/codex-integration/main-account-policy-binding-wait.test.ts @@ -22,6 +22,8 @@ import { clearAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/qu import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { handleResponses } from "../../src/server/responses"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; /** * The admission fence during an owned startup's main-policy binding (#5694 made it default-on). @@ -44,6 +46,7 @@ const previousHome = process.env.OPENCODEX_HOME; const previousCodexHome = process.env.CODEX_HOME; let releases: Array<() => Promise> = []; let openGates: Array<() => void> = []; +const originalFetch = globalThis.fetch; function bearerFor(id: string): string { const payload = Buffer.from(JSON.stringify({ @@ -162,6 +165,7 @@ beforeEach(() => { }); afterEach(async () => { + globalThis.fetch = originalFetch; // A case may end with the hold still up (the deadline case has to). Open every gate first: // `release` below awaits convergence, which cannot finish behind a closed barrier. for (const open of openGates.splice(0)) open(); @@ -178,6 +182,93 @@ afterEach(async () => { }); describe("a caller-owned request waits out the startup policy binding", () => { + for (const source of ["loopback", "dedicated"] as const) { + test(`handleResponses missing caller bearer keeps immediate 401 during binding under ${source} admission`, async () => { + const startup = await holdStartup(); + const cfg = directConfig(); + cfg.codexMainAccountHardLockFallback = { model: "rescue/rescue-model" }; + cfg.providers.rescue = { adapter: "openai-responses", authMode: "key", apiKey: "fixture-rescue", + baseUrl: "https://fallback.example/v1" }; + let sends = 0; + globalThis.fetch = (async () => { sends++; throw new Error("missing caller must not send"); }) as typeof fetch; + const request = handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), cfg, { model: "", provider: "" }, { + admission: source === "loopback" ? { kind: "loopback", source } + : { kind: "configured", keyId: "fixture-admission", source }, + }); + const settled = trackSettlement(request); + const outcome = request.then(response => ({ response }), error => ({ error })); + try { + await Bun.sleep(25); + expect(startup.pending()).toBe(true); + expect(settled()).toBe(true); + const result = await outcome; + if (!("response" in result)) throw result.error; + expect(result.response.status).toBe(401); + await result.response.text(); + expect(sends).toBe(0); + } finally { + startup.release(); + const result = await outcome; + if ("response" in result && !result.response.bodyUsed && !result.response.body?.locked) { + await result.response.body?.cancel(); + } + } + }); + } + + for (const [initialUsage, releaseBlocked] of [[100, true], [100, false], [undefined, true]] as const) { + test(`handleResponses rechecks fallback after binding settles from ${initialUsage ?? "unknown"} with blocked=${releaseBlocked}`, async () => { + const startup = await holdStartup(); + observeMainQuotaIdentity(accountId); + observeMainQuotaCredential(token, accountId); + const writer = captureMainQuotaWriter(accountId)!; + if (initialUsage !== undefined) setAccountQuotaFromParsed(MAIN, { weeklyPercent: initialUsage }, undefined, writer); + const cfg = directConfig(); + cfg.providers.rescue = { adapter: "openai-responses", authMode: "key", apiKey: "fixture-rescue", + baseUrl: "https://fallback.example/v1" }; + cfg.codexMainAccountHardLockFallback = { model: "rescue/rescue-model" }; + const seen: string[] = []; + const releaseSpend = acquireOwnedSpendHome(); + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + seen.push(String(input)); + const body = JSON.parse(String(init?.body ?? "{}")); + return Response.json({ id: "resp_binding_fixture", status: "completed", model: body.model, + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "ok" }] }], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }); + }) as typeof fetch; + const request = handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: callerHeaders(), + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), cfg, { model: "", provider: "" }); + // Observe rejection too, so a failed assertion never leaves an unhandled request. + const outcome = request.then(response => ({ response }), error => ({ error })); + try { + await Bun.sleep(25); + expect(startup.pending()).toBe(true); + expect(seen).toEqual([]); + setAccountQuotaFromParsed(MAIN, { weeklyPercent: releaseBlocked ? 100 : 10 }, undefined, writer); + startup.release(); + const result = await outcome; + if (!("response" in result)) throw result.error; + expect(result.response.status).toBe(200); + await result.response.text(); + expect(seen).toEqual([releaseBlocked + ? "https://fallback.example/v1/responses" : "https://chatgpt.com/backend-api/codex/responses"]); + } finally { + startup.release(); + try { + const result = await outcome; + if ("response" in result && !result.response.bodyUsed && !result.response.body?.locked) { + await result.response.body?.cancel(); + } + } finally { releaseSpend(); } + } + }); + } + test("default config waits for convergence and then resolves an auth context", async () => { const startup = await holdStartup(); expect(startup.pending()).toBe(true); From 006b9d25fc73fb4e4fcd3bb3a4a2bf79f10f3da6 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 10:31:57 -0700 Subject: [PATCH 04/17] fix(responses): break hard-lock caller-check import cycle Move the caller-check factory into core-auth, which owns the credential-domain helpers, and update its request-preparation import. Keep the function body and routing/authentication policy unchanged. --- src/server/responses/core-auth.ts | 20 ++++++++++++++++ .../responses/main-hard-lock-routing.ts | 24 +------------------ src/server/responses/request-prepare.ts | 3 ++- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/server/responses/core-auth.ts b/src/server/responses/core-auth.ts index 4734debe617..ea44d955a22 100644 --- a/src/server/responses/core-auth.ts +++ b/src/server/responses/core-auth.ts @@ -2,6 +2,7 @@ import type { OcxProviderConfig, OcxConfig } from "../../types"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import type { CodexAuthContext } from "../../codex/auth-context"; import type { RouteResult } from "../../router"; +import { routeConcreteModel } from "../../router"; import type { HandleResponsesOptions } from "./core-options"; import { hasForwardableCodexBearer, @@ -24,6 +25,7 @@ import { CodexAuthContextError, applyCodexAuthContextToProvider, stripCodexRuntimeProviderFields, + mainAccountHardLockAppliesToCaller, } from "../../codex/auth-context"; import { codexAccountSelectionForTurn, tryClaimNativeMainProfileForTurn } from "../lifecycle"; import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup"; @@ -590,3 +592,21 @@ export async function refreshNativeMainForwardAuth(args: { }) ?? nativeMainRefreshFailureResponse(error) }; } } + +/** Inspect the original credential domain, before fallback changes the route or drops headers. */ +export function mainHardLockFallbackCallerCheck( + req: Request, config: OcxConfig, options: HandleResponsesOptions, +): (sourceModel: string) => Promise { + return async sourceModel => { + let sourceRoute: ReturnType; + try { sourceRoute = routeConcreteModel(config, sourceModel); } + // This preview cannot replace the normal routing phase's existing error response. + catch { return false; } + const headers = codexRouteCredentialDomainHeaders(req, sourceRoute, options, false, config); + const { substituteMainCredential } = codexRouteCredentialOwnership(headers, config, sourceRoute, options); + return mainAccountHardLockAppliesToCaller(headers, options.codexAuthPolicy ?? config, { + substituteMainCredentialForDirect: substituteMainCredential, + signal: options.abortSignal ?? req.signal, + }); + }; +} diff --git a/src/server/responses/main-hard-lock-routing.ts b/src/server/responses/main-hard-lock-routing.ts index 931bbc5c2e4..9ff94a98b69 100644 --- a/src/server/responses/main-hard-lock-routing.ts +++ b/src/server/responses/main-hard-lock-routing.ts @@ -1,6 +1,6 @@ import type { OcxConfig } from "../../types"; import { resolveComboId } from "../../combos"; -import { isBareOpenAiFamilyModel, routeConcreteModel } from "../../router"; +import { isBareOpenAiFamilyModel } from "../../router"; import { NATIVE_RESERVE_MODEL } from "../../codex/catalog/native-models"; import { getMainAccountHardLockStatus, @@ -10,30 +10,8 @@ import { codexQuotaScopeForModel } from "../../codex/routing/health-store"; import { getEligiblePoolAccounts } from "../../codex/routing/selection"; import { OPENAI_CODEX_PROVIDER_ID, isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { providerCodexAccountMode } from "../../providers/registry"; -import { mainAccountHardLockAppliesToCaller } from "../../codex/auth-context"; -import { codexRouteCredentialDomainHeaders, codexRouteCredentialOwnership } from "./core-auth"; -import type { HandleResponsesOptions } from "./core-options"; - export type MainHardLockCallerCheck = (sourceModel: string) => Promise; -/** Inspect the original credential domain, before fallback changes the route or drops headers. */ -export function mainHardLockFallbackCallerCheck( - req: Request, config: OcxConfig, options: HandleResponsesOptions, -): MainHardLockCallerCheck { - return async sourceModel => { - let sourceRoute: ReturnType; - try { sourceRoute = routeConcreteModel(config, sourceModel); } - // This preview cannot replace the normal routing phase's existing error response. - catch { return false; } - const headers = codexRouteCredentialDomainHeaders(req, sourceRoute, options, false, config); - const { substituteMainCredential } = codexRouteCredentialOwnership(headers, config, sourceRoute, options); - return mainAccountHardLockAppliesToCaller(headers, options.codexAuthPolicy ?? config, { - substituteMainCredentialForDirect: substituteMainCredential, - signal: options.abortSignal ?? req.signal, - }); - }; -} - function isConfiguredComboSelector(config: OcxConfig, modelId: string): boolean { const comboId = resolveComboId(config, modelId); return Boolean(comboId && Object.hasOwn(config.combos ?? {}, comboId)); diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index f64ed70fdab..6b33dd11f41 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -100,6 +100,7 @@ import type { import { codexRouteCredentialDomainHeaders, codexRouteCredentialOwnership, + mainHardLockFallbackCallerCheck, resolveResponsesCodexAuth, withClaudeNativeSession, } from "./core-auth"; @@ -158,7 +159,7 @@ import { } from "./core-codex-account"; import { acquireUpstreamHostAdmission } from "../../codex/upstream-host-health"; import { applyCompactionRoutingOverride, compactionRoutingKeepsProviderIdentity } from "./compaction-routing"; -import { applyMainHardLockFallbackOverride, mainHardLockFallbackCallerCheck } from "./main-hard-lock-routing"; +import { applyMainHardLockFallbackOverride } from "./main-hard-lock-routing"; import { codexAuthContextLogLabel } from "../../codex/account-label"; import { conversationStateBindingFromAuth, From c77f226f8eb545b6a26457ac4c0acb177a11576a Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 17:28:11 -0700 Subject: [PATCH 05/17] fix(codex): revalidate hard-lock fallback targets before request rewriting --- .../responses/main-hard-lock-routing.ts | 18 +++- structure/codex-account-controls.md | 5 ++ ...account-hard-lock-fallback-routing.test.ts | 87 ++++++++++++++++++- 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/src/server/responses/main-hard-lock-routing.ts b/src/server/responses/main-hard-lock-routing.ts index 9ff94a98b69..e2a3586bd5f 100644 --- a/src/server/responses/main-hard-lock-routing.ts +++ b/src/server/responses/main-hard-lock-routing.ts @@ -1,6 +1,8 @@ import type { OcxConfig } from "../../types"; import { resolveComboId } from "../../combos"; -import { isBareOpenAiFamilyModel } from "../../router"; +import { isBareOpenAiFamilyModel, previewRouteModel } from "../../router"; +import { resolveChosenTarget } from "./shadow-target-availability"; +import { getRoutingProfile, resolvePolicyProfileId } from "../../routing/profile"; import { NATIVE_RESERVE_MODEL } from "../../codex/catalog/native-models"; import { getMainAccountHardLockStatus, @@ -101,6 +103,20 @@ export async function applyMainHardLockFallbackOverride( if (getEligiblePoolAccounts(config, undefined, now ?? Date.now(), codexQuotaScopeForModel(parsed.model)).length > 0) return null; } + // Combos and policies keep their own dispatch admission and declared failover/evidence. + // Other chosen targets must resolve now; a removed qualifier cannot become a default send. + const policyId = resolvePolicyProfileId(config, fallback.model); + if (!isConfiguredComboSelector(config, fallback.model) + && !(policyId && getRoutingProfile(config, policyId))) { + try { + const target = resolveChosenTarget(fallback.model, model => previewRouteModel(config, model)); + if ("unavailable" in target) return null; + } catch { + // An unavailable target keeps the source lock's refusal without touching the body. + return null; + } + } + parsed.raw.model = fallback.model; if (fallback.reasoningEffort !== undefined) { const reasoning = parsed.raw.reasoning; diff --git a/structure/codex-account-controls.md b/structure/codex-account-controls.md index 6c6523b30de..505cc2f3e1e 100644 --- a/structure/codex-account-controls.md +++ b/structure/codex-account-controls.md @@ -103,6 +103,11 @@ protects: - A malformed section (blank or padded model, undeclared effort) disables the reroute entirely — the same disable-don't-widen reading as the compaction override — while the load and write boundaries report the malformation. The lock's own refusal behavior is untouched either way. +- Before rewriting, concrete targets resolve against the current configuration through the + shared chosen-target check. Unavailable targets and qualified selectors that only reach the + terminal default provider retain the source lock's refusal. Bare default targets, declared + slash-containing model ids, configured combos and policy profiles keep their ordinary routing + semantics; policies retain request-evidence evaluation at dispatch preparation. The dashboard row mirrors the shadow-call row: the model pick, the effort and the switch each save immediately, and the switch writes `enabled: false` — a remembered pause that keeps the diff --git a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts index c807325eb77..8f5cdc27151 100644 --- a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts +++ b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts @@ -57,7 +57,7 @@ function observe(quota: Omit): void { } function config(overrides: Partial = {}): OcxConfig { - return { providers: {}, codexAccounts: [], ...overrides } as OcxConfig; + return { providers: { deepseek: { adapter: "openai-responses", authMode: "key", baseUrl: "https://deepseek.example.test/v1" } }, codexAccounts: [], ...overrides } as OcxConfig; } function withPoolAccount(target: OcxConfig): OcxConfig { @@ -265,6 +265,7 @@ describe("applyMainHardLockFallbackOverride", async () => { deleteAllScopedHealth("pool-a"); const directCfg = withPoolAccount(config({ providers: { + deepseek: { adapter: "openai-responses", authMode: "key", baseUrl: "https://deepseek.example.test/v1" }, openai: { adapter: "openai-responses", authMode: "forward", @@ -329,6 +330,90 @@ describe("applyMainHardLockFallbackOverride", async () => { expect(await applyMainHardLockFallbackOverride({ model: 42 }, cfg)).toBeNull(); }); + test("unavailable chosen targets keep the blocked source and reasoning intact", () => { + observe({ weeklyPercent: 100 }); + const gateway = { adapter: "openai-responses", authMode: "key", baseUrl: "https://gateway.example.test/v1" } as const; + for (const target of ["private/secret-model", "former/secret-model", "combo/removed", "policy/removed", "retired-account/gpt-5.3"]) { + for (const providers of [{ gateway }, {}]) { + const cfg = config({ providers, defaultProvider: "gateway", codexMainAccountHardLockFallback: { model: target, reasoningEffort: "low" } } as Partial); + const body = { model: "gpt-5.3", reasoning: { effort: "high" } }; + expect(applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); + expect(body).toEqual({ model: "gpt-5.3", reasoning: { effort: "high" } }); + } + } + const cfg = config({ providers: { gateway, private: { ...gateway, disabled: true } }, defaultProvider: "gateway", codexMainAccountHardLockFallback: { model: "private/secret-model" } } as Partial); + expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + }); + + test("target admission is rechecked after provider removal, re-addition and alias changes", () => { + observe({ weeklyPercent: 100 }); + const provider = { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1", alias: "chosen" } as const; + const cfg = config({ providers: { private: provider }, defaultProvider: "private", codexMainAccountHardLockFallback: { model: "chosen/secret-model" } } as Partial); + const apply = () => applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg); + expect(apply()).not.toBeNull(); + delete cfg.providers.private; + expect(apply()).toBeNull(); + cfg.providers.private = provider; + expect(apply()).not.toBeNull(); + cfg.providers.private = { ...provider, alias: "renamed" }; + expect(apply()).toBeNull(); + cfg.codexMainAccountHardLockFallback = { model: "private/secret-model" }; + expect(apply()).not.toBeNull(); + }); + + test("bare fallback and explicitly declared slash model ids stay valid", () => { + observe({ weeklyPercent: 100 }); + const provider = { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1", models: ["vendor/native-model"] } as const; + for (const model of ["bare-model", "vendor/native-model", "private/vendor/native-model"]) { + const cfg = config({ providers: { private: { ...provider, models: [...provider.models] } }, defaultProvider: "private", codexMainAccountHardLockFallback: { model } } as Partial); + expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)?.targetModel).toBe(model); + } + }); + + test("configured combo aliases, policy profiles and account namespaces keep their admission", () => { + observe({ weeklyPercent: 100 }); + const cfg = config({ + providers: { private: { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1" }, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" } }, + combos: { rescue: { alias: "chosen-combo", targets: [{ provider: "private", model: "secret-model" }] } }, + routingProfiles: { rescue: { alias: "chosen-policy", candidates: [{ provider: "private", model: "secret-model" }], require: { tools: true } } }, + codexAccountNamespaces: { pinned: "pool-a" }, + } as Partial); + for (const model of ["combo/rescue", "chosen-combo", "policy/rescue", "chosen-policy", "pinned/gpt-5.3"]) { + cfg.codexMainAccountHardLockFallback = { model }; + expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)?.targetModel).toBe(model); + } + }); + + test("a stale qualified fallback never sends to the enabled default provider", async () => { + observeMainQuotaCredential("fixture-caller", "fallback-account-a"); + observe({ weeklyPercent: 100 }); + const cfg = config({ + providers: { + openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", baseUrl: "https://chatgpt.com/backend-api/codex" }, + gateway: { adapter: "openai-responses", authMode: "key", apiKey: "fixture-key", baseUrl: "https://gateway.example.test/v1" }, + }, + defaultProvider: "gateway", + codexMainAccountHardLockFallback: { model: "private/secret-model" }, + } as Partial); + let sends = 0; + const sentUrls: string[] = []; + globalThis.fetch = (async (input: unknown) => { + sends += 1; + sentUrls.push(String(input)); + return Response.json({ id: "resp_fixture", status: "completed", output: [] }); + }) as typeof fetch; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer fixture-caller", "chatgpt-account-id": "fallback-account-a" }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "fixture" }), + }), cfg, { model: "", provider: "" }); + await response.text(); + expect(sentUrls).toEqual([]); + expect(sends).toBe(0); + expect(response.status).toBeGreaterThanOrEqual(400); + }); + test("handleResponses strips caller credentials, preserves requestedModel on combo fallback, and yields to shadow/compaction turns", async () => { observe({ weeklyPercent: 100 }); // This caller is the observed main credential that actually owes the hard lock. From 943cf76a174f086ed9f2138893ef17f8f8d054bf Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 01:01:32 -0700 Subject: [PATCH 06/17] fix(codex): clear failure windows when account identities are removed Forget account-scoped failure samples and pinned degradation warnings during deletion and physical main-account replacement. Preserve sibling account windows and cover both lifecycle paths. --- src/codex/account-lifecycle.ts | 11 +++- structure/providers/openai-accounts.md | 2 +- .../codex-account-delete-atomicity.test.ts | 58 +++++++++++++++++++ .../codex-main-rotation.test.ts | 28 +++++++++ 4 files changed, 95 insertions(+), 4 deletions(-) diff --git a/src/codex/account-lifecycle.ts b/src/codex/account-lifecycle.ts index 65e35aa5f10..ffb7a5be540 100644 --- a/src/codex/account-lifecycle.ts +++ b/src/codex/account-lifecycle.ts @@ -14,6 +14,7 @@ import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account"; import { clearAccountQuota } from "./quota"; import { forgetManualPreference } from "./routing/active-account"; import { deleteAllHealthForAccount } from "./routing/health-store"; +import { forgetCodexFailureWindow } from "./routing/failure-window"; import { clearThreadAccountMapForAccount } from "./routing/thread-affinity"; import { clearCodexPoolRefreshFailure } from "./pool-refresh-backoff"; @@ -49,11 +50,15 @@ export function purgeCodexAccountRuntimeState(accountId: string): void { clearAccountNeedsReauth(accountId); clearAccountQuota(accountId); clearThreadAccountMapForAccount(accountId); - // Deliberately not via `./routing`: the account-delete path only needs the health rows and - // the manual preference gone, and importing the router here closed a cycle through - // routing/health-store and routing/active-account. + // Deliberately not via `./routing`: the account-delete path only needs the health rows + // and their sibling routing state gone, and importing the router barrel here closed a + // cycle through routing/health-store and routing/active-account. The leaf imports below + // carry every per-account forget that barrel helper used to perform — including the + // failure window, whose degrade and pin-warning state must not leak into a replacement + // account or a new physical login behind __main__. deleteAllHealthForAccount(accountId); forgetManualPreference(accountId); + forgetCodexFailureWindow(accountId); clearCodexPoolRefreshFailure(accountId); if (accountId === MAIN_CODEX_ACCOUNT_ID) { clearMainAccountInfoCache(); diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index ed6675f64ae..760631eb719 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-account-delete-atomicity.test.ts b/tests/codex-integration/codex-account-delete-atomicity.test.ts index b1df61a9400..9ec8ae90591 100644 --- a/tests/codex-integration/codex-account-delete-atomicity.test.ts +++ b/tests/codex-integration/codex-account-delete-atomicity.test.ts @@ -32,6 +32,12 @@ import { getConfigPath, loadConfig, saveConfig } from "../../src/config"; import * as configModule from "../../src/config"; import { setCodexAccountAutoSwitchThresholdOverride } from "../../src/codex/account-auto-switch"; import { clearAllManualPreferences, manualPreferenceBlocks, resetCodexRoutingForManualSelection } from "../../src/codex/routing/active-account"; +import { + failureWindowSteersNewThreads, + forgetCodexFailureWindow, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { POOL_KEY_CODEX } from "../../src/oauth/pool-kernel"; import { prepareConfigObjectChildDeletionRebase } from "../../src/config/rebase-provenance"; import type { OcxConfig } from "../../src/types"; @@ -429,4 +435,56 @@ describe("Codex account delete persistence ordering", () => { clearAllManualPreferences(); } }); + + test("deleting the account forgets its failure-window degradation but keeps sibling state", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (accountId: string): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, accountId, true, now - index); + }; + degrade(ACCOUNT_ID); + degrade("sibling-account"); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(true); + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + try { + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(false); + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + // A different account's window is untouched by the deletion. + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + } finally { + forgetCodexFailureWindow(ACCOUNT_ID); + forgetCodexFailureWindow("sibling-account"); + } + }); + + test("deleting the account forgets its pinned-degrade warning", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, ACCOUNT_ID, true, now - index); + }; + forgetCodexFailureWindow(ACCOUNT_ID); + degrade(); + const warnSpy = spyOn(console, "warn").mockImplementation(() => {}); + try { + // The seeded config pins ACCOUNT_ID under the default "hold" policy, so steering + // logs the pin-degraded warning exactly once per process-lifetime of the state. + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + expect(warnSpy).toHaveBeenCalledTimes(1); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + // Deletion clears the pin as well; re-pin to model a recreated same-id account the + // operator selects again. If the warned set survived the purge, a re-degraded + // same-id pinned account would stay silent forever; the forget lets it fire again. + config.activeCodexAccountPinned = ACCOUNT_ID; + degrade(); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(2); + } finally { + warnSpy.mockRestore(); + forgetCodexFailureWindow(ACCOUNT_ID); + } + }); }); diff --git a/tests/codex-integration/codex-main-rotation.test.ts b/tests/codex-integration/codex-main-rotation.test.ts index ffa98019c1e..bfc114c935d 100644 --- a/tests/codex-integration/codex-main-rotation.test.ts +++ b/tests/codex-integration/codex-main-rotation.test.ts @@ -21,6 +21,11 @@ import { reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests, } from "../../src/codex/account-lifecycle"; +import { + forgetCodexFailureWindows, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "../../src/codex/main-account"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { @@ -243,6 +248,29 @@ describe("main account rotation (Option A)", () => { } }); + test("a physical main account switch forgets its failure-window degradation", () => { + const config = makeConfig(); + const now = Date.now(); + for (let index = 0; index < 24; index += 1) { + noteCodexFailureWindowSample(config, MAIN_CODEX_ACCOUNT_ID, true, now - index); + } + try { + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(true); + // First reconcile observes the fixture identity; the physical switch happens on the + // replacement write below and must drop the alias-keyed window like every other + // runtime row the purge owns. + expect(reconcileMainCodexAccountRuntimeState()).toBe(false); + writeFileSync( + join(CODEX_DIR, "auth.json"), + JSON.stringify({ tokens: { access_token: "replacement_access", account_id: "replacement_acct" } }), + ); + expect(reconcileMainCodexAccountRuntimeState()).toBe(true); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(false); + } finally { + forgetCodexFailureWindows(); + } + }); + test("does not treat a transient missing auth file as an account switch", () => { expect(reconcileMainCodexAccountRuntimeState()).toBe(false); recordCodexUpstreamOutcome(makeConfig(), MAIN_CODEX_ACCOUNT_ID, 429, { retryAfter: "3600" }); From 29ea3e37a0d541d26060561e8cb49fadf1d595f2 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 02:18:11 -0700 Subject: [PATCH 07/17] fix(codex): fence late failure-window samples by dispatch identity Carry the main dispatch proof and stored Pool credential generation into terminal outcomes. Reject stale success and transient samples after identity or credential replacement, while preserving existing health and probe handling. Cover main identity ABA, same-ID Pool recreation, context reuse, and valid-current controls. --- src/codex/auth-context.ts | 4 + src/codex/routing.ts | 6 +- src/codex/routing/cooldown-math.ts | 5 + src/codex/routing/failure-window.ts | 23 +++ src/providers/openai-sidecar.ts | 5 + src/server/responses/compact.ts | 2 + src/server/responses/core-codex-account.ts | 35 ++-- src/server/responses/passthrough-delivery.ts | 2 + src/server/responses/passthrough-dispatch.ts | 2 + structure/providers/openai-accounts.md | 2 +- .../codex-failure-window.test.ts | 154 +++++++++++++++++- 11 files changed, 219 insertions(+), 21 deletions(-) diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index 577ee8ebad5..9c3841dde9e 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -324,6 +324,8 @@ export type CodexAuthContext = writerGeneration: number; /** Captured before async credential work; never reconstructed after the upstream response. */ mainQuotaWriter?: MainQuotaWriter; + /** Immutable credential proof captured while materializing this physical attempt's headers. */ + mainQuotaDispatch?: MainQuotaDispatch; reserveAuthorization?: MainReserveAuthorization; accessToken: string; chatgptAccountId: string; @@ -1778,6 +1780,8 @@ export function materializeCodexUpstreamAuth( if (ctx.kind === "main-pool") { ctx.mainQuotaWriter = observeSelectedMainCredential(ctx, ctx.mainQuotaWriter); assertMainAccountPolicy(options.config); + ctx.mainQuotaDispatch = ctx.mainQuotaWriter && isMainQuotaWriterLive(ctx.mainQuotaWriter) + ? selectedMainQuotaDispatch(selected) : undefined; } assertMaterializedReserve(selected, ctx, options); return selected; diff --git a/src/codex/routing.ts b/src/codex/routing.ts index 7eefb87e4e2..57bf246ec45 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -23,7 +23,7 @@ import { failureWindowSteersNewThreads, forgetCodexFailureWindow, forgetCodexFailureWindows, - noteCodexFailureWindowSample, + noteCodexFailureWindowOutcome, settleCodexSuccessStreak, } from "./routing/failure-window"; @@ -1305,7 +1305,7 @@ export function recordCodexUpstreamOutcome( */ dropSpentCredentialFailure(accountId); if (outcomeClass === "success") { - noteCodexFailureWindowSample(config, accountId, false, now); + noteCodexFailureWindowOutcome(config, accountId, false, now, meta); // The operator's one-shot is spent by a dispatch that actually worked, and only by that. // A failed lookup leaves it unspent so the intent survives the failure. consumeManualPreference(accountId, codexPoolKeyForScope(quotaScope)); @@ -1555,7 +1555,7 @@ export function recordCodexUpstreamOutcome( // A transient failure concludes an owning probe; an unrelated 5xx must not // consume someone else's live lease or drop hard-cooldown bookkeeping (#433). const transientBase = ownsProbeLease(current, meta) ? withProbeLeaseReleased(current!, now) : current; - noteCodexFailureWindowSample(config, accountId, true, now); + noteCodexFailureWindowOutcome(config, accountId, true, now, meta); const stale = current?.lastFailureAt ? now - current.lastFailureAt > CODEX_FAILURE_WINDOW_MS : false; const hardCooldownUntil = getCodexAccountCooldownUntil(accountId, now) ?? undefined; // Soft avoid + affinity clears are part of failover. When threshold is 0, leave diff --git a/src/codex/routing/cooldown-math.ts b/src/codex/routing/cooldown-math.ts index 3e03e029aae..515b4d9eb22 100644 --- a/src/codex/routing/cooldown-math.ts +++ b/src/codex/routing/cooldown-math.ts @@ -6,6 +6,7 @@ import { isThirtyDayOnlyCodexPlan } from "../plan"; import { hasSpendableCodexCredits, isTerminalShortWindow, type CodexSpendableCredits } from "../quota-types"; import type { CodexQuotaScope } from "./health-store"; import type { TransientProbeGrant } from "./thread-affinity"; +import type { MainQuotaDispatch } from "../main-account-cache"; export { TERMINAL_SHORT_WINDOW_FRESHNESS_MS } from "../quota-types"; @@ -99,6 +100,10 @@ export type CodexUpstreamOutcomeMeta = { * means the caller cannot supply lineage and the historical unfenced handling stands. */ credentialGeneration?: number; + /** Main credential proof captured before dispatch; null means no owned proof was available. */ + mainFailureWindowDispatch?: MainQuotaDispatch | null; + /** Pool sample provenance retained by a terminal callback even if its context is reused. */ + failureWindowCredentialGeneration?: number; }; export function computeCodexUsageScore(quota: { diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index 71aa43ebee0..f0cb2c4967f 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -1,5 +1,9 @@ import { pinnedCodexAccountId } from "../account-priority"; import type { OcxConfig } from "../../types"; +import { isCodexAccountGenerationLive } from "../account-store"; +import { isMainQuotaDispatchLive } from "../main-account-cache"; +import type { CodexAuthContext } from "../auth-context"; +import type { CodexUpstreamOutcomeMeta } from "./cooldown-math"; import { deleteAccountHealth, setAccountHealth, @@ -20,6 +24,25 @@ type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number const windows = new Map(); const pinnedDegradeWarned = new Set(); +/** Read only dispatch-time provenance retained by auth materialization, never recapture at arrival. */ +export function codexFailureWindowOutcomeMeta( + context: CodexAuthContext, +): Pick { + if (context.kind === "pool") return { failureWindowCredentialGeneration: context.generation }; + return context.kind === "main-pool" ? { mainFailureWindowDispatch: context.mainQuotaDispatch ?? null } : {}; +} + +/** Fence window samples only; other health and recovery-lease semantics stay with their owners. */ +export function noteCodexFailureWindowOutcome( + config: OcxConfig, accountId: string, failed: boolean, now: number, meta: CodexUpstreamOutcomeMeta, +): void { + if (meta.mainFailureWindowDispatch !== undefined + && (!meta.mainFailureWindowDispatch || !isMainQuotaDispatchLive(meta.mainFailureWindowDispatch))) return; + const generation = meta.failureWindowCredentialGeneration ?? meta.credentialGeneration; + if (generation !== undefined && !isCodexAccountGenerationLive(accountId, generation)) return; + noteCodexFailureWindowSample(config, accountId, failed, now); +} + export function codexFailureWindowEnabled(config: OcxConfig): boolean { return config.codexFailureWindow !== false; } diff --git a/src/providers/openai-sidecar.ts b/src/providers/openai-sidecar.ts index c68c0c73e7b..93e3398ae18 100644 --- a/src/providers/openai-sidecar.ts +++ b/src/providers/openai-sidecar.ts @@ -12,6 +12,7 @@ import { type CodexAuthPolicyConfig, } from "../codex/auth-context"; import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome } from "../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../codex/routing/failure-window"; import { inspectChatGptDomainClaim } from "../oauth/chatgpt"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential, type DataPlaneAdmission } from "../server/auth-cors"; import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "../types"; @@ -205,6 +206,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); throw new CodexPoolAuthenticationError("Selected Codex account is unavailable"); } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -220,6 +222,7 @@ export async function resolveFirstUsableOpenAiSidecar( probeLeaseId: authContext.probeLeaseId, probeQuotaScope: authContext.probeQuotaScope, writerGeneration: authContext.writerGeneration, + ...windowMeta, // 401/403 here is evidence about this exact stored credential; without the generation a // replacement inherits the quarantine (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), @@ -256,6 +259,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); continue; } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -271,6 +275,7 @@ export async function resolveFirstUsableOpenAiSidecar( threadId: authContext.affinityKey, probeLeaseId: authContext.probeLeaseId, writerGeneration: authContext.writerGeneration, + ...windowMeta, // Same fence as the exact-account recorder above (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), }, diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index cb93a79f280..32263104a72 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -88,6 +88,7 @@ import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome, } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountChangeConversationStateScrub, conversationStateBindingFromAuth, @@ -1062,6 +1063,7 @@ export async function handleResponsesCompact( if ((outcome === 401 || outcome === 403) && upstream && wasPermanentAccountRefusal(upstream)) return; recordCodexUpstreamOutcome(config, ctx.accountId, outcome, { ...meta, + ...codexFailureWindowOutcomeMeta(ctx), threadId: ctx.kind === "pool" || ctx.kind === "main-pool" ? ctx.affinityKey : undefined, fixedAccount: ctx.fixedAccount, modelId: selectedModelId, diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index a2b35f7da73..b51b7ce32c1 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -11,6 +11,7 @@ import { claimMainQuotaDispatchForWs, isMainQuotaDispatchLive, renewMainQuotaDis import type { CodexWsQuotaObserver } from "./codex-ws-metadata"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isCodexAccountGenerationLive } from "../../codex/account-store"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountQuotaFromUpstreamHeaders as applyCapturedCodexQuota } from "../../codex/quota"; import type { RouteResult } from "../../router"; import { @@ -93,20 +94,19 @@ export function sidecarOutcomeRecorder( config: OcxConfig, authCtx: CodexAuthContext, ): ((outcome: CodexUpstreamOutcome) => void) | undefined { - return authCtx.kind === "pool" || authCtx.kind === "main-pool" - ? outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { - threadId: authCtx.affinityKey, - fixedAccount: authCtx.fixedAccount, - probeLeaseId: authCtx.probeLeaseId, - probeQuotaScope: authCtx.probeQuotaScope, - writerGeneration: authCtx.writerGeneration, - // A vision or web-search sidecar can return 401/403, and that is evidence about the exact - // stored credential it used. Without the generation it becomes an account-wide quarantine - // that a replacement inherits (#2892 gap 4). `main-pool` has no stored-record generation, so - // it keeps the unfenced account-wide semantics. - ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), - }) - : undefined; + if (authCtx.kind !== "pool" && authCtx.kind !== "main-pool") return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); + return outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, + threadId: authCtx.affinityKey, + fixedAccount: authCtx.fixedAccount, + probeLeaseId: authCtx.probeLeaseId, + probeQuotaScope: authCtx.probeQuotaScope, + writerGeneration: authCtx.writerGeneration, + // Quarantine retains its stored-credential fence. main-pool has no stored-record + // generation; its separate dispatch proof above fences only failure-window samples. + ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), + }); } @@ -148,6 +148,7 @@ export function recordCodexPreOutputRefusalOutcome( if (!usesCodexForwardPoolAuth(authCtx, provider) || wasPermanentAccountRefusal(response)) return; noteCodexWsPreOutputRefusalOutcome(response); recordCodexUpstreamOutcome(config, authCtx.accountId, response.status, { + ...codexFailureWindowOutcomeMeta(authCtx), threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -676,6 +677,7 @@ export async function retryCodexPoolOnAlternateAccount( if (codexWsPreOutputRefusalOutcomeRecorded(firstResponse)) return; noteCodexWsPreOutputRefusalOutcome(firstResponse); recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, fixedAccount: firstAuthCtx.fixedAccount, modelId: route.modelId, @@ -693,6 +695,7 @@ export async function retryCodexPoolOnAlternateAccount( if (outcomeStatus === firstResponse.status || (outcomeStatus !== 429 && outcomeStatus !== 402)) return; recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...codexQuotaOutcomeMeta(firstResponse), + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -862,6 +865,7 @@ export async function retryCodexPoolOnAlternateAccount( } recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...quotaMeta, + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -1149,6 +1153,7 @@ export function codexForwardTerminalOutcomeRecorder( logCtx?: RequestLogContext, ): ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined { if (!usesCodexForwardPoolAuth(authCtx, provider)) return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); return (status, httpStatusOverride) => { const quotaStatus = [httpStatusOverride, logCtx?.terminalHttpStatus] .find(value => value === 429 || value === 402); @@ -1157,6 +1162,7 @@ export function codexForwardTerminalOutcomeRecorder( // request. Don't penalize account health; record success to clear any // prior soft-avoid so a healthy account isn't stuck avoided. recordCodexUpstreamOutcome(config, authCtx.accountId, 200, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -1180,6 +1186,7 @@ export function codexForwardTerminalOutcomeRecorder( ? 200 : (quotaStatus ?? httpStatusOverride ?? logCtx?.terminalHttpStatus ?? 502); recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 3d286dec672..cac782e0e28 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -37,6 +37,7 @@ import type { ResponsesTerminalStatus } from "../../bridge"; import { isCodexWsQuotaObservedResponse, isCodexWsUpstreamResponse, isCodexWsPreludeProjection } from "./ws-upstream"; import { recordSubagentQuotaFailureForThreadSpawn } from "../../codex/subagent-model-fallback"; import { recordCodexUpstreamOutcome } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { codexProbeLeaseId, codexProbeQuotaScope, codexTransientProbeGrant, releaseCodexAuthContextProbeLease } from "../../codex/auth-context"; import { consumeComboFailure } from "./core-combo-failure"; import { clientCancelledResponse, readDisplaySafeErrorText } from "./core-errors"; @@ -535,6 +536,7 @@ export async function deliverPassthroughResponse( // request. The sibling recorders on this path already decline: the terminal recorder // needs an ok streaming body, and the quota-header snapshot finds no quota headers. recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, upstreamResponse.status, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), ...quotaMeta, threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index e33d91206e2..21e9f091b64 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -18,6 +18,7 @@ import type { ResponsesEffects } from "./response-effects"; import type { ResponsesSendBudget } from "./request-send-budget"; import { transientSendCapFor } from "./request-send-budget"; import { renewMainQuotaDispatchForAttempt } from "../../codex/main-account-cache"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isLocalUpstream } from "../../lib/local-upstream"; import { codexSafetyBufferingFilterOptions, terminalStatusFromParsed } from "../relay"; @@ -990,6 +991,7 @@ export async function preparePassthroughExchange( } if (usesCodexForwardPoolAuth(admissionState.authCtx, route.provider)) { recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, outcome, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, modelId: route.modelId, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 760631eb719..a77fda304e4 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 55e5e773b24..3ab8478150b 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,8 +1,14 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import { headersForCodexAuthContext, type CodexAuthContext } from "../../src/codex/auth-context"; +import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { captureMainQuotaWriter, clearMainAccountInfoCache } from "../../src/codex/main-account-cache"; +import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/account-id"; +import { captureConfigGeneration } from "../../src/lib/state-store-sweeper"; +import { codexForwardTerminalOutcomeRecorder, sidecarOutcomeRecorder } from "../../src/server/responses/core-codex-account"; import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../../src/codex/auth-api"; import { clearCodexUpstreamHealth, @@ -16,7 +22,7 @@ import { noteCodexFailureWindowSample, } from "../../src/codex/routing/failure-window"; import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; -import type { OcxConfig } from "../../src/types"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const NOW = 1_700_000_000_000; @@ -47,13 +53,19 @@ function fill(accountId: string, failures: number, successes: number, now = NOW) let home = ""; let previousHome: string | undefined; +let previousCodexHome: string | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-window-")); setIcaclsRunnerForTests(() => ICACLS_OK); setAsyncIcaclsRunnerForTests(async () => ICACLS_OK); process.env.OPENCODEX_HOME = home; + process.env.CODEX_HOME = join(home, "codex"); + mkdirSync(process.env.CODEX_HOME, { recursive: true }); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); @@ -75,11 +87,147 @@ afterEach(async () => { clearThreadAccountMap(); clearAccountQuota(); forgetCodexFailureWindows(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; await removeTreeWithRetry(home); }); +const forwardProvider = { adapter: "openai-responses", authMode: "forward" } as OcxProviderConfig; + +function terminal(config: OcxConfig, context: CodexAuthContext) { + headersForCodexAuthContext(new Headers(), context, config); + return codexForwardTerminalOutcomeRecorder(config, context, forwardProvider)!; +} + +function mainTerminal(config: OcxConfig, identity: string, accessToken = `access-${identity}`) { + writeFileSync(join(process.env.CODEX_HOME!, "auth.json"), JSON.stringify({ tokens: { access_token: accessToken, account_id: identity } })); + reconcileMainCodexAccountRuntimeState(); + return terminal(config, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + mainQuotaWriter: captureMainQuotaWriter(identity), accessToken, chatgptAccountId: identity, fixedAccount: true, + }); +} + +function poolTerminal(config: OcxConfig) { + const record = readCodexAccountRecord("a")!; + return terminal(config, { + kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), generation: record.generation, + accessToken: record.credential!.accessToken, chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true, + }); +} + +test("late terminal outcomes cannot refill a replacement main failure window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const replacement = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) replacement("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + replacement("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a returned main identity does not revive the first login's terminal writer", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const first = mainTerminal(cfg, "physical-a"); + mainTerminal(cfg, "physical-b"); + const returned = mainTerminal(cfg, "physical-a"); + for (let index = 0; index < 20; index++) first("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) returned("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("late successful terminals cannot count toward the replacement main window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const current = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + old("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("sidecar recorder retains the old main dispatch even if its context is reused", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const context: CodexAuthContext = { kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, + writerGeneration: captureConfigGeneration(), mainQuotaWriter: captureMainQuotaWriter("physical-a"), + accessToken: "access-physical-a", chatgptAccountId: "physical-a" }; + headersForCodexAuthContext(new Headers(), context, cfg); + const old = sidecarOutcomeRecorder(cfg, context)!; + const current = mainTerminal(cfg, "physical-b"); + context.mainQuotaWriter = captureMainQuotaWriter("physical-b"); + context.accessToken = "access-physical-b"; + context.chatgptAccountId = "physical-b"; + headersForCodexAuthContext(new Headers(), context, cfg); + for (let index = 0; index < 20; index++) old(500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a replaced main credential retires its terminal samples without an identity change", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a", "old-access"); + const current = mainTerminal(cfg, "physical-a", "new-access"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + current("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("same-id pool recreation rejects the deleted credential's late terminal samples", () => { + const cfg = config(); + const old = poolTerminal(cfg); + deleteCodexAccount(cfg, "a"); + saveCodexAccountCredential("a", { accessToken: "replacement-access", refreshToken: "replacement-refresh", chatgptAccountId: "replacement-account", expiresAt: Date.now() + 60_000 }); + cfg.codexAccounts!.push({ id: "a", email: "replacement@test", isMain: false }); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("terminal pool samples keep the dispatched generation if the context is reused", () => { + const cfg = config(); + const record = readCodexAccountRecord("a")!; + const context: CodexAuthContext = { kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), + generation: record.generation, accessToken: record.credential!.accessToken, + chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true }; + const old = terminal(cfg, context); + context.generation = saveCodexAccountCredential("a", { ...record.credential!, accessToken: "new-access" }); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a main terminal without an owned dispatch proof cannot seed the window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const record = codexForwardTerminalOutcomeRecorder(cfg, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + accessToken: "unobserved-access", chatgptAccountId: "physical-a", + }, forwardProvider)!; + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); +}); + test("fewer than 20 samples do not degrade an account", () => { fill("a", 19, 0); expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); From 07dc642d40eb60cfd483594f87c3838fee3b08df Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 02:59:34 -0700 Subject: [PATCH 08/17] fix(codex): revalidate tagged failure samples after credential replacement Retain each stored Pool sample's dispatch credential generation and filter it against one current store snapshot per insertion or lookup. Reset retired-generation degradation, recovery timers and pin warnings without clearing a replacement on stale arrival. Cover both unlocked-read boundaries, derived state and the 512-sample snapshot bound. --- src/codex/routing/failure-window.ts | 45 ++++-- structure/providers/openai-accounts.md | 2 +- .../codex-failure-window.test.ts | 129 +++++++++++++++++- 3 files changed, 163 insertions(+), 13 deletions(-) diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index f0cb2c4967f..4610867e4b7 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -1,6 +1,6 @@ import { pinnedCodexAccountId } from "../account-priority"; import type { OcxConfig } from "../../types"; -import { isCodexAccountGenerationLive } from "../account-store"; +import { beginCodexAccountGenerationLiveCheck, isCodexAccountGenerationLive } from "../account-store"; import { isMainQuotaDispatchLive } from "../main-account-cache"; import type { CodexAuthContext } from "../auth-context"; import type { CodexUpstreamOutcomeMeta } from "./cooldown-math"; @@ -18,8 +18,8 @@ export const CODEX_SLIDING_FAILURE_RECOVER_RATIO = 0.1; export const CODEX_SLIDING_FAILURE_RECOVER_SUSTAIN_MS = 30_000; const MAX_SAMPLES = 512; -type Sample = { at: number; failed: boolean }; -type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number }; +type Sample = { at: number; failed: boolean; credentialGeneration?: number }; +type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number; credentialGeneration?: number }; const windows = new Map(); const pinnedDegradeWarned = new Set(); @@ -40,7 +40,7 @@ export function noteCodexFailureWindowOutcome( && (!meta.mainFailureWindowDispatch || !isMainQuotaDispatchLive(meta.mainFailureWindowDispatch))) return; const generation = meta.failureWindowCredentialGeneration ?? meta.credentialGeneration; if (generation !== undefined && !isCodexAccountGenerationLive(accountId, generation)) return; - noteCodexFailureWindowSample(config, accountId, failed, now); + appendSample(config, accountId, failed, now, generation); } export function codexFailureWindowEnabled(config: OcxConfig): boolean { @@ -56,11 +56,25 @@ function stateFor(accountId: string): WindowState { return state; } -function recompute(state: WindowState, now: number): void { - const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; - if (state.samples.length > 0 && state.samples[0]!.at <= cutoff) { - state.samples = state.samples.filter(sample => sample.at > cutoff); +function recompute( + accountId: string, state: WindowState, now: number, + generationLive?: ReturnType, +): void { + // One store snapshot per insertion/read, never one file load per retained sample. The + // unlocked insertion precheck can race another process; provenance must survive it. + const tagged = state.credentialGeneration !== undefined + || state.samples.some(sample => sample.credentialGeneration !== undefined); + const live = generationLive ?? (tagged ? beginCodexAccountGenerationLiveCheck() : undefined); + if (state.credentialGeneration !== undefined && !live?.(accountId, state.credentialGeneration)) { + state.degraded = false; + state.healthySince = undefined; + state.credentialGeneration = undefined; + pinnedDegradeWarned.delete(accountId); } + const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; + state.samples = state.samples.filter(sample => sample.at > cutoff + && (sample.credentialGeneration === undefined || live?.(accountId, sample.credentialGeneration))); + state.credentialGeneration ??= state.samples.find(sample => sample.credentialGeneration !== undefined)?.credentialGeneration; const samples = state.samples.length; const failed = state.samples.reduce((count, sample) => count + (sample.failed ? 1 : 0), 0); const degraded = samples >= CODEX_SLIDING_FAILURE_MIN_SAMPLES @@ -94,19 +108,28 @@ export function noteCodexFailureWindowSample( accountId: string, failed: boolean, now: number, +): void { + appendSample(config, accountId, failed, now); +} + +function appendSample( + config: OcxConfig, accountId: string, failed: boolean, now: number, credentialGeneration?: number, ): void { if (!codexFailureWindowEnabled(config)) return; + const live = credentialGeneration === undefined ? undefined : beginCodexAccountGenerationLiveCheck(); + // A stale arrival is not an ownership transition and must not delete a live replacement's state. + if (credentialGeneration !== undefined && !live?.(accountId, credentialGeneration)) return; const state = stateFor(accountId); - state.samples.push({ at: now, failed }); + state.samples.push({ at: now, failed, ...(credentialGeneration === undefined ? {} : { credentialGeneration }) }); if (state.samples.length > MAX_SAMPLES) state.samples.splice(0, state.samples.length - MAX_SAMPLES); - recompute(state, now); + recompute(accountId, state, now, live); if (!state.degraded) pinnedDegradeWarned.delete(accountId); } export function isCodexFailureWindowDegraded(accountId: string, now = Date.now()): boolean { const state = windows.get(accountId); if (!state) return false; - recompute(state, now); + recompute(accountId, state, now); if (!state.degraded) pinnedDegradeWarned.delete(accountId); return state.degraded; } diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index a77fda304e4..a6e457742f1 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 3ab8478150b..c48eeea7a9d 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,8 +1,9 @@ -import { afterEach, beforeEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import * as accountStore from "../../src/codex/account-store"; import { headersForCodexAuthContext, type CodexAuthContext } from "../../src/codex/auth-context"; import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; import { captureMainQuotaWriter, clearMainAccountInfoCache } from "../../src/codex/main-account-cache"; @@ -18,6 +19,7 @@ import { } from "../../src/codex/routing"; import { forgetCodexFailureWindows, + failureWindowSteersNewThreads, isCodexFailureWindowDegraded, noteCodexFailureWindowSample, } from "../../src/codex/routing/failure-window"; @@ -228,6 +230,131 @@ test("a main terminal without an owned dispatch proof cannot seed the window", ( expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); }); +test("a cross-process replacement after the unlocked live check cannot donate a terminal sample", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.isCodexAccountGenerationLive; + let swapped = false; + const liveCheck = spyOn(accountStore, "isCodexAccountGenerationLive").mockImplementation((id, generation) => { + const live = original(id, generation); + if (id === "a" && live && !swapped) { + swapped = true; + const credential = readCodexAccountRecord(id)!.credential!; + saveCodexAccountCredential(id, { ...credential, accessToken: "replacement-after-read" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + liveCheck.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a replacement does not inherit degraded state or the prior pinned warning", () => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const old = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-access" }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(2); + // Rejecting a stale writer must not erase the replacement's healthy or degraded window. + old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + } finally { + warn.mockRestore(); + } +}); + +test("a replacement after the insertion snapshot is filtered at window read time", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.beginCodexAccountGenerationLiveCheck; + let swapped = false; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(() => { + const live = original(); + if (!swapped) { + swapped = true; + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-snapshot" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + snapshot.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a full tagged window loads one current credential snapshot per read", () => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 512; index++) { + recordCodexUpstreamOutcome(cfg, "a", 200, { failureWindowCredentialGeneration: generation, now: NOW }); + } + const original = accountStore.beginCodexAccountGenerationLiveCheck; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(original); + try { + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "next-full-window" }); + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(2); + } finally { + snapshot.mockRestore(); + } +}); + +test("a replacement does not inherit the previous generation's recovery timer", () => { + const cfg = config(); + const firstGeneration = readCodexAccountRecord("a")!.generation; + const record = (outcome: number, generation: number, now: number) => recordCodexUpstreamOutcome(cfg, "a", outcome, + { failureWindowCredentialGeneration: generation, now }); + const fillGeneration = (generation: number, failed: boolean, now: number) => { + for (let index = 0; index < 20; index++) record(failed ? 500 : 200, generation, now); + }; + fillGeneration(firstGeneration, true, NOW); + fillGeneration(firstGeneration, false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + const credential = readCodexAccountRecord("a")!.credential!; + const nextGeneration = saveCodexAccountCredential("a", { ...credential, accessToken: "next-access" }); + fillGeneration(nextGeneration, false, NOW + 62_000); + expect(isCodexFailureWindowDegraded("a", NOW + 62_000)).toBe(false); + fillGeneration(nextGeneration, true, NOW + 63_000); + expect(isCodexFailureWindowDegraded("a", NOW + 63_000)).toBe(true); + // Only G2 successes remain in-window; its own 30-second recovery starts here. + fillGeneration(nextGeneration, false, NOW + 124_000); + expect(isCodexFailureWindowDegraded("a", NOW + 153_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 154_000)).toBe(false); +}); + test("fewer than 20 samples do not degrade an account", () => { fill("a", 19, 0); expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); From 19d8843db9cd44618faf2f5ab99c3ef7275cd948 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 04:00:41 -0700 Subject: [PATCH 09/17] fix(codex): preserve failure windows when credential snapshots fail Distinguish failed reads or parsing from valid absent/empty credential stores. Preserve retained samples and derived failure-window state while evidence is unknown, keeping existing boolean liveness callers compatible. Cover fixture read/parse failures, recovery and empty-store retirement without adding snapshot reuse optimization. --- src/codex/account-store.ts | 34 +++++-- src/codex/routing/failure-window.ts | 8 +- structure/providers/openai-accounts.md | 2 +- .../codex-failure-window.test.ts | 88 ++++++++++++++++++- 4 files changed, 121 insertions(+), 11 deletions(-) diff --git a/src/codex/account-store.ts b/src/codex/account-store.ts index bfe677c2d8e..b89047c8919 100644 --- a/src/codex/account-store.ts +++ b/src/codex/account-store.ts @@ -136,10 +136,17 @@ export function peekCodexAccountRecordStore(): Record boolean) + & { readonly snapshotStatus: "ok" | "failed" }; + /** * The same verdict as {@link isCodexAccountGenerationLive}, over ONE store load. * @@ -409,12 +424,15 @@ export function isCodexAccountGenerationLive(id: string, generation: number): bo * several accounts in one synchronous pass on the request path, so it opens one checker and * closes over the snapshot instead. */ -export function beginCodexAccountGenerationLiveCheck(): (id: string, generation: number) => boolean { - const snapshot = loadCodexAccountRecordSnapshot(); - return (id, generation) => { +export function beginCodexAccountGenerationLiveCheck(): CodexAccountGenerationLiveCheck { + let result: CodexAccountRecordStoreResult; + try { result = loadCodexAccountRecordStoreResult(); } + catch { result = { status: "failed" }; } + const snapshot = result.status === "ok" ? result.records : {}; + return Object.assign((id: string, generation: number) => { const record = snapshot[id]; return !!record?.credential && record.deletedAt == null && record.generation === generation; - }; + }, { snapshotStatus: result.status }); } export function saveCodexAccountCredentialIfGeneration( diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index 4610867e4b7..61ae76149be 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -65,6 +65,8 @@ function recompute( const tagged = state.credentialGeneration !== undefined || state.samples.some(sample => sample.credentialGeneration !== undefined); const live = generationLive ?? (tagged ? beginCodexAccountGenerationLiveCheck() : undefined); + // Unknown evidence is not a credential removal: preserve samples and all derived state. + if (live?.snapshotStatus === "failed") return; if (state.credentialGeneration !== undefined && !live?.(accountId, state.credentialGeneration)) { state.degraded = false; state.healthySince = undefined; @@ -116,7 +118,11 @@ function appendSample( config: OcxConfig, accountId: string, failed: boolean, now: number, credentialGeneration?: number, ): void { if (!codexFailureWindowEnabled(config)) return; - const live = credentialGeneration === undefined ? undefined : beginCodexAccountGenerationLiveCheck(); + const existing = windows.get(accountId); + const tagged = credentialGeneration !== undefined || existing?.credentialGeneration !== undefined + || existing?.samples.some(sample => sample.credentialGeneration !== undefined); + const live = tagged ? beginCodexAccountGenerationLiveCheck() : undefined; + if (live?.snapshotStatus === "failed") return; // A stale arrival is not an ownership transition and must not delete a live replacement's state. if (credentialGeneration !== undefined && !live?.(accountId, credentialGeneration)) return; const state = stateFor(accountId); diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index a6e457742f1..04e3259b9ab 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. A snapshot read or parse failure is explicitly unknown and preserves the samples, degraded flag, recovery timer, retained generation and pin warning; a genuinely absent or valid empty store still retires its old generations. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index c48eeea7a9d..6a3a123ae7e 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import * as fs from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; @@ -332,6 +333,91 @@ test("a full tagged window loads one current credential snapshot per read", () = } }); +function withFailedStoreSnapshot(kind: "read" | "parse", run: () => void): void { + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const original = fs.readFileSync; + const read = kind === "read" ? spyOn(fs, "readFileSync").mockImplementation((...args: Parameters) => { + if (String(args[0]) === path) throw Object.assign(new Error("fixture read unavailable"), { code: "EACCES" }); + return original(...args); + }) : undefined; + if (kind === "parse") writeFileSync(path, "{incomplete-json"); + try { + run(); + } finally { + read?.mockRestore(); + writeFileSync(path, bytes); + } +} + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves degraded samples and the pinned warning", kind => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + withFailedStoreSnapshot(kind, () => { + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + } +}); + +test.each(["read", "parse"] as const)("a temporary %s failure does not erase or append to nineteen retained samples", kind => { + const cfg = config(); + const record = poolTerminal(cfg); + for (let index = 0; index < 19; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + withFailedStoreSnapshot(kind, () => { + noteCodexFailureWindowSample(cfg, "a", true, Date.now()); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves the recovery timer and retained generation", kind => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + const record = (failed: boolean, now: number) => recordCodexUpstreamOutcome(cfg, "a", failed ? 500 : 200, + { failureWindowCredentialGeneration: generation, now }); + for (let index = 0; index < 20; index++) record(true, NOW); + for (let index = 0; index < 20; index++) record(false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + withFailedStoreSnapshot(kind, () => expect(isCodexFailureWindowDegraded("a", NOW + 75_000)).toBe(true)); + expect(isCodexFailureWindowDegraded("a", NOW + 90_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 91_000)).toBe(false); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-failure" }); + const current = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 19; index++) recordCodexUpstreamOutcome(cfg, "a", 500, + { failureWindowCredentialGeneration: current, now: NOW + 92_000 }); + expect(isCodexFailureWindowDegraded("a", NOW + 92_000)).toBe(false); +}); + +test.each(["absent", "empty"] as const)("a valid %s snapshot still retires the previous generation", kind => { + const cfg = config(); + const old = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + const path = join(home, "codex-accounts.json"); + if (kind === "absent") unlinkSync(path); + else writeFileSync(path, "{}"); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("ok"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); +}); + test("a replacement does not inherit the previous generation's recovery timer", () => { const cfg = config(); const firstGeneration = readCodexAccountRecord("a")!.generation; From 404006d979ed15e835bdadfb80736212f1e49f80 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sun, 18 Oct 2026 04:31:21 -0700 Subject: [PATCH 10/17] fix(codex): reject non-object credential snapshot shapes Require a non-null, non-array object before normalizing store rows. Preserve failure-window evidence for invalid top-level arrays or scalars while keeping missing files and empty objects valid. Cover six top-level shapes without changing row normalization. --- src/codex/account-store.ts | 1 + .../codex-failure-window.test.ts | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/codex/account-store.ts b/src/codex/account-store.ts index b89047c8919..b384ab5588b 100644 --- a/src/codex/account-store.ts +++ b/src/codex/account-store.ts @@ -150,6 +150,7 @@ function loadCodexAccountRecordStoreResult(): CodexAccountRecordStoreResult { const normalized: CodexAccountStore = {}; try { const raw = JSON.parse(readFileSync(path, "utf-8")) as RawCodexAccountStore; + if (!isObject(raw)) throw new Error("invalid Codex account store shape"); for (const [id, value] of Object.entries(raw)) { const record = normalizeRecord(value); if (record) normalized[id] = record; diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 6a3a123ae7e..ee16af342df 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -418,6 +418,32 @@ test.each(["absent", "empty"] as const)("a valid %s snapshot still retires the p expect(isCodexFailureWindowDegraded("a")).toBe(false); }); +test.each(["[]", "false", "0", "null", "\"\"", "\"text\""])("a wrong-shaped top-level snapshot %s preserves the failure window", contents => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, contents); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, bytes); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + writeFileSync(path, bytes); + } +}); + test("a replacement does not inherit the previous generation's recovery timer", () => { const cfg = config(); const firstGeneration = readCodexAccountRecord("a")!.generation; From 9d5c2b07f4a977de4918820d5de23920f378a5b9 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 20 Oct 2026 05:50:29 +0900 Subject: [PATCH 11/17] refactor(responses): keep passthrough-dispatch within its line budget --- src/codex/routing.ts | 1 + src/server/responses/passthrough-dispatch.ts | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codex/routing.ts b/src/codex/routing.ts index 57bf246ec45..5008d7ac569 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -220,6 +220,7 @@ export { isEffectiveCodexAccountPinned, } from "./routing/active-account"; export { codexAccountPinDrainReason } from "./routing/pin-drain"; +export { codexFailureWindowOutcomeMeta } from "./routing/failure-window"; export type { CodexPinDrainReason } from "./routing/pin-drain"; // A shared refresh can outlive the request that opened it. Register the affinity diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 21e9f091b64..08415fc8956 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -18,7 +18,6 @@ import type { ResponsesEffects } from "./response-effects"; import type { ResponsesSendBudget } from "./request-send-budget"; import { transientSendCapFor } from "./request-send-budget"; import { renewMainQuotaDispatchForAttempt } from "../../codex/main-account-cache"; -import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isLocalUpstream } from "../../lib/local-upstream"; import { codexSafetyBufferingFilterOptions, terminalStatusFromParsed } from "../relay"; @@ -133,7 +132,7 @@ import { import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error"; import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability"; import { captureAccountAutoPauseAttempt, wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; -import { recordCodexUpstreamOutcome } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta, recordCodexUpstreamOutcome } from "../../codex/routing"; import { describeUpstreamConnectFailure } from "./upstream-error"; import type { OpaqueBlobRecoveryGuard } from "./core-opaque-recovery"; import { From b5de9ca0d2c4bbc5d040f27c8928b8d40146142a Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 20 Oct 2026 05:51:16 +0900 Subject: [PATCH 12/17] fix(codex): keep a paused main caller out of Pool fallback Carries the paused-main regression from #7290 and the production guard it proves is needed, plus a per-generation credit-renewal pacing test requested at intake. Co-authored-by: Hylouis233 <88263959+Hylouis233@users.noreply.github.com> --- src/codex/auth-context.ts | 2 ++ .../codex-auth-context.test.ts | 17 +++++++++++ .../codex-quota-prime.test.ts | 29 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index 9c3841dde9e..e27a2e0ed01 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1446,6 +1446,7 @@ async function resolveCodexAuthContextUnannotated( if ( requestScopedMainCredential && fixedAccountId === undefined + && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID ) { return await resolveCallerOwnedMainContext(); @@ -1604,6 +1605,7 @@ async function resolveCodexAuthContextUnannotated( // alternate is eligible. A validated caller may still serve this request, // just as it can after an upstream rejection, without changing Pool state. if (requestScopedMainCredential && fixedAccountId === undefined + && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID && !callerIsCooledPoolAccount(headers, config, accountId)) { return await resolveCallerOwnedMainContext(); diff --git a/tests/codex-integration/codex-auth-context.test.ts b/tests/codex-integration/codex-auth-context.test.ts index 9f1244eb43e..cdd00a433db 100644 --- a/tests/codex-integration/codex-auth-context.test.ts +++ b/tests/codex-integration/codex-auth-context.test.ts @@ -2431,3 +2431,20 @@ test("context Direct bearer admission uses real stored-main materialization and globalThis.fetch = originalFetch; } }); + +test("pool exhaustion never falls back to an observed paused main caller", async () => { + const cfg = config(); + cfg.codexAccounts = []; + cfg.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + cfg.codexMainAccountHardLock = false; + observeMainQuotaIdentity("paused-main-account"); + observeMainQuotaCredential("paused-main-bearer", "paused-main-account"); + const headers = new Headers({ authorization: "Bearer paused-main-bearer", "chatgpt-account-id": "paused-main-account" }); + await expect(resolveCodexAuthContext(headers, cfg, "pool", { + requestScopedMainCredential: true, + })).rejects.toBeInstanceOf(CodexMainProfileDrainingError); + // Direct remains caller-owned; the pause applies to Pool fallback, not unrelated callers. + expect(await resolveCodexAuthContext(headers, cfg, "direct", { + requestScopedMainCredential: true, + })).toMatchObject({ kind: "main", accountId: null }); +}); diff --git a/tests/codex-integration/codex-quota-prime.test.ts b/tests/codex-integration/codex-quota-prime.test.ts index 3ccd538b678..f972cdeb24c 100644 --- a/tests/codex-integration/codex-quota-prime.test.ts +++ b/tests/codex-integration/codex-quota-prime.test.ts @@ -226,6 +226,35 @@ describe("primeCodexPoolQuotas", () => { } finally { globalThis.fetch = originalFetch; } }); + test("credit renewal pacing is per credential generation and coalesces concurrent sweeps (#7290 intake)", async () => { + const config = makeConfig({ creditCodexAccountIds: ["p1"] }); + seedPoolAccount(config, "p1", "plus"); + const now = Date.now(); + setAccountQuotaFromParsed("p1", { shortPercent: 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - 360_000 } }); + const originalFetch = globalThis.fetch; + const tokens: string[] = []; + try { + globalThis.fetch = (async (input, init) => { + expect(String(input)).toContain("/backend-api/wham/usage"); + expect(init?.method ?? "GET").toBe("GET"); + tokens.push(new Headers(init?.headers).get("authorization") ?? ""); + return new Response(null, { status: 503 }); + }) as typeof fetch; + // Two sweeps racing on one generation share one metadata read. + await Promise.all([primeCodexPoolQuotas(config, "credit-renewal"), primeCodexPoolQuotas(config, "credit-renewal")]); + expect(tokens).toEqual(["Bearer access-p1"]); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(tokens).toHaveLength(1); + // Replacing the credential starts a new generation, which is not bound by the old failure. + saveCodexAccountCredential("p1", { accessToken: "access-p1-next", refreshToken: "refresh-p1-next", + expiresAt: Date.now() + 5 * 60_000, chatgptAccountId: "acct-p1" }); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(tokens).toEqual(["Bearer access-p1", "Bearer access-p1-next"]); + expect(isCodexAccountHeldForCredits(config, "p1", getAccountQuota("p1"), "plus", Date.now())).toBe(true); + } finally { globalThis.fetch = originalFetch; } + }); + test("startup may observe pending account quota without inference or validation", async () => { const config = makeConfig(); seedPoolAccount(config, "pending", "pro"); From 177a96757be79f86d4635d877408f174a73f2312 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 20 Oct 2026 05:53:34 +0900 Subject: [PATCH 13/17] fix(codex): recheck a predicted-reset main hard lock with metadata (#7312) A predicted reset now arms the capped WHAM recovery schedule instead of suppressing it, so an early reset redeemed outside opencodex is learned from verified metadata. The schedule never runs past the predicted reset, skips paused and needs-reauth mains, and a predicted reset still never lifts the lock by itself. --- .../docs/reference/cli/providers-accounts.md | 7 +- src/codex/auth-api/pool-mode-gate.ts | 28 +++++-- structure/providers/openai-tiers.md | 8 +- .../main-account-hard-lock-recovery.test.ts | 76 ++++++++++++++++--- 4 files changed, 96 insertions(+), 23 deletions(-) diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 946d51fd489..06d3e75eaf3 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -470,9 +470,10 @@ reset changes and expires after six hours or at the observed reset, whichever co An unreadable 5h reading cannot hide a weekly block. Unknown usage does not fabricate a zero, and a missing reading does not erase an already measured blocking tuple. A predicted reset time alone does not unlock it. -While blocked, the minute sweep waits for the latest known blocking reset, then checks owned usage. -If no future reset is known or a check remains blocked, recovery uses a capped 5/10/20/40/60-minute -schedule; a longer `Retry-After` also delays profile and token preparation. Only fresh valid usage or authoritative window-absence evidence +While blocked, recovery checks owned usage metadata on a capped 5/10/20/40/60-minute schedule, so a reset +redeemed outside opencodex is noticed before the predicted reset; the first check waits five minutes, no +check is scheduled past the latest known blocking reset, and a paused or reauthentication-required main is +not checked. A longer `Retry-After` also delays profile and token preparation. Only fresh valid usage or authoritative window-absence evidence can lift the block. In Pool mode, a quota `--refresh` bypasses cache freshness but still honors failed-read pacing; a deferred read makes no new diagnostic attempt. Other pause, reauthentication, and upstream limits remain independent. diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index fc29bd2410e..20dfc8f69d7 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -96,6 +96,12 @@ function poolCreditRenewalDue(config: OcxConfig, accountId: string, now = Date.n let mainHardLockRecoveryInFlight: Promise | null = null; let mainHardLockRecoveryAttempt: { identity: number; credential: number; after: number; delay: number } | undefined; +/** The predicted reset of a blocked main, or undefined when no future reset is known. */ +function predictedMainResetHold(status: ReturnType): number | undefined { + return status.state === "blocked" && status.resetAt !== undefined && status.resetAt > Date.now() + ? status.resetAt : undefined; +} + /** Metadata-only recovery on the existing sweep; failures retain the observed policy block. */ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise { if (mainHardLockRecoveryInFlight) return mainHardLockRecoveryInFlight; @@ -115,8 +121,11 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise const status = getMainAccountHardLockStatus(config); if (status.state !== "blocked" && !creditRecoveryNeeded()) { mainHardLockRecoveryAttempt = undefined; return; } if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; - // A predicted reset is a scheduling hint, never proof that the hard lock can be lifted. - if (status.state === "blocked" && status.resetAt !== undefined && status.resetAt > Date.now()) return; + // A predicted reset is a scheduling hint, never proof that the hard lock can be lifted. The + // hold is rechecked on the capped WHAM schedule so an early reset redeemed elsewhere is + // learned from metadata (#7312); a paused main receives no hold rechecks. + const holdResetAt = predictedMainResetHold(status); + if (holdResetAt !== undefined && isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID)) return; // Reconcile the owned physical credential before skipping work for an older one. reconcileMainCodexAccountRuntimeState(); const physical = readCodexTokens(); @@ -124,6 +133,12 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise const identity = captureMainAccountIdentityGeneration(); const credential = getMainQuotaCredentialGeneration(); const previous = mainHardLockRecoveryAttempt; + if (holdResetAt !== undefined && (previous?.identity !== identity || previous.credential !== credential)) { + // The block was just observed: wait one base interval, and never past the predicted reset. + const delay = nextQuotaQueryDelay(); + mainHardLockRecoveryAttempt = { identity, credential, delay, after: Math.min(Date.now() + delay, holdResetAt) }; + return; + } const queryAfter = nextCodexUsageQueryAt(`main:${captureConfigGeneration()}:${credential}`) ?? 0; if ((previous?.identity === identity && previous.credential === credential && previous.after > Date.now()) || queryAfter > Date.now()) return; @@ -145,12 +160,11 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise // Native refresh retains its own grant-scoped refusal; global quarantine would outlive it. return; } - const currentStatus = getMainAccountHardLockStatus(config); + const currentHold = predictedMainResetHold(getMainAccountHardLockStatus(config)); if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID) || !isMainAccountIdentityGenerationLive(identityGeneration) || (passive && !creditRecoveryNeeded()) - || (currentStatus.state === "blocked" && currentStatus.resetAt !== undefined - && currentStatus.resetAt > Date.now())) return; + || (currentHold !== undefined && isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID))) return; const credential = getMainQuotaCredentialGeneration(); // Reconcile the physical bearer before consulting the previous credential's delay. if (previous?.identity === identityGeneration && previous.credential === credential @@ -177,8 +191,10 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise if (transientAuth) mainHardLockRecoveryAttempt = undefined; else { const delay = nextQuotaQueryDelay(previousDelay || undefined); + const hold = predictedMainResetHold(getMainAccountHardLockStatus(config)); mainHardLockRecoveryAttempt = getMainAccountHardLockStatus(config).state === "blocked" || creditRecoveryNeeded() - ? { identity: identityGeneration, credential, delay, after: Math.max(Date.now() + delay, queryAfter) } + ? { identity: identityGeneration, credential, delay, + after: Math.max(Math.min(Date.now() + delay, hold ?? Infinity), queryAfter) } : undefined; } } else if (queryAfter > Date.now()) { diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 854c502fc6c..6ceae204e81 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -378,9 +378,11 @@ It blocks newly admitted identity-matched main-account requests. Pool alternativ explicit main selection and stored Direct substitution do not override it. It neither pauses the account nor clears upstream cooldown/reauth state, and management quota refresh remains available. Fresh valid usage below the effective threshold, including 0%, or validated WHAM absence retires a measured short block; -passing a reset timestamp alone does not. The minute sweep waits locally until the latest known blocking reset; -when no future reset is known or reads remain blocked, main recovery uses the same capped -5/10/20/40/60-minute delay calculation as usage-query failures. Skipped ticks do not extend it; +passing a reset timestamp alone does not. A future known reset is a scheduling hint (#7312): recovery +arms one base delay when the hold is first seen, then reads WHAM metadata on the same capped +5/10/20/40/60-minute delay calculation as usage-query failures, never scheduling a read past that +reset, so an early reset redeemed elsewhere is learned without inference or reset-credit use. A paused +or needs-reauth main gets no hold rechecks. Skipped ticks do not extend the delay; the physical bearer is reconciled before checking the delay, and late results cannot charge a replacement credential. A longer valid Retry-After from any main usage reader is checked for the current credential before the recovery worker takes a profile lease or prepares a token; a replacement diff --git a/tests/codex-integration/main-account-hard-lock-recovery.test.ts b/tests/codex-integration/main-account-hard-lock-recovery.test.ts index 72f7bac7a25..5cd5c61ddfa 100644 --- a/tests/codex-integration/main-account-hard-lock-recovery.test.ts +++ b/tests/codex-integration/main-account-hard-lock-recovery.test.ts @@ -70,6 +70,13 @@ function usage(percent = 0): Response { } }); } +/** A metadata read that still reports the block with a future predicted reset. */ +function heldUsage(resetAt: number): Response { + return Response.json({ plan_type: "plus", rate_limit: { + primary_window: { used_percent: 99, limit_window_seconds: 18_000, reset_at: Math.floor(resetAt / 1000) }, + } }); +} + /** * Stub recovery HTTP calls, requiring a known metadata/token URL and an active native-main drain. * Return the captured URL list so tests can verify the requests made by background recovery. @@ -136,23 +143,38 @@ afterEach(async () => { }); describe("main hard-lock background recovery", () => { - test("known reset waits locally, then verifies recovery instead of unlocking by time", async () => { + test("known reset rechecks metadata on a capped schedule and never unlocks by time (#7312)", async () => { let now = Date.now(); const clock = spyOn(Date, "now").mockImplementation(() => now); try { - const deadline = now + 3_600_000; + const deadline = now + 6 * 3_600_000; setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, shortResetAt: deadline }, undefined, captureMainQuotaWriter(accountId)); - const calls = fetchWith(async () => usage(99)); - for (let tick = 0; tick < 60; tick++, now += 60_000) await runMainAccountHardLockRecovery(config()); - expect(calls).toEqual([]); - expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); - await runMainAccountHardLockRecovery(config()); - expect(calls).toEqual([whamUrl]); - now += 60_000; + const readAt: number[] = []; + const calls = fetchWith(async (_url, init) => { + // Metadata only: a GET of WHAM usage, never inference or reset redemption. + expect(init?.method ?? "GET").toBe("GET"); + readAt.push(now); + return heldUsage(deadline); + }); + const start = now; + for (; now < deadline; now += 60_000) { + await runMainAccountHardLockRecovery(config()); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + } + expect(new Set(calls)).toEqual(new Set([whamUrl])); + // The block was just observed, so the first recheck waits one base interval. + expect(readAt[0]! - start).toBeGreaterThanOrEqual(5 * 60_000); + const gaps = readAt.slice(1).map((at, i) => at - readAt[i]!); + expect(gaps.every(gap => gap >= 5 * 60_000 && gap <= 60 * 60_000)).toBe(true); + expect(gaps).toEqual([...gaps].sort((a, b) => a - b)); + // Six hours of one-minute sweeps stay within the hourly cap plus the ramp. + expect(readAt.length).toBeGreaterThanOrEqual(6); + expect(readAt.length).toBeLessThanOrEqual(10); + // Reaching the predicted reset is not proof: a still-blocked read keeps the lock. await runMainAccountHardLockRecovery(config()); - expect(calls).toHaveLength(1); - now += 240_000; + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + now += 60 * 60_000; const recovery = fetchWith(async () => usage(0)); await runMainAccountHardLockRecovery(config()); expect(recovery).toEqual([whamUrl]); @@ -160,6 +182,38 @@ describe("main hard-lock background recovery", () => { } finally { clock.mockRestore(); } }); + test("an early reset redeemed elsewhere is learned from metadata before the predicted reset (#7312)", async () => { + let now = Date.now(); + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 5 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const calls = fetchWith(async () => usage(0)); + for (let tick = 0; tick < 5; tick++, now += 60_000) await runMainAccountHardLockRecovery(config()); + expect(calls).toEqual([]); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + await runMainAccountHardLockRecovery(config()); + expect(calls).toEqual([whamUrl]); + expect(getMainAccountHardLockStatus(config()).state).toBe("ready"); + } finally { clock.mockRestore(); } + }); + + test.each(["paused", "needs-reauth"])("a %s main receives no predicted-reset rechecks (#7312)", async kind => { + let now = Date.now(); + const clock = spyOn(Date, "now").mockImplementation(() => now); + const cfg = { ...config(), ...(kind === "paused" ? { pausedCodexAccountIds: [MAIN] } : {}) }; + try { + const deadline = now + 3 * 3_600_000; + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: deadline }, undefined, captureMainQuotaWriter(accountId)); + if (kind === "needs-reauth") markAccountNeedsReauth(MAIN); + const calls = fetchWith(async () => usage(0)); + for (; now < deadline; now += 60_000) await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([]); + expect(getMainAccountHardLockStatus(cfg).state).toBe("blocked"); + } finally { clock.mockRestore(); } + }); + test("manual reads and hard-lock recovery share Retry-After without claiming fresh evidence", async () => { let now = Date.now(); const clock = spyOn(Date, "now").mockImplementation(() => now); From 5804071010124ac801670cdfd93978db4c1240b3 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 20 Oct 2026 06:01:11 +0900 Subject: [PATCH 14/17] test(codex): await the async hard-lock fallback override in #7340 cases #7328 made applyMainHardLockFallbackOverride async for the Direct caller check; the #7340 revalidation cases were written against the synchronous form and asserted on a Promise once both landed. Co-authored-by: luvs01 <27862058+luvs01@users.noreply.github.com> --- ...account-hard-lock-fallback-routing.test.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts index 8f5cdc27151..621eb183a10 100644 --- a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts +++ b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts @@ -330,47 +330,47 @@ describe("applyMainHardLockFallbackOverride", async () => { expect(await applyMainHardLockFallbackOverride({ model: 42 }, cfg)).toBeNull(); }); - test("unavailable chosen targets keep the blocked source and reasoning intact", () => { + test("unavailable chosen targets keep the blocked source and reasoning intact", async () => { observe({ weeklyPercent: 100 }); const gateway = { adapter: "openai-responses", authMode: "key", baseUrl: "https://gateway.example.test/v1" } as const; for (const target of ["private/secret-model", "former/secret-model", "combo/removed", "policy/removed", "retired-account/gpt-5.3"]) { for (const providers of [{ gateway }, {}]) { const cfg = config({ providers, defaultProvider: "gateway", codexMainAccountHardLockFallback: { model: target, reasoningEffort: "low" } } as Partial); const body = { model: "gpt-5.3", reasoning: { effort: "high" } }; - expect(applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); expect(body).toEqual({ model: "gpt-5.3", reasoning: { effort: "high" } }); } } const cfg = config({ providers: { gateway, private: { ...gateway, disabled: true } }, defaultProvider: "gateway", codexMainAccountHardLockFallback: { model: "private/secret-model" } } as Partial); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); }); - test("target admission is rechecked after provider removal, re-addition and alias changes", () => { + test("target admission is rechecked after provider removal, re-addition and alias changes", async () => { observe({ weeklyPercent: 100 }); const provider = { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1", alias: "chosen" } as const; const cfg = config({ providers: { private: provider }, defaultProvider: "private", codexMainAccountHardLockFallback: { model: "chosen/secret-model" } } as Partial); const apply = () => applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg); - expect(apply()).not.toBeNull(); + expect(await apply()).not.toBeNull(); delete cfg.providers.private; - expect(apply()).toBeNull(); + expect(await apply()).toBeNull(); cfg.providers.private = provider; - expect(apply()).not.toBeNull(); + expect(await apply()).not.toBeNull(); cfg.providers.private = { ...provider, alias: "renamed" }; - expect(apply()).toBeNull(); + expect(await apply()).toBeNull(); cfg.codexMainAccountHardLockFallback = { model: "private/secret-model" }; - expect(apply()).not.toBeNull(); + expect(await apply()).not.toBeNull(); }); - test("bare fallback and explicitly declared slash model ids stay valid", () => { + test("bare fallback and explicitly declared slash model ids stay valid", async () => { observe({ weeklyPercent: 100 }); const provider = { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1", models: ["vendor/native-model"] } as const; for (const model of ["bare-model", "vendor/native-model", "private/vendor/native-model"]) { const cfg = config({ providers: { private: { ...provider, models: [...provider.models] } }, defaultProvider: "private", codexMainAccountHardLockFallback: { model } } as Partial); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)?.targetModel).toBe(model); + expect((await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg))?.targetModel).toBe(model); } }); - test("configured combo aliases, policy profiles and account namespaces keep their admission", () => { + test("configured combo aliases, policy profiles and account namespaces keep their admission", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ providers: { private: { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1" }, @@ -381,7 +381,7 @@ describe("applyMainHardLockFallbackOverride", async () => { } as Partial); for (const model of ["combo/rescue", "chosen-combo", "policy/rescue", "chosen-policy", "pinned/gpt-5.3"]) { cfg.codexMainAccountHardLockFallback = { model }; - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)?.targetModel).toBe(model); + expect((await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg))?.targetModel).toBe(model); } }); From 3f4479d658baa7bcc2dc0b88995a17ee31aba061 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 20 Oct 2026 06:04:43 +0900 Subject: [PATCH 15/17] fix(codex): recheck Direct hard-lock ownership synchronously before rewriting The async caller check now resolves to a synchronous ownership proof that is evaluated after the last await, so a main identity or bearer change during the policy-binding wait cannot reroute the original caller onto the fallback. Co-authored-by: luvs01 <27862058+luvs01@users.noreply.github.com> --- src/codex/auth-context.ts | 14 ++++++++ src/server/responses/core-auth.ts | 8 ++--- .../responses/main-hard-lock-routing.ts | 7 ++-- ...account-hard-lock-fallback-routing.test.ts | 36 ++++++++++++++++++- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index e27a2e0ed01..5c7ab655d69 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1095,6 +1095,20 @@ export async function mainAccountHardLockAppliesToCaller( return callerMatchesObservedMain(headers); } +/** + * Resolve ownership once, then hand back a synchronous proof. Main identity or its observed bearer + * can change across the await; the hard-lock reroute rechecks the proof with no await between it + * and the body rewrite, so a replacement main's lock never reroutes the original caller. + */ +export async function mainAccountHardLockCallerProof( + headers: Headers, + policy: CodexAuthPolicyConfig, + options: Parameters[2] = {}, +): Promise<(() => boolean) | null> { + if (!await mainAccountHardLockAppliesToCaller(headers, policy, options)) return null; + return options.substituteMainCredentialForDirect === true ? () => true : () => callerMatchesObservedMain(headers); +} + /** * Resolve the credential for one Codex turn. A main hard-lock reached through unpinned pool * selection is annotated with the earliest known pool recovery, so the refusal can name a retry diff --git a/src/server/responses/core-auth.ts b/src/server/responses/core-auth.ts index ea44d955a22..e9d49346a05 100644 --- a/src/server/responses/core-auth.ts +++ b/src/server/responses/core-auth.ts @@ -25,7 +25,7 @@ import { CodexAuthContextError, applyCodexAuthContextToProvider, stripCodexRuntimeProviderFields, - mainAccountHardLockAppliesToCaller, + mainAccountHardLockCallerProof, } from "../../codex/auth-context"; import { codexAccountSelectionForTurn, tryClaimNativeMainProfileForTurn } from "../lifecycle"; import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup"; @@ -596,15 +596,15 @@ export async function refreshNativeMainForwardAuth(args: { /** Inspect the original credential domain, before fallback changes the route or drops headers. */ export function mainHardLockFallbackCallerCheck( req: Request, config: OcxConfig, options: HandleResponsesOptions, -): (sourceModel: string) => Promise { +): (sourceModel: string) => Promise<(() => boolean) | null> { return async sourceModel => { let sourceRoute: ReturnType; try { sourceRoute = routeConcreteModel(config, sourceModel); } // This preview cannot replace the normal routing phase's existing error response. - catch { return false; } + catch { return null; } const headers = codexRouteCredentialDomainHeaders(req, sourceRoute, options, false, config); const { substituteMainCredential } = codexRouteCredentialOwnership(headers, config, sourceRoute, options); - return mainAccountHardLockAppliesToCaller(headers, options.codexAuthPolicy ?? config, { + return mainAccountHardLockCallerProof(headers, options.codexAuthPolicy ?? config, { substituteMainCredentialForDirect: substituteMainCredential, signal: options.abortSignal ?? req.signal, }); diff --git a/src/server/responses/main-hard-lock-routing.ts b/src/server/responses/main-hard-lock-routing.ts index e2a3586bd5f..09f129a01be 100644 --- a/src/server/responses/main-hard-lock-routing.ts +++ b/src/server/responses/main-hard-lock-routing.ts @@ -12,7 +12,8 @@ import { codexQuotaScopeForModel } from "../../codex/routing/health-store"; import { getEligiblePoolAccounts } from "../../codex/routing/selection"; import { OPENAI_CODEX_PROVIDER_ID, isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { providerCodexAccountMode } from "../../providers/registry"; -export type MainHardLockCallerCheck = (sourceModel: string) => Promise; +/** Resolves to a synchronous ownership proof, or null when the caller does not own the locked main. */ +export type MainHardLockCallerCheck = (sourceModel: string) => Promise<(() => boolean) | null>; function isConfiguredComboSelector(config: OcxConfig, modelId: string): boolean { const comboId = resolveComboId(config, modelId); @@ -96,7 +97,9 @@ export async function applyMainHardLockFallbackOverride( if (accountMode === "direct") { // Direct may forward a different caller's healthy credential. Only stored-main substitution // or an observed-main match owes this lock. Recheck usage after the startup binding wait. - if (!callerCheck || !await callerCheck(parsed.model)) return null; + const stillOwnsLockedMain = callerCheck ? await callerCheck(parsed.model) : null; + // Everything from here to the rewrite is synchronous, so this proof cannot go stale before it. + if (!stillOwnsLockedMain?.()) return null; if (getMainAccountHardLockStatus(config, now ?? Date.now()).state !== "blocked") return null; } else { if (getMainAccountHardLockStatus(config, now ?? Date.now()).state !== "blocked") return null; diff --git a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts index 621eb183a10..ddfa97d006d 100644 --- a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts +++ b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts @@ -3,6 +3,8 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { applyMainHardLockFallbackOverride } from "../../src/server/responses/main-hard-lock-routing"; +import { mainHardLockFallbackCallerCheck } from "../../src/server/responses/core-auth"; +import type { HandleResponsesOptions } from "../../src/server/responses/core-options"; import { resolveMainAccountHardLockFallback } from "../../src/codex/main-account-hard-lock"; import { captureMainQuotaWriter, clearMainAccountInfoCache, observeMainQuotaCredential, observeMainQuotaIdentity } from "../../src/codex/main-account-cache"; import { clearAccountQuota, setAccountQuotaFromParsed, type StoredAccountQuota } from "../../src/codex/quota"; @@ -279,10 +281,42 @@ describe("applyMainHardLockFallbackOverride", async () => { // Direct fails closed without the caller ownership proof supplied by request preparation. expect(await applyMainHardLockFallbackOverride(directBody, directCfg, now)).toBeNull(); expect(directBody.model).toBe("gpt-5.3"); - expect(await applyMainHardLockFallbackOverride(directBody, directCfg, now, async () => true)) + expect(await applyMainHardLockFallbackOverride(directBody, directCfg, now, async () => () => true)) .toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); }); ++ test.each(["none", "identity", "bearer"] as const)("Direct ownership is rechecked after the async caller check (change: %s)", async change => { + observeMainQuotaCredential("fixture-caller-a", "fallback-account-a"); + observe({ weeklyPercent: 100 }); + const cfg = config({ + providers: { + deepseek: { adapter: "openai-responses", authMode: "key", baseUrl: "https://deepseek.example.test/v1" }, + openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + codexMainAccountHardLockFallback: FALLBACK, + } as Partial); + const req = new Request("http://127.0.0.1/v1/responses", { method: "POST", + headers: { authorization: "Bearer fixture-caller-a", "chatgpt-account-id": "fallback-account-a" } }); + const body = { model: "gpt-5.3", reasoning: { effort: "high" } }; + const pending = applyMainHardLockFallbackOverride(body, cfg, undefined, + mainHardLockFallbackCallerCheck(req, cfg, {} as HandleResponsesOptions)); + // The replacement is blocked too, so only the ownership recheck can refuse the reroute. + queueMicrotask(() => { + if (change === "identity") { + observeMainQuotaIdentity("fallback-account-b"); + observeMainQuotaCredential("fixture-caller-b", "fallback-account-b"); + setAccountQuotaFromParsed("__main__", { weeklyPercent: 100 }, undefined, captureMainQuotaWriter("fallback-account-b")!); + } else if (change === "bearer") observeMainQuotaCredential("fixture-caller-a-rotated", "fallback-account-a"); + }); + const result = await pending; + if (change === "none") { + expect(result).toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); + return; + } + expect(result).toBeNull(); + expect(body).toEqual({ model: "gpt-5.3", reasoning: { effort: "high" } }); + }); + test("only bare Codex-native selectors qualify", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ From 7f35db5e419847b86a8c013acb9004d1771f16f2 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 20 Oct 2026 06:05:41 +0900 Subject: [PATCH 16/17] fix(codex): keep predicted-reset hold rechecks paced across transitions (#7312) Track the first hold per identity/credential separately from the backoff reservation so transient 401/403 retries are not re-armed, arm a hold that appears during token preparation instead of reading immediately, pull the due time in when the predicted reset moves earlier or passes, and keep hold reads at least one base interval apart. --- .../docs/reference/cli/providers-accounts.md | 6 +- src/codex/auth-api/pool-mode-gate.ts | 49 +++++++++++++---- structure/providers/openai-tiers.md | 10 ++-- .../main-account-hard-lock-recovery.test.ts | 55 +++++++++++++++++++ 4 files changed, 103 insertions(+), 17 deletions(-) diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 06d3e75eaf3..1ac81f11ec9 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -471,9 +471,9 @@ reset changes and expires after six hours or at the observed reset, whichever co An unreadable 5h reading cannot hide a weekly block. Unknown usage does not fabricate a zero, and a missing reading does not erase an already measured blocking tuple. A predicted reset time alone does not unlock it. While blocked, recovery checks owned usage metadata on a capped 5/10/20/40/60-minute schedule, so a reset -redeemed outside opencodex is noticed before the predicted reset; the first check waits five minutes, no -check is scheduled past the latest known blocking reset, and a paused or reauthentication-required main is -not checked. A longer `Retry-After` also delays profile and token preparation. Only fresh valid usage or authoritative window-absence evidence +redeemed outside opencodex is noticed before the predicted reset; the first check waits five minutes, checks +stay at least five minutes apart, and the next check is pulled in to the latest known blocking reset. A paused +or reauthentication-required main receives no checks before its predicted reset. A longer `Retry-After` also delays profile and token preparation. Only fresh valid usage or authoritative window-absence evidence can lift the block. In Pool mode, a quota `--refresh` bypasses cache freshness but still honors failed-read pacing; a deferred read makes no new diagnostic attempt. Other pause, reauthentication, and upstream limits remain independent. diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index 20dfc8f69d7..a4767a08d67 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -94,7 +94,12 @@ function poolCreditRenewalDue(config: OcxConfig, accountId: string, now = Date.n } let mainHardLockRecoveryInFlight: Promise | null = null; -let mainHardLockRecoveryAttempt: { identity: number; credential: number; after: number; delay: number } | undefined; +/** `floor` and `resetBound` apply only while a predicted reset is (or just was) holding the lock. */ +type MainRecoveryAttempt = { identity: number; credential: number; after: number; delay: number; floor?: number; resetBound?: number }; +let mainHardLockRecoveryAttempt: MainRecoveryAttempt | undefined; +/** The identity/credential that already waited out its first hold interval. */ +let mainHardLockHoldSeen: { identity: number; credential: number } | undefined; +const MAIN_HOLD_MIN_INTERVAL_MS = nextQuotaQueryDelay(); /** The predicted reset of a blocked main, or undefined when no future reset is known. */ function predictedMainResetHold(status: ReturnType): number | undefined { @@ -102,6 +107,25 @@ function predictedMainResetHold(status: ReturnType + mainHardLockHoldSeen?.identity === identity && mainHardLockHoldSeen.credential === credential; + /** Metadata-only recovery on the existing sweep; failures retain the observed policy block. */ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise { if (mainHardLockRecoveryInFlight) return mainHardLockRecoveryInFlight; @@ -119,6 +143,7 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise || (observedAt !== undefined && now - observedAt >= MAIN_CREDIT_RENEWAL_AGE_MS)); }; const status = getMainAccountHardLockStatus(config); + if (status.state !== "blocked") mainHardLockHoldSeen = undefined; if (status.state !== "blocked" && !creditRecoveryNeeded()) { mainHardLockRecoveryAttempt = undefined; return; } if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; // A predicted reset is a scheduling hint, never proof that the hard lock can be lifted. The @@ -133,15 +158,14 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise const identity = captureMainAccountIdentityGeneration(); const credential = getMainQuotaCredentialGeneration(); const previous = mainHardLockRecoveryAttempt; - if (holdResetAt !== undefined && (previous?.identity !== identity || previous.credential !== credential)) { - // The block was just observed: wait one base interval, and never past the predicted reset. - const delay = nextQuotaQueryDelay(); - mainHardLockRecoveryAttempt = { identity, credential, delay, after: Math.min(Date.now() + delay, holdResetAt) }; + if (holdResetAt !== undefined && !mainHoldSeenFor(identity, credential)) { + armMainResetHold(identity, credential, holdResetAt); return; } + const sameAttempt = previous?.identity === identity && previous.credential === credential; + if (sameAttempt && holdResetAt !== undefined) previous.resetBound = holdResetAt; const queryAfter = nextCodexUsageQueryAt(`main:${captureConfigGeneration()}:${credential}`) ?? 0; - if ((previous?.identity === identity && previous.credential === credential - && previous.after > Date.now()) || queryAfter > Date.now()) return; + if ((sameAttempt && mainRecoveryDueAt(previous, holdResetAt) > Date.now()) || queryAfter > Date.now()) return; const lease = tryAcquireNativeMainProfileClaim(); if (!lease) return; mainHardLockRecoveryInFlight = (async () => { @@ -166,9 +190,14 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise || (passive && !creditRecoveryNeeded()) || (currentHold !== undefined && isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID))) return; const credential = getMainQuotaCredentialGeneration(); + // A hold that appeared during preparation is first seen now and waits its base interval. + if (currentHold !== undefined && !mainHoldSeenFor(identityGeneration, credential)) { + armMainResetHold(identityGeneration, credential, currentHold); + return; + } // Reconcile the physical bearer before consulting the previous credential's delay. if (previous?.identity === identityGeneration && previous.credential === credential - && previous.after > Date.now()) return; + && mainRecoveryDueAt(previous, currentHold) > Date.now()) return; const currentQueryAfter = nextCodexUsageQueryAt(`main:${captureConfigGeneration()}:${credential}`) ?? 0; if (currentQueryAfter > Date.now()) { mainHardLockRecoveryAttempt = { identity: identityGeneration, credential, @@ -193,8 +222,8 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise const delay = nextQuotaQueryDelay(previousDelay || undefined); const hold = predictedMainResetHold(getMainAccountHardLockStatus(config)); mainHardLockRecoveryAttempt = getMainAccountHardLockStatus(config).state === "blocked" || creditRecoveryNeeded() - ? { identity: identityGeneration, credential, delay, - after: Math.max(Math.min(Date.now() + delay, hold ?? Infinity), queryAfter) } + ? { identity: identityGeneration, credential, delay, after: Math.max(Date.now() + delay, queryAfter), + ...(hold !== undefined ? { floor: Date.now() + MAIN_HOLD_MIN_INTERVAL_MS, resetBound: hold } : {}) } : undefined; } } else if (queryAfter > Date.now()) { diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 6ceae204e81..caffc0dd5bf 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -379,10 +379,12 @@ explicit main selection and stored Direct substitution do not override it. It ne account nor clears upstream cooldown/reauth state, and management quota refresh remains available. Fresh valid usage below the effective threshold, including 0%, or validated WHAM absence retires a measured short block; passing a reset timestamp alone does not. A future known reset is a scheduling hint (#7312): recovery -arms one base delay when the hold is first seen, then reads WHAM metadata on the same capped -5/10/20/40/60-minute delay calculation as usage-query failures, never scheduling a read past that -reset, so an early reset redeemed elsewhere is learned without inference or reset-credit use. A paused -or needs-reauth main gets no hold rechecks. Skipped ticks do not extend the delay; +arms one base delay the first time an identity/credential sees the hold (including a hold that appears +during token preparation), then reads WHAM metadata on the same capped 5/10/20/40/60-minute delay +calculation as usage-query failures. The due time is pulled in to the latest known reset, including one +that moved earlier or just passed, but hold reads stay at least one base interval apart; a nonterminal +401/403 retry does not re-arm the first-hold delay. An early reset redeemed elsewhere is thus learned +without inference or reset-credit use. A paused or needs-reauth main gets no hold rechecks. Skipped ticks do not extend the delay; the physical bearer is reconciled before checking the delay, and late results cannot charge a replacement credential. A longer valid Retry-After from any main usage reader is checked for the current credential before the recovery worker takes a profile lease or prepares a token; a replacement diff --git a/tests/codex-integration/main-account-hard-lock-recovery.test.ts b/tests/codex-integration/main-account-hard-lock-recovery.test.ts index 5cd5c61ddfa..cab9f6bc40c 100644 --- a/tests/codex-integration/main-account-hard-lock-recovery.test.ts +++ b/tests/codex-integration/main-account-hard-lock-recovery.test.ts @@ -214,6 +214,61 @@ describe("main hard-lock background recovery", () => { } finally { clock.mockRestore(); } }); ++ test("a predicted reset that moves earlier pulls the next hold read in, never under one base interval (#7312)", async () => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 6 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push((now - start) / 60_000); return heldUsage(start + 6 * 3_600_000); }); + for (; now - start <= 6 * 60_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5]); + // Same-identity evidence now predicts the reset at minute 8; the old reservation was minute 15. + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: start + 8 * 60_000 }, undefined, captureMainQuotaWriter(accountId)); + const recovered = fetchWith(async () => { readAt.push((now - start) / 60_000); return usage(0); }); + for (; now - start <= 12 * 60_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5, 10]); + expect(recovered).toEqual([whamUrl]); + expect(getMainAccountHardLockStatus(config()).state).toBe("ready"); + } finally { clock.mockRestore(); } + }); + + test("a moving near-term reset keeps hold reads at least one base interval apart (#7312)", async () => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 2 * 60_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push(now - start); return heldUsage(now + 2 * 60_000); }); + for (; now - start < 3_600_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + const gaps = readAt.slice(1).map((at, i) => at - readAt[i]!); + expect(readAt.length).toBeGreaterThan(1); + expect(gaps.every(gap => gap >= 5 * 60_000)).toBe(true); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + } finally { clock.mockRestore(); } + }); + + test.each([401, 403])("a nonterminal HTTP %s during a predicted-reset hold retries on the next sweep (#7312)", async status => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 6 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push((now - start) / 60_000); return Response.json({}, { status }); }); + for (; now - start <= 6 * 60_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5, 6]); + expect(isAccountNeedsReauth(MAIN)).toBe(false); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + } finally { clock.mockRestore(); } + }); + test("manual reads and hard-lock recovery share Retry-After without claiming fresh evidence", async () => { let now = Date.now(); const clock = spyOn(Date, "now").mockImplementation(() => now); From 6a3657af014c25dcd31894121495099fd80e97bd Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 20 Oct 2026 06:10:17 +0900 Subject: [PATCH 17/17] fix(codex): honor a revised reset that elapses between hold sweeps (#7312) Recovery scheduling now reads the latest blocking reset from current evidence even after it has passed, so a reset revised earlier and expired before the next sweep releases the old reservation. It remains scheduling evidence only; admission still requires verified metadata. --- src/codex/auth-api/pool-mode-gate.ts | 12 +++++---- src/codex/main-account-hard-lock.ts | 14 ++++++++++ .../main-account-hard-lock-recovery.test.ts | 27 +++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index a4767a08d67..e024473cdb9 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -13,7 +13,7 @@ import { isAccountNeedsReauth } from "../account-runtime-state"; import { getValidMainAccountToken, MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan } from "../main-account"; import { captureConfigGeneration, registerStateSweepAfterTick } from "../../lib/state-store-sweeper"; import { observeMainQuotaCredential, getMainQuotaCredentialGeneration, captureMainAccountIdentityGeneration, isMainAccountIdentityGenerationLive } from "../main-account-cache"; -import { getMainAccountHardLockStatus } from "../main-account-hard-lock"; +import { getMainAccountHardLockStatus, latestMainBlockingResetAt } from "../main-account-hard-lock"; import type { OcxConfig } from "../../types"; import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; import { providerCodexAccountMode } from "../../providers/registry"; @@ -111,8 +111,10 @@ function predictedMainResetHold(status: ReturnType Date.now()) || queryAfter > Date.now()) return; + if ((sameAttempt && mainRecoveryDueAt(previous, holdResetAt, config) > Date.now()) || queryAfter > Date.now()) return; const lease = tryAcquireNativeMainProfileClaim(); if (!lease) return; mainHardLockRecoveryInFlight = (async () => { @@ -197,7 +199,7 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise } // Reconcile the physical bearer before consulting the previous credential's delay. if (previous?.identity === identityGeneration && previous.credential === credential - && mainRecoveryDueAt(previous, currentHold) > Date.now()) return; + && mainRecoveryDueAt(previous, currentHold, config) > Date.now()) return; const currentQueryAfter = nextCodexUsageQueryAt(`main:${captureConfigGeneration()}:${credential}`) ?? 0; if (currentQueryAfter > Date.now()) { mainHardLockRecoveryAttempt = { identity: identityGeneration, credential, diff --git a/src/codex/main-account-hard-lock.ts b/src/codex/main-account-hard-lock.ts index 5c9a273ccec..c7cee364f36 100644 --- a/src/codex/main-account-hard-lock.ts +++ b/src/codex/main-account-hard-lock.ts @@ -121,3 +121,17 @@ export function getMainAccountHardLockStatus( export function isMainAccountHardLocked(config: PolicyConfig, now = Date.now()): boolean { return getMainAccountHardLockStatus(config, now).state === "blocked"; } + +/** + * Scheduling evidence only: the latest blocking reset even after it has passed, so recovery can + * notice a revised reset that expired between sweeps. Never an admission or unlock signal. + */ +export function latestMainBlockingResetAt(config: PolicyConfig): number | undefined { + if (!isMainAccountHardLockEnabled(config)) return undefined; + const quota = getMainPolicyQuota(); + if (!quota) return undefined; + const thresholds = resolveMainAccountHardLockThresholds(config); + const resets = governingWindows(quota) + .filter(w => validPercent(w.percent) && w.percent >= thresholds[w.kind]).map(w => resetTimestamp(w.resetAt)); + return resets.length > 0 && resets.every(r => r !== undefined) ? Math.max(...(resets as number[])) : undefined; +} diff --git a/tests/codex-integration/main-account-hard-lock-recovery.test.ts b/tests/codex-integration/main-account-hard-lock-recovery.test.ts index cab9f6bc40c..9d22e29efd5 100644 --- a/tests/codex-integration/main-account-hard-lock-recovery.test.ts +++ b/tests/codex-integration/main-account-hard-lock-recovery.test.ts @@ -236,6 +236,33 @@ describe("main hard-lock background recovery", () => { } finally { clock.mockRestore(); } }); ++ test("a revised reset that expires between sweeps still releases the old reservation (#7312)", async () => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 6 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push((now - start) / 60_000); return heldUsage(start + 6 * 3_600_000); }); + await runMainAccountHardLockRecovery(config()); + now = start + 5 * 60_000; + await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5]); + // Revised to minute 8 at minute 7, with no sweep until after that boundary has passed. + now = start + 7 * 60_000; + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: start + 8 * 60_000 }, undefined, captureMainQuotaWriter(accountId)); + now = start + 9 * 60_000; + await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5]); + now = start + 10 * 60_000; + await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5, 10]); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + } finally { clock.mockRestore(); } + }); + test("a moving near-term reset keeps hold reads at least one base interval apart (#7312)", async () => { let now = Date.now(); const start = now;