diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 946d51fd489..1ac81f11ec9 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -470,9 +470,10 @@ reset changes and expires after six hours or at the observed reset, whichever co An unreadable 5h reading cannot hide a weekly block. Unknown usage does not fabricate a zero, and a missing reading does not erase an already measured blocking tuple. A predicted reset time alone does not unlock it. -While blocked, the minute sweep waits for the latest known blocking reset, then checks owned usage. -If no future reset is known or a check remains blocked, recovery uses a capped 5/10/20/40/60-minute -schedule; a longer `Retry-After` also delays profile and token preparation. Only fresh valid usage or authoritative window-absence evidence +While blocked, recovery checks owned usage metadata on a capped 5/10/20/40/60-minute schedule, so a reset +redeemed outside opencodex is noticed before the predicted reset; the first check waits five minutes, checks +stay at least five minutes apart, and the next check is pulled in to the latest known blocking reset. A paused +or reauthentication-required main receives no checks before its predicted reset. A longer `Retry-After` also delays profile and token preparation. Only fresh valid usage or authoritative window-absence evidence can lift the block. In Pool mode, a quota `--refresh` bypasses cache freshness but still honors failed-read pacing; a deferred read makes no new diagnostic attempt. Other pause, reauthentication, and upstream limits remain independent. diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 62bb5f93c0f..7065600ad25 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -106,6 +106,9 @@ separate. Full request URLs such as `/api/v1/responses` are not provider base UR | `cacheRetention?` | `"none" \| "short" \| "long"` | first-party Anthropic: `"long"`; other Anthropic-compatible endpoints: `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. When unset, requests to `api.anthropic.com` (API key or the OAuth account pools) use the 1-hour TTL, because agentic turns often pause longer than five minutes and a 1-hour write (2x base input instead of 1.25x) pays for itself on the first reuse; Anthropic-compatible gateways and vendor endpoints keep the 5-minute TTL because their 1-hour support is undocumented. On native `/v1/messages` passthrough, caller breakpoints without a `ttl` get `"1h"` unless the request pins any breakpoint to `"5m"`, in which case it is forwarded unchanged. On a translated `/v1/messages` route, caller breakpoints that all ask for `ttl: "1h"` mean `"long"` and any explicit `"5m"` means `"short"`. An explicit value always wins. Changing the TTL makes each open conversation write its cache once more, since 5-minute and 1-hour entries are separate. Cost estimates price writes reported as 1-hour at the 1-hour rate. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | +Opted-in pool accounts with a full included usage window refresh their credit evidence through the existing background sweep from three minutes of age. Ordinary response headers do not refresh credit balances. A failed metadata read never extends the five-minute spending-evidence lifetime. + + Across these strategies, each account's effective threshold is its `codexAccountAutoSwitchThresholds` entry when present, otherwise global `autoSwitchThreshold`. A zero override disables only proactive usage switching; startup binding, hard-lock, cooldown, entitlement checks, and failure recovery remain enforced. Setting an account's switching threshold to `0` does not disable main-account hard-lock, startup diff --git a/src/codex/account-lifecycle.ts b/src/codex/account-lifecycle.ts index 65e35aa5f10..ffb7a5be540 100644 --- a/src/codex/account-lifecycle.ts +++ b/src/codex/account-lifecycle.ts @@ -14,6 +14,7 @@ import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account"; import { clearAccountQuota } from "./quota"; import { forgetManualPreference } from "./routing/active-account"; import { deleteAllHealthForAccount } from "./routing/health-store"; +import { forgetCodexFailureWindow } from "./routing/failure-window"; import { clearThreadAccountMapForAccount } from "./routing/thread-affinity"; import { clearCodexPoolRefreshFailure } from "./pool-refresh-backoff"; @@ -49,11 +50,15 @@ export function purgeCodexAccountRuntimeState(accountId: string): void { clearAccountNeedsReauth(accountId); clearAccountQuota(accountId); clearThreadAccountMapForAccount(accountId); - // Deliberately not via `./routing`: the account-delete path only needs the health rows and - // the manual preference gone, and importing the router here closed a cycle through - // routing/health-store and routing/active-account. + // Deliberately not via `./routing`: the account-delete path only needs the health rows + // and their sibling routing state gone, and importing the router barrel here closed a + // cycle through routing/health-store and routing/active-account. The leaf imports below + // carry every per-account forget that barrel helper used to perform — including the + // failure window, whose degrade and pin-warning state must not leak into a replacement + // account or a new physical login behind __main__. deleteAllHealthForAccount(accountId); forgetManualPreference(accountId); + forgetCodexFailureWindow(accountId); clearCodexPoolRefreshFailure(accountId); if (accountId === MAIN_CODEX_ACCOUNT_ID) { clearMainAccountInfoCache(); diff --git a/src/codex/account-store.ts b/src/codex/account-store.ts index bfe677c2d8e..b384ab5588b 100644 --- a/src/codex/account-store.ts +++ b/src/codex/account-store.ts @@ -136,25 +136,37 @@ export function peekCodexAccountRecordStore(): Record boolean) + & { readonly snapshotStatus: "ok" | "failed" }; + /** * The same verdict as {@link isCodexAccountGenerationLive}, over ONE store load. * @@ -409,12 +425,15 @@ export function isCodexAccountGenerationLive(id: string, generation: number): bo * several accounts in one synchronous pass on the request path, so it opens one checker and * closes over the snapshot instead. */ -export function beginCodexAccountGenerationLiveCheck(): (id: string, generation: number) => boolean { - const snapshot = loadCodexAccountRecordSnapshot(); - return (id, generation) => { +export function beginCodexAccountGenerationLiveCheck(): CodexAccountGenerationLiveCheck { + let result: CodexAccountRecordStoreResult; + try { result = loadCodexAccountRecordStoreResult(); } + catch { result = { status: "failed" }; } + const snapshot = result.status === "ok" ? result.records : {}; + return Object.assign((id: string, generation: number) => { const record = snapshot[id]; return !!record?.credential && record.deletedAt == null && record.generation === generation; - }; + }, { snapshotStatus: result.status }); } export function saveCodexAccountCredentialIfGeneration( diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index 92b02e0ed2f..e024473cdb9 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -13,7 +13,7 @@ import { isAccountNeedsReauth } from "../account-runtime-state"; import { getValidMainAccountToken, MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan } from "../main-account"; import { captureConfigGeneration, registerStateSweepAfterTick } from "../../lib/state-store-sweeper"; import { observeMainQuotaCredential, getMainQuotaCredentialGeneration, captureMainAccountIdentityGeneration, isMainAccountIdentityGenerationLive } from "../main-account-cache"; -import { getMainAccountHardLockStatus } from "../main-account-hard-lock"; +import { getMainAccountHardLockStatus, latestMainBlockingResetAt } from "../main-account-hard-lock"; import type { OcxConfig } from "../../types"; import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; import { providerCodexAccountMode } from "../../providers/registry"; @@ -78,8 +78,55 @@ export async function runCodexCooldownRecoveryProbes(config: OcxConfig, now = Da const MAIN_CREDIT_RENEWAL_AGE_MS = 3 * 60_000; +/** Traffic headers renew usage, never the independent credit observation (#7377). */ +function poolCreditRenewalDue(config: OcxConfig, accountId: string, now = Date.now()): boolean { + const account = configuredPoolAccount(config, accountId); + if (!account || !codexAccountUsesCreditsAfterLimit(config, accountId) + || isCodexAccountPaused(config, accountId) || isAccountNeedsReauth(accountId)) return false; + const quota = getAccountQuota(accountId); + if (codexUsageLimitResetAt(quota, account.plan, now) === undefined) return false; + const observedAt = quota?.credits?.observedAt; + if (observedAt !== undefined && Number.isFinite(observedAt) + && now - observedAt >= 0 && now - observedAt < MAIN_CREDIT_RENEWAL_AGE_MS) return false; + const attempt = poolQuotaPrimeAttemptedAt.get(accountId); + return !attempt || attempt.generation !== readCodexAccountRecord(accountId)?.generation + || now - attempt.at >= MAIN_CREDIT_RENEWAL_AGE_MS; +} + let mainHardLockRecoveryInFlight: Promise | null = null; -let mainHardLockRecoveryAttempt: { identity: number; credential: number; after: number; delay: number } | undefined; +/** `floor` and `resetBound` apply only while a predicted reset is (or just was) holding the lock. */ +type MainRecoveryAttempt = { identity: number; credential: number; after: number; delay: number; floor?: number; resetBound?: number }; +let mainHardLockRecoveryAttempt: MainRecoveryAttempt | undefined; +/** The identity/credential that already waited out its first hold interval. */ +let mainHardLockHoldSeen: { identity: number; credential: number } | undefined; +const MAIN_HOLD_MIN_INTERVAL_MS = nextQuotaQueryDelay(); + +/** The predicted reset of a blocked main, or undefined when no future reset is known. */ +function predictedMainResetHold(status: ReturnType): number | undefined { + return status.state === "blocked" && status.resetAt !== undefined && status.resetAt > Date.now() + ? status.resetAt : undefined; +} + +/** + * A hold read is due at its backoff deadline, pulled in to the latest known predicted reset (even + * one that moved earlier or just passed) but never sooner than one base interval after the last read. + */ +function mainRecoveryDueAt(attempt: MainRecoveryAttempt, holdResetAt: number | undefined, config: OcxConfig): number { + // Once a hold elapses the status omits its reset; current evidence may still name a revised one. + const bound = holdResetAt ?? (attempt.resetBound === undefined ? undefined + : Math.min(attempt.resetBound, latestMainBlockingResetAt(config) ?? Infinity)); + return bound === undefined ? attempt.after : Math.max(attempt.floor ?? 0, Math.min(attempt.after, bound)); +} + +/** First sight of a hold for this credential waits one base interval; transient retries do not re-arm it. */ +function armMainResetHold(identity: number, credential: number, holdResetAt: number): void { + const at = Date.now() + MAIN_HOLD_MIN_INTERVAL_MS; + mainHardLockHoldSeen = { identity, credential }; + mainHardLockRecoveryAttempt = { identity, credential, delay: MAIN_HOLD_MIN_INTERVAL_MS, after: at, floor: at, resetBound: holdResetAt }; +} + +const mainHoldSeenFor = (identity: number, credential: number): boolean => + mainHardLockHoldSeen?.identity === identity && mainHardLockHoldSeen.credential === credential; /** Metadata-only recovery on the existing sweep; failures retain the observed policy block. */ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise { @@ -98,10 +145,14 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise || (observedAt !== undefined && now - observedAt >= MAIN_CREDIT_RENEWAL_AGE_MS)); }; const status = getMainAccountHardLockStatus(config); + if (status.state !== "blocked") mainHardLockHoldSeen = undefined; if (status.state !== "blocked" && !creditRecoveryNeeded()) { mainHardLockRecoveryAttempt = undefined; return; } if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)) return; - // A predicted reset is a scheduling hint, never proof that the hard lock can be lifted. - if (status.state === "blocked" && status.resetAt !== undefined && status.resetAt > Date.now()) return; + // A predicted reset is a scheduling hint, never proof that the hard lock can be lifted. The + // hold is rechecked on the capped WHAM schedule so an early reset redeemed elsewhere is + // learned from metadata (#7312); a paused main receives no hold rechecks. + const holdResetAt = predictedMainResetHold(status); + if (holdResetAt !== undefined && isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID)) return; // Reconcile the owned physical credential before skipping work for an older one. reconcileMainCodexAccountRuntimeState(); const physical = readCodexTokens(); @@ -109,9 +160,14 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise const identity = captureMainAccountIdentityGeneration(); const credential = getMainQuotaCredentialGeneration(); const previous = mainHardLockRecoveryAttempt; + if (holdResetAt !== undefined && !mainHoldSeenFor(identity, credential)) { + armMainResetHold(identity, credential, holdResetAt); + return; + } + const sameAttempt = previous?.identity === identity && previous.credential === credential; + if (sameAttempt && holdResetAt !== undefined) previous.resetBound = holdResetAt; const queryAfter = nextCodexUsageQueryAt(`main:${captureConfigGeneration()}:${credential}`) ?? 0; - if ((previous?.identity === identity && previous.credential === credential - && previous.after > Date.now()) || queryAfter > Date.now()) return; + if ((sameAttempt && mainRecoveryDueAt(previous, holdResetAt, config) > Date.now()) || queryAfter > Date.now()) return; const lease = tryAcquireNativeMainProfileClaim(); if (!lease) return; mainHardLockRecoveryInFlight = (async () => { @@ -130,16 +186,20 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise // Native refresh retains its own grant-scoped refusal; global quarantine would outlive it. return; } - const currentStatus = getMainAccountHardLockStatus(config); + const currentHold = predictedMainResetHold(getMainAccountHardLockStatus(config)); if (isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID) || !isMainAccountIdentityGenerationLive(identityGeneration) || (passive && !creditRecoveryNeeded()) - || (currentStatus.state === "blocked" && currentStatus.resetAt !== undefined - && currentStatus.resetAt > Date.now())) return; + || (currentHold !== undefined && isCodexAccountPaused(config, MAIN_CODEX_ACCOUNT_ID))) return; const credential = getMainQuotaCredentialGeneration(); + // A hold that appeared during preparation is first seen now and waits its base interval. + if (currentHold !== undefined && !mainHoldSeenFor(identityGeneration, credential)) { + armMainResetHold(identityGeneration, credential, currentHold); + return; + } // Reconcile the physical bearer before consulting the previous credential's delay. if (previous?.identity === identityGeneration && previous.credential === credential - && previous.after > Date.now()) return; + && mainRecoveryDueAt(previous, currentHold, config) > Date.now()) return; const currentQueryAfter = nextCodexUsageQueryAt(`main:${captureConfigGeneration()}:${credential}`) ?? 0; if (currentQueryAfter > Date.now()) { mainHardLockRecoveryAttempt = { identity: identityGeneration, credential, @@ -162,8 +222,10 @@ export async function runMainAccountHardLockRecovery(config: OcxConfig): Promise if (transientAuth) mainHardLockRecoveryAttempt = undefined; else { const delay = nextQuotaQueryDelay(previousDelay || undefined); + const hold = predictedMainResetHold(getMainAccountHardLockStatus(config)); mainHardLockRecoveryAttempt = getMainAccountHardLockStatus(config).state === "blocked" || creditRecoveryNeeded() - ? { identity: identityGeneration, credential, delay, after: Math.max(Date.now() + delay, queryAfter) } + ? { identity: identityGeneration, credential, delay, after: Math.max(Date.now() + delay, queryAfter), + ...(hold !== undefined ? { floor: Date.now() + MAIN_HOLD_MIN_INTERVAL_MS, resetBound: hold } : {}) } : undefined; } } else if (queryAfter > Date.now()) { @@ -186,6 +248,7 @@ export function registerCodexCooldownRecoveryProbeWorker(config: OcxConfig): voi afterTick: () => { void runCodexCooldownRecoveryProbes(config); void runMainAccountHardLockRecovery(config); + void primeCodexPoolQuotas(config, "credit-renewal"); }, }); } @@ -265,6 +328,7 @@ export async function primeCodexPoolQuotas( primeInFlight = trackBackgroundWork(`codex-pool-quota-prime:${reason}`, (async () => { const pool = (runtimeConfig.codexAccounts ?? []).filter(isSelectableCodexPoolAccount); const stale = pool.filter(a => { + if (reason === "credit-renewal") return poolCreditRenewalDue(runtimeConfig, a.id); const q = getAccountQuota(a.id); const observationStale = reason === "priority-failback" && q && !codexQuotaHasFreshUsage(q, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS); @@ -278,7 +342,10 @@ export async function primeCodexPoolQuotas( if (lastAttempt.generation !== readCodexAccountRecord(a.id)?.generation) return true; return Date.now() - lastAttempt.at >= POOL_CACHE_TTL; }); + if (reason === "credit-renewal" && stale.length === 0) return; const primeMain = async () => { + // Main renewal owns its physical-identity claims and passive probe separately. + if (reason === "credit-renewal") return; const mainLease = tryAcquireNativeMainPrimeLease(); if (!mainLease) return; try { @@ -311,11 +378,12 @@ export async function primeCodexPoolQuotas( primeMain(), mapWithConcurrency(stale, POOL_QUOTA_REFRESH_CONCURRENCY, async a => { if (!getCodexAccountCredential(a.id)) return; + if (reason === "credit-renewal" && !poolCreditRenewalDue(getRuntimeConfig(config), a.id)) return; let result: PoolQuotaResult; try { const quota = getAccountQuota(a.id); - const bypassCachedQuota = reason === "priority-failback" && quota !== null - && !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS); + const bypassCachedQuota = reason === "credit-renewal" || (reason === "priority-failback" && quota !== null + && !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS)); result = await fetchPoolAccountQuota(a.id, bypassCachedQuota, a.plan, getValidPoolTokenForPrime); } catch (error) { // Local quota-flight saturation proves no WHAM request existed for this account. diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index c4cd30b325d..5c7ab655d69 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -324,6 +324,8 @@ export type CodexAuthContext = writerGeneration: number; /** Captured before async credential work; never reconstructed after the upstream response. */ mainQuotaWriter?: MainQuotaWriter; + /** Immutable credential proof captured while materializing this physical attempt's headers. */ + mainQuotaDispatch?: MainQuotaDispatch; reserveAuthorization?: MainReserveAuthorization; accessToken: string; chatgptAccountId: string; @@ -1080,6 +1082,33 @@ async function awaitMainAccountPolicyBindingSettled( if (!settled) throw new CodexMainProfileDrainingError(); } +/** Memory-only ownership check shared by Direct admission and pre-auth hard-lock routing. */ +export async function mainAccountHardLockAppliesToCaller( + headers: Headers, + policy: CodexAuthPolicyConfig, + options: Pick = {}, +): Promise { + if (options.substituteMainCredentialForDirect === true) return true; + if (!hasCallerCodexBearer(headers)) return false; + if (isMainAccountHardLockEnabled(policy)) await awaitMainAccountPolicyBindingSettled(options); + return callerMatchesObservedMain(headers); +} + +/** + * Resolve ownership once, then hand back a synchronous proof. Main identity or its observed bearer + * can change across the await; the hard-lock reroute rechecks the proof with no await between it + * and the body rewrite, so a replacement main's lock never reroutes the original caller. + */ +export async function mainAccountHardLockCallerProof( + headers: Headers, + policy: CodexAuthPolicyConfig, + options: Parameters[2] = {}, +): Promise<(() => boolean) | null> { + if (!await mainAccountHardLockAppliesToCaller(headers, policy, options)) return null; + return options.substituteMainCredentialForDirect === true ? () => true : () => callerMatchesObservedMain(headers); +} + /** * Resolve the credential for one Codex turn. A main hard-lock reached through unpinned pool * selection is annotated with the earliest known pool recovery, so the refusal can name a retry @@ -1172,8 +1201,7 @@ async function resolveCodexAuthContextUnannotated( // Trusted substitution still has to claim and validate stored main below. if (!substituteStoredMain && !hasCallerCodexBearer(headers)) throw new CodexDirectAuthenticationError(); if (!substituteStoredMain) { - if (isMainAccountHardLockEnabled(policy)) await awaitMainAccountPolicyBindingSettled(options); - if (callerMatchesObservedMain(headers)) assertMainAccountPolicy(policy); + if (await mainAccountHardLockAppliesToCaller(headers, policy, options)) assertMainAccountPolicy(policy); assertCallerOwnedMainPoolNotCooled(); if (reserve) { const selected = materializeCodexUpstreamAuth(headers, { kind: "main", accountId: null }, { config: policy }); @@ -1432,6 +1460,7 @@ async function resolveCodexAuthContextUnannotated( if ( requestScopedMainCredential && fixedAccountId === undefined + && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID ) { return await resolveCallerOwnedMainContext(); @@ -1590,6 +1619,7 @@ async function resolveCodexAuthContextUnannotated( // alternate is eligible. A validated caller may still serve this request, // just as it can after an upstream rejection, without changing Pool state. if (requestScopedMainCredential && fixedAccountId === undefined + && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID && !callerIsCooledPoolAccount(headers, config, accountId)) { return await resolveCallerOwnedMainContext(); @@ -1766,6 +1796,8 @@ export function materializeCodexUpstreamAuth( if (ctx.kind === "main-pool") { ctx.mainQuotaWriter = observeSelectedMainCredential(ctx, ctx.mainQuotaWriter); assertMainAccountPolicy(options.config); + ctx.mainQuotaDispatch = ctx.mainQuotaWriter && isMainQuotaWriterLive(ctx.mainQuotaWriter) + ? selectedMainQuotaDispatch(selected) : undefined; } assertMaterializedReserve(selected, ctx, options); return selected; diff --git a/src/codex/main-account-hard-lock.ts b/src/codex/main-account-hard-lock.ts index 5c9a273ccec..c7cee364f36 100644 --- a/src/codex/main-account-hard-lock.ts +++ b/src/codex/main-account-hard-lock.ts @@ -121,3 +121,17 @@ export function getMainAccountHardLockStatus( export function isMainAccountHardLocked(config: PolicyConfig, now = Date.now()): boolean { return getMainAccountHardLockStatus(config, now).state === "blocked"; } + +/** + * Scheduling evidence only: the latest blocking reset even after it has passed, so recovery can + * notice a revised reset that expired between sweeps. Never an admission or unlock signal. + */ +export function latestMainBlockingResetAt(config: PolicyConfig): number | undefined { + if (!isMainAccountHardLockEnabled(config)) return undefined; + const quota = getMainPolicyQuota(); + if (!quota) return undefined; + const thresholds = resolveMainAccountHardLockThresholds(config); + const resets = governingWindows(quota) + .filter(w => validPercent(w.percent) && w.percent >= thresholds[w.kind]).map(w => resetTimestamp(w.resetAt)); + return resets.length > 0 && resets.every(r => r !== undefined) ? Math.max(...(resets as number[])) : undefined; +} diff --git a/src/codex/routing.ts b/src/codex/routing.ts index 7eefb87e4e2..5008d7ac569 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -23,7 +23,7 @@ import { failureWindowSteersNewThreads, forgetCodexFailureWindow, forgetCodexFailureWindows, - noteCodexFailureWindowSample, + noteCodexFailureWindowOutcome, settleCodexSuccessStreak, } from "./routing/failure-window"; @@ -220,6 +220,7 @@ export { isEffectiveCodexAccountPinned, } from "./routing/active-account"; export { codexAccountPinDrainReason } from "./routing/pin-drain"; +export { codexFailureWindowOutcomeMeta } from "./routing/failure-window"; export type { CodexPinDrainReason } from "./routing/pin-drain"; // A shared refresh can outlive the request that opened it. Register the affinity @@ -1305,7 +1306,7 @@ export function recordCodexUpstreamOutcome( */ dropSpentCredentialFailure(accountId); if (outcomeClass === "success") { - noteCodexFailureWindowSample(config, accountId, false, now); + noteCodexFailureWindowOutcome(config, accountId, false, now, meta); // The operator's one-shot is spent by a dispatch that actually worked, and only by that. // A failed lookup leaves it unspent so the intent survives the failure. consumeManualPreference(accountId, codexPoolKeyForScope(quotaScope)); @@ -1555,7 +1556,7 @@ export function recordCodexUpstreamOutcome( // A transient failure concludes an owning probe; an unrelated 5xx must not // consume someone else's live lease or drop hard-cooldown bookkeeping (#433). const transientBase = ownsProbeLease(current, meta) ? withProbeLeaseReleased(current!, now) : current; - noteCodexFailureWindowSample(config, accountId, true, now); + noteCodexFailureWindowOutcome(config, accountId, true, now, meta); const stale = current?.lastFailureAt ? now - current.lastFailureAt > CODEX_FAILURE_WINDOW_MS : false; const hardCooldownUntil = getCodexAccountCooldownUntil(accountId, now) ?? undefined; // Soft avoid + affinity clears are part of failover. When threshold is 0, leave diff --git a/src/codex/routing/cooldown-math.ts b/src/codex/routing/cooldown-math.ts index 3e03e029aae..515b4d9eb22 100644 --- a/src/codex/routing/cooldown-math.ts +++ b/src/codex/routing/cooldown-math.ts @@ -6,6 +6,7 @@ import { isThirtyDayOnlyCodexPlan } from "../plan"; import { hasSpendableCodexCredits, isTerminalShortWindow, type CodexSpendableCredits } from "../quota-types"; import type { CodexQuotaScope } from "./health-store"; import type { TransientProbeGrant } from "./thread-affinity"; +import type { MainQuotaDispatch } from "../main-account-cache"; export { TERMINAL_SHORT_WINDOW_FRESHNESS_MS } from "../quota-types"; @@ -99,6 +100,10 @@ export type CodexUpstreamOutcomeMeta = { * means the caller cannot supply lineage and the historical unfenced handling stands. */ credentialGeneration?: number; + /** Main credential proof captured before dispatch; null means no owned proof was available. */ + mainFailureWindowDispatch?: MainQuotaDispatch | null; + /** Pool sample provenance retained by a terminal callback even if its context is reused. */ + failureWindowCredentialGeneration?: number; }; export function computeCodexUsageScore(quota: { diff --git a/src/codex/routing/failure-window.ts b/src/codex/routing/failure-window.ts index 71aa43ebee0..61ae76149be 100644 --- a/src/codex/routing/failure-window.ts +++ b/src/codex/routing/failure-window.ts @@ -1,5 +1,9 @@ import { pinnedCodexAccountId } from "../account-priority"; import type { OcxConfig } from "../../types"; +import { beginCodexAccountGenerationLiveCheck, isCodexAccountGenerationLive } from "../account-store"; +import { isMainQuotaDispatchLive } from "../main-account-cache"; +import type { CodexAuthContext } from "../auth-context"; +import type { CodexUpstreamOutcomeMeta } from "./cooldown-math"; import { deleteAccountHealth, setAccountHealth, @@ -14,12 +18,31 @@ export const CODEX_SLIDING_FAILURE_RECOVER_RATIO = 0.1; export const CODEX_SLIDING_FAILURE_RECOVER_SUSTAIN_MS = 30_000; const MAX_SAMPLES = 512; -type Sample = { at: number; failed: boolean }; -type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number }; +type Sample = { at: number; failed: boolean; credentialGeneration?: number }; +type WindowState = { samples: Sample[]; degraded: boolean; healthySince?: number; credentialGeneration?: number }; const windows = new Map(); const pinnedDegradeWarned = new Set(); +/** Read only dispatch-time provenance retained by auth materialization, never recapture at arrival. */ +export function codexFailureWindowOutcomeMeta( + context: CodexAuthContext, +): Pick { + if (context.kind === "pool") return { failureWindowCredentialGeneration: context.generation }; + return context.kind === "main-pool" ? { mainFailureWindowDispatch: context.mainQuotaDispatch ?? null } : {}; +} + +/** Fence window samples only; other health and recovery-lease semantics stay with their owners. */ +export function noteCodexFailureWindowOutcome( + config: OcxConfig, accountId: string, failed: boolean, now: number, meta: CodexUpstreamOutcomeMeta, +): void { + if (meta.mainFailureWindowDispatch !== undefined + && (!meta.mainFailureWindowDispatch || !isMainQuotaDispatchLive(meta.mainFailureWindowDispatch))) return; + const generation = meta.failureWindowCredentialGeneration ?? meta.credentialGeneration; + if (generation !== undefined && !isCodexAccountGenerationLive(accountId, generation)) return; + appendSample(config, accountId, failed, now, generation); +} + export function codexFailureWindowEnabled(config: OcxConfig): boolean { return config.codexFailureWindow !== false; } @@ -33,11 +56,27 @@ function stateFor(accountId: string): WindowState { return state; } -function recompute(state: WindowState, now: number): void { - const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; - if (state.samples.length > 0 && state.samples[0]!.at <= cutoff) { - state.samples = state.samples.filter(sample => sample.at > cutoff); +function recompute( + accountId: string, state: WindowState, now: number, + generationLive?: ReturnType, +): void { + // One store snapshot per insertion/read, never one file load per retained sample. The + // unlocked insertion precheck can race another process; provenance must survive it. + const tagged = state.credentialGeneration !== undefined + || state.samples.some(sample => sample.credentialGeneration !== undefined); + const live = generationLive ?? (tagged ? beginCodexAccountGenerationLiveCheck() : undefined); + // Unknown evidence is not a credential removal: preserve samples and all derived state. + if (live?.snapshotStatus === "failed") return; + if (state.credentialGeneration !== undefined && !live?.(accountId, state.credentialGeneration)) { + state.degraded = false; + state.healthySince = undefined; + state.credentialGeneration = undefined; + pinnedDegradeWarned.delete(accountId); } + const cutoff = now - CODEX_SLIDING_FAILURE_WINDOW_MS; + state.samples = state.samples.filter(sample => sample.at > cutoff + && (sample.credentialGeneration === undefined || live?.(accountId, sample.credentialGeneration))); + state.credentialGeneration ??= state.samples.find(sample => sample.credentialGeneration !== undefined)?.credentialGeneration; const samples = state.samples.length; const failed = state.samples.reduce((count, sample) => count + (sample.failed ? 1 : 0), 0); const degraded = samples >= CODEX_SLIDING_FAILURE_MIN_SAMPLES @@ -71,19 +110,32 @@ export function noteCodexFailureWindowSample( accountId: string, failed: boolean, now: number, +): void { + appendSample(config, accountId, failed, now); +} + +function appendSample( + config: OcxConfig, accountId: string, failed: boolean, now: number, credentialGeneration?: number, ): void { if (!codexFailureWindowEnabled(config)) return; + const existing = windows.get(accountId); + const tagged = credentialGeneration !== undefined || existing?.credentialGeneration !== undefined + || existing?.samples.some(sample => sample.credentialGeneration !== undefined); + const live = tagged ? beginCodexAccountGenerationLiveCheck() : undefined; + if (live?.snapshotStatus === "failed") return; + // A stale arrival is not an ownership transition and must not delete a live replacement's state. + if (credentialGeneration !== undefined && !live?.(accountId, credentialGeneration)) return; const state = stateFor(accountId); - state.samples.push({ at: now, failed }); + state.samples.push({ at: now, failed, ...(credentialGeneration === undefined ? {} : { credentialGeneration }) }); if (state.samples.length > MAX_SAMPLES) state.samples.splice(0, state.samples.length - MAX_SAMPLES); - recompute(state, now); + recompute(accountId, state, now, live); if (!state.degraded) pinnedDegradeWarned.delete(accountId); } export function isCodexFailureWindowDegraded(accountId: string, now = Date.now()): boolean { const state = windows.get(accountId); if (!state) return false; - recompute(state, now); + recompute(accountId, state, now); if (!state.degraded) pinnedDegradeWarned.delete(accountId); return state.degraded; } diff --git a/src/providers/openai-sidecar.ts b/src/providers/openai-sidecar.ts index c68c0c73e7b..93e3398ae18 100644 --- a/src/providers/openai-sidecar.ts +++ b/src/providers/openai-sidecar.ts @@ -12,6 +12,7 @@ import { type CodexAuthPolicyConfig, } from "../codex/auth-context"; import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome } from "../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../codex/routing/failure-window"; import { inspectChatGptDomainClaim } from "../oauth/chatgpt"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential, type DataPlaneAdmission } from "../server/auth-cors"; import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "../types"; @@ -205,6 +206,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); throw new CodexPoolAuthenticationError("Selected Codex account is unavailable"); } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -220,6 +222,7 @@ export async function resolveFirstUsableOpenAiSidecar( probeLeaseId: authContext.probeLeaseId, probeQuotaScope: authContext.probeQuotaScope, writerGeneration: authContext.writerGeneration, + ...windowMeta, // 401/403 here is evidence about this exact stored credential; without the generation a // replacement inherits the quarantine (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), @@ -256,6 +259,7 @@ export async function resolveFirstUsableOpenAiSidecar( releaseCodexAuthContextProbeLease(authContext); continue; } + const windowMeta = codexFailureWindowOutcomeMeta(authContext); return { ...candidate, authContext, @@ -271,6 +275,7 @@ export async function resolveFirstUsableOpenAiSidecar( threadId: authContext.affinityKey, probeLeaseId: authContext.probeLeaseId, writerGeneration: authContext.writerGeneration, + ...windowMeta, // Same fence as the exact-account recorder above (#2892 gap 4). ...(authContext.kind === "pool" ? { credentialGeneration: authContext.generation } : {}), }, diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index cb93a79f280..32263104a72 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -88,6 +88,7 @@ import { recordCodexUpstreamOutcome, type CodexUpstreamOutcome, } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountChangeConversationStateScrub, conversationStateBindingFromAuth, @@ -1062,6 +1063,7 @@ export async function handleResponsesCompact( if ((outcome === 401 || outcome === 403) && upstream && wasPermanentAccountRefusal(upstream)) return; recordCodexUpstreamOutcome(config, ctx.accountId, outcome, { ...meta, + ...codexFailureWindowOutcomeMeta(ctx), threadId: ctx.kind === "pool" || ctx.kind === "main-pool" ? ctx.affinityKey : undefined, fixedAccount: ctx.fixedAccount, modelId: selectedModelId, diff --git a/src/server/responses/core-auth.ts b/src/server/responses/core-auth.ts index 4734debe617..e9d49346a05 100644 --- a/src/server/responses/core-auth.ts +++ b/src/server/responses/core-auth.ts @@ -2,6 +2,7 @@ import type { OcxProviderConfig, OcxConfig } from "../../types"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import type { CodexAuthContext } from "../../codex/auth-context"; import type { RouteResult } from "../../router"; +import { routeConcreteModel } from "../../router"; import type { HandleResponsesOptions } from "./core-options"; import { hasForwardableCodexBearer, @@ -24,6 +25,7 @@ import { CodexAuthContextError, applyCodexAuthContextToProvider, stripCodexRuntimeProviderFields, + mainAccountHardLockCallerProof, } from "../../codex/auth-context"; import { codexAccountSelectionForTurn, tryClaimNativeMainProfileForTurn } from "../lifecycle"; import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup"; @@ -590,3 +592,21 @@ export async function refreshNativeMainForwardAuth(args: { }) ?? nativeMainRefreshFailureResponse(error) }; } } + +/** Inspect the original credential domain, before fallback changes the route or drops headers. */ +export function mainHardLockFallbackCallerCheck( + req: Request, config: OcxConfig, options: HandleResponsesOptions, +): (sourceModel: string) => Promise<(() => boolean) | null> { + return async sourceModel => { + let sourceRoute: ReturnType; + try { sourceRoute = routeConcreteModel(config, sourceModel); } + // This preview cannot replace the normal routing phase's existing error response. + catch { return null; } + const headers = codexRouteCredentialDomainHeaders(req, sourceRoute, options, false, config); + const { substituteMainCredential } = codexRouteCredentialOwnership(headers, config, sourceRoute, options); + return mainAccountHardLockCallerProof(headers, options.codexAuthPolicy ?? config, { + substituteMainCredentialForDirect: substituteMainCredential, + signal: options.abortSignal ?? req.signal, + }); + }; +} diff --git a/src/server/responses/core-codex-account.ts b/src/server/responses/core-codex-account.ts index a2b35f7da73..b51b7ce32c1 100644 --- a/src/server/responses/core-codex-account.ts +++ b/src/server/responses/core-codex-account.ts @@ -11,6 +11,7 @@ import { claimMainQuotaDispatchForWs, isMainQuotaDispatchLive, renewMainQuotaDis import type { CodexWsQuotaObserver } from "./codex-ws-metadata"; import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { isCodexAccountGenerationLive } from "../../codex/account-store"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { applyAccountQuotaFromUpstreamHeaders as applyCapturedCodexQuota } from "../../codex/quota"; import type { RouteResult } from "../../router"; import { @@ -93,20 +94,19 @@ export function sidecarOutcomeRecorder( config: OcxConfig, authCtx: CodexAuthContext, ): ((outcome: CodexUpstreamOutcome) => void) | undefined { - return authCtx.kind === "pool" || authCtx.kind === "main-pool" - ? outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { - threadId: authCtx.affinityKey, - fixedAccount: authCtx.fixedAccount, - probeLeaseId: authCtx.probeLeaseId, - probeQuotaScope: authCtx.probeQuotaScope, - writerGeneration: authCtx.writerGeneration, - // A vision or web-search sidecar can return 401/403, and that is evidence about the exact - // stored credential it used. Without the generation it becomes an account-wide quarantine - // that a replacement inherits (#2892 gap 4). `main-pool` has no stored-record generation, so - // it keeps the unfenced account-wide semantics. - ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), - }) - : undefined; + if (authCtx.kind !== "pool" && authCtx.kind !== "main-pool") return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); + return outcome => recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, + threadId: authCtx.affinityKey, + fixedAccount: authCtx.fixedAccount, + probeLeaseId: authCtx.probeLeaseId, + probeQuotaScope: authCtx.probeQuotaScope, + writerGeneration: authCtx.writerGeneration, + // Quarantine retains its stored-credential fence. main-pool has no stored-record + // generation; its separate dispatch proof above fences only failure-window samples. + ...(authCtx.kind === "pool" ? { credentialGeneration: authCtx.generation } : {}), + }); } @@ -148,6 +148,7 @@ export function recordCodexPreOutputRefusalOutcome( if (!usesCodexForwardPoolAuth(authCtx, provider) || wasPermanentAccountRefusal(response)) return; noteCodexWsPreOutputRefusalOutcome(response); recordCodexUpstreamOutcome(config, authCtx.accountId, response.status, { + ...codexFailureWindowOutcomeMeta(authCtx), threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -676,6 +677,7 @@ export async function retryCodexPoolOnAlternateAccount( if (codexWsPreOutputRefusalOutcomeRecorded(firstResponse)) return; noteCodexWsPreOutputRefusalOutcome(firstResponse); recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, fixedAccount: firstAuthCtx.fixedAccount, modelId: route.modelId, @@ -693,6 +695,7 @@ export async function retryCodexPoolOnAlternateAccount( if (outcomeStatus === firstResponse.status || (outcomeStatus !== 429 && outcomeStatus !== 402)) return; recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...codexQuotaOutcomeMeta(firstResponse), + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -862,6 +865,7 @@ export async function retryCodexPoolOnAlternateAccount( } recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...quotaMeta, + ...codexFailureWindowOutcomeMeta(firstAuthCtx), threadId: firstAuthCtx.affinityKey, modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), @@ -1149,6 +1153,7 @@ export function codexForwardTerminalOutcomeRecorder( logCtx?: RequestLogContext, ): ((status: ResponsesTerminalStatus, httpStatusOverride?: number) => void) | undefined { if (!usesCodexForwardPoolAuth(authCtx, provider)) return undefined; + const windowMeta = codexFailureWindowOutcomeMeta(authCtx); return (status, httpStatusOverride) => { const quotaStatus = [httpStatusOverride, logCtx?.terminalHttpStatus] .find(value => value === 429 || value === 402); @@ -1157,6 +1162,7 @@ export function codexForwardTerminalOutcomeRecorder( // request. Don't penalize account health; record success to clear any // prior soft-avoid so a healthy account isn't stuck avoided. recordCodexUpstreamOutcome(config, authCtx.accountId, 200, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, @@ -1180,6 +1186,7 @@ export function codexForwardTerminalOutcomeRecorder( ? 200 : (quotaStatus ?? httpStatusOverride ?? logCtx?.terminalHttpStatus ?? 502); recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + ...windowMeta, threadId: authCtx.affinityKey, fixedAccount: authCtx.fixedAccount, modelId, diff --git a/src/server/responses/main-hard-lock-routing.ts b/src/server/responses/main-hard-lock-routing.ts index e24081f5d8b..09f129a01be 100644 --- a/src/server/responses/main-hard-lock-routing.ts +++ b/src/server/responses/main-hard-lock-routing.ts @@ -1,6 +1,8 @@ import type { OcxConfig } from "../../types"; import { resolveComboId } from "../../combos"; -import { isBareOpenAiFamilyModel } from "../../router"; +import { isBareOpenAiFamilyModel, previewRouteModel } from "../../router"; +import { resolveChosenTarget } from "./shadow-target-availability"; +import { getRoutingProfile, resolvePolicyProfileId } from "../../routing/profile"; import { NATIVE_RESERVE_MODEL } from "../../codex/catalog/native-models"; import { getMainAccountHardLockStatus, @@ -10,6 +12,8 @@ import { codexQuotaScopeForModel } from "../../codex/routing/health-store"; import { getEligiblePoolAccounts } from "../../codex/routing/selection"; import { OPENAI_CODEX_PROVIDER_ID, isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { providerCodexAccountMode } from "../../providers/registry"; +/** Resolves to a synchronous ownership proof, or null when the caller does not own the locked main. */ +export type MainHardLockCallerCheck = (sourceModel: string) => Promise<(() => boolean) | null>; function isConfiguredComboSelector(config: OcxConfig, modelId: string): boolean { const comboId = resolveComboId(config, modelId); @@ -72,11 +76,12 @@ function rawModelOf(body: unknown): { raw: Record; model: strin * - Luna Reserve keeps its own admission policy (`assertReserveAdmission`); its helper * traffic is not operator chat and must not silently change models. */ -export function applyMainHardLockFallbackOverride( +export async function applyMainHardLockFallbackOverride( body: unknown, config: OcxConfig, - now = Date.now(), -): MainHardLockFallbackOverride | null { + now?: number, + callerCheck?: MainHardLockCallerCheck, +): Promise { const parsed = rawModelOf(body); if (!parsed) return null; const fallback = resolveMainAccountHardLockFallback(config); @@ -85,14 +90,34 @@ export function applyMainHardLockFallbackOverride( if (!isBareOpenAiFamilyModel(parsed.model) || parsed.model === NATIVE_RESERVE_MODEL || isConfiguredComboSelector(config, parsed.model)) return null; - if (getMainAccountHardLockStatus(config, now).state !== "blocked") return null; const accountMode = providerCodexAccountMode( OPENAI_CODEX_PROVIDER_ID, config.providers?.[OPENAI_CODEX_PROVIDER_ID], ); - if (accountMode !== "direct" - && getEligiblePoolAccounts(config, undefined, now, codexQuotaScopeForModel(parsed.model)).length > 0) { - return null; + if (accountMode === "direct") { + // Direct may forward a different caller's healthy credential. Only stored-main substitution + // or an observed-main match owes this lock. Recheck usage after the startup binding wait. + const stillOwnsLockedMain = callerCheck ? await callerCheck(parsed.model) : null; + // Everything from here to the rewrite is synchronous, so this proof cannot go stale before it. + if (!stillOwnsLockedMain?.()) return null; + if (getMainAccountHardLockStatus(config, now ?? Date.now()).state !== "blocked") return null; + } else { + if (getMainAccountHardLockStatus(config, now ?? Date.now()).state !== "blocked") return null; + if (getEligiblePoolAccounts(config, undefined, now ?? Date.now(), codexQuotaScopeForModel(parsed.model)).length > 0) return null; + } + + // Combos and policies keep their own dispatch admission and declared failover/evidence. + // Other chosen targets must resolve now; a removed qualifier cannot become a default send. + const policyId = resolvePolicyProfileId(config, fallback.model); + if (!isConfiguredComboSelector(config, fallback.model) + && !(policyId && getRoutingProfile(config, policyId))) { + try { + const target = resolveChosenTarget(fallback.model, model => previewRouteModel(config, model)); + if ("unavailable" in target) return null; + } catch { + // An unavailable target keeps the source lock's refusal without touching the body. + return null; + } } parsed.raw.model = fallback.model; diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 3d286dec672..cac782e0e28 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -37,6 +37,7 @@ import type { ResponsesTerminalStatus } from "../../bridge"; import { isCodexWsQuotaObservedResponse, isCodexWsUpstreamResponse, isCodexWsPreludeProjection } from "./ws-upstream"; import { recordSubagentQuotaFailureForThreadSpawn } from "../../codex/subagent-model-fallback"; import { recordCodexUpstreamOutcome } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta } from "../../codex/routing/failure-window"; import { codexProbeLeaseId, codexProbeQuotaScope, codexTransientProbeGrant, releaseCodexAuthContextProbeLease } from "../../codex/auth-context"; import { consumeComboFailure } from "./core-combo-failure"; import { clientCancelledResponse, readDisplaySafeErrorText } from "./core-errors"; @@ -535,6 +536,7 @@ export async function deliverPassthroughResponse( // request. The sibling recorders on this path already decline: the terminal recorder // needs an ok streaming body, and the quota-header snapshot finds no quota headers. recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, upstreamResponse.status, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), ...quotaMeta, threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index e33d91206e2..08415fc8956 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -132,7 +132,7 @@ import { import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error"; import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability"; import { captureAccountAutoPauseAttempt, wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; -import { recordCodexUpstreamOutcome } from "../../codex/routing"; +import { codexFailureWindowOutcomeMeta, recordCodexUpstreamOutcome } from "../../codex/routing"; import { describeUpstreamConnectFailure } from "./upstream-error"; import type { OpaqueBlobRecoveryGuard } from "./core-opaque-recovery"; import { @@ -990,6 +990,7 @@ export async function preparePassthroughExchange( } if (usesCodexForwardPoolAuth(admissionState.authCtx, route.provider)) { recordCodexUpstreamOutcome(config, admissionState.authCtx.accountId, outcome, { + ...codexFailureWindowOutcomeMeta(admissionState.authCtx), threadId: admissionState.authCtx.affinityKey, fixedAccount: admissionState.authCtx.fixedAccount, modelId: route.modelId, diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index e7ebf8eedf8..6b33dd11f41 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -100,6 +100,7 @@ import type { import { codexRouteCredentialDomainHeaders, codexRouteCredentialOwnership, + mainHardLockFallbackCallerCheck, resolveResponsesCodexAuth, withClaudeNativeSession, } from "./core-auth"; @@ -364,7 +365,16 @@ export async function prepareResponsesRequest( // must run before comboIdFromRawBody reads `model`. if (!concreteSelection && !options.compactionRoutingOverride && !isCompactionTrigger && !options.internalDecisionCall && !shadowCallIntercepted && !shadowCallMatches && !memoryModelApplies && !threadSpawn && inboundWire === "responses") { - options.mainHardLockFallbackOverride = applyMainHardLockFallbackOverride(body, config); + try { + options.mainHardLockFallbackOverride = await applyMainHardLockFallbackOverride( + body, config, undefined, mainHardLockFallbackCallerCheck(req, config, options), + ); + } catch (error) { + if (options.abortSignal?.aborted || req.signal.aborted) return clientCancelledResponse(); + const mapped = mapCodexAuthContextErrorToResponse(error, { now: Date.now() }); + if (mapped) return mapped; + throw error; + } } const comboId = !concreteSelection ? comboIdFromRawBody(body, config) : null; // A decision model may be a plain model or a non-JEV combo. A JEV combo here would recurse diff --git a/structure/INDEX.md b/structure/INDEX.md index e6c9e5e80bb..ea555bd6e75 100644 --- a/structure/INDEX.md +++ b/structure/INDEX.md @@ -84,6 +84,7 @@ Per-vendor contracts and the adapter authority that constructs them. | [`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md) | Anthropic OAuth instances (primary and Pool 2), account pause, model routes, quota labels, and pool-bound helpers, quota, reset grants and management surfaces. | | [`providers/oauth-code-display-login.md`](providers/oauth-code-display-login.md) | Headless code-display OAuth login: flow ownership, expiry, unsupported providers, single-operator store. | | [`providers/openai-tiers.md`](providers/openai-tiers.md) | Pool/Direct account modes, API-key separation, and the public provider and quota contract. | +| [`providers/pool-credit-renewal.md`](providers/pool-credit-renewal.md) | Independent credit-evidence renewal for opted-in exhausted Codex pool accounts. | | [`providers/openai-accounts.md`](providers/openai-accounts.md) | Migration and restore, wire identity, store concurrency, pool ordering and exclusions, quota observations, and account-bound retention. | | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | Opt-in effective priority bands that spend other pool accounts before the signed-in identity, its identity observation and management API. | | [`providers/cursor.md`](providers/cursor.md) | Cursor native exec, parameterized models, checkpoints, and active-context usage. | @@ -185,6 +186,7 @@ A source area can be described by more than one doc, because these docs are orga | `src/codex/account-preservation.ts` | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | | `src/codex/app-server-processes.ts` | [`codex-cli-update.md`](codex-cli-update.md) | | `src/codex/auth-api/account-preservation.ts` | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) | +| `src/codex/auth-api/pool-mode-gate.ts` | [`providers/pool-credit-renewal.md`](providers/pool-credit-renewal.md) | | `src/codex/catalog/effort.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md)
[`catalog-effort-clamp.md`](catalog-effort-clamp.md) | | `src/codex/catalog/model-hints.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md) | | `src/codex/catalog/ownership.ts` | [`catalog-routed-ownership.md`](catalog-routed-ownership.md) | diff --git a/structure/codex-account-controls.md b/structure/codex-account-controls.md index 6b186fd74c9..505cc2f3e1e 100644 --- a/structure/codex-account-controls.md +++ b/structure/codex-account-controls.md @@ -84,9 +84,14 @@ protects: - `getEligiblePoolAccounts` is consulted under the request model's quota scope whenever canonical `openai` is in Pool mode, so "the Pool can serve it" shares the dispatch's own eligibility and - scoped-cooldown rules. An empty eligible list (or Direct mode, which never consults stored Pool - accounts) is what makes the fallback the request's only way through; the lock excludes the main - account there itself. + scoped-cooldown rules. An empty eligible list permits fallback; the lock excludes the main + account from that list itself. +- Direct does not consult stored Pool accounts. Fallback applies only when the original source + route's credential ownership selects stored-main substitution, or the caller's bearer and + effective workspace match the currently observed main credential. Caller-owned checks wait + boundedly for pending startup policy binding, then recheck identity and the blocked state before + rewriting. Other callers retain their native route; a missing caller bearer keeps its normal + authentication refusal. - Only bare Codex-native selectors qualify. Provider-qualified rows, combos, account namespaces (including `nativeAlias` combos), compaction triggers, shadow-intercepted helper calls and policy routes name their own destination on purpose, and Luna Reserve keeps its own admission. A @@ -98,6 +103,11 @@ protects: - A malformed section (blank or padded model, undeclared effort) disables the reroute entirely — the same disable-don't-widen reading as the compaction override — while the load and write boundaries report the malformation. The lock's own refusal behavior is untouched either way. +- Before rewriting, concrete targets resolve against the current configuration through the + shared chosen-target check. Unavailable targets and qualified selectors that only reach the + terminal default provider retain the source lock's refusal. Bare default targets, declared + slash-containing model ids, configured combos and policy profiles keep their ordinary routing + semantics; policies retain request-evidence evaluation at dispatch preparation. The dashboard row mirrors the shadow-call row: the model pick, the effort and the switch each save immediately, and the switch writes `enabled: false` — a remembered pause that keeps the @@ -105,7 +115,8 @@ model — rather than clearing it. Settings projection and writes live beside th `src/server/management/config-routes.ts`; `null` clears the section. Coverage: `tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts` and the fallback cases in `tests/config/settings-main-account-hard-lock.test.ts` plus -`gui/tests/main-account-hard-lock-setting.test.tsx`. +`gui/tests/main-account-hard-lock-setting.test.tsx`. Startup-binding rechecks are covered by +`tests/codex-integration/main-account-policy-binding-wait.test.ts`. ## Stored-account authentication policy diff --git a/structure/decisions/ADR-7377-pool-credit-renewal.md b/structure/decisions/ADR-7377-pool-credit-renewal.md new file mode 100644 index 00000000000..fbea59f7e83 --- /dev/null +++ b/structure/decisions/ADR-7377-pool-credit-renewal.md @@ -0,0 +1,15 @@ +# ADR-7377: decision recorded under "Pool credit renewal" + +- Purpose: keep an explicitly opted-in credited pool account observable after its included usage + fills, without requiring a dashboard/manual metadata read every five minutes (#7377). +- Constraint: response headers update aggregate usage but do not observe credit balance. Reusing + aggregate freshness can therefore starve credit refresh while normal requests keep arriving. +- Alternatives: extend credit validity from traffic, add another polling loop, or reuse the existing + sweep and bounded pool quota primer. Extending validity invents spendable balance evidence; + another loop duplicates lifecycle/concurrency ownership. +- Decision: a pool-only prime reason selects exhausted, opted-in eligible accounts by the independent + credit clock, bypasses aggregate cache freshness, and reuses generation-aware probe/backoff owners. +- Consequences: metadata work starts from three minutes of evidence age with a matching minimum + per-credential retry gap. Native main keeps its separate passive/physical-identity path. Failed + reads do not make an account selectable or authorize any inference. A network outage can still + let credits expire; admission correctly remains closed until fresh evidence returns. diff --git a/structure/manifest.json b/structure/manifest.json index 43e7ed5815b..64dc945629b 100644 --- a/structure/manifest.json +++ b/structure/manifest.json @@ -532,6 +532,13 @@ "src/codex/" ] }, + { + "path": "providers/pool-credit-renewal.md", + "tier": 4, + "title": "Pool credit renewal", + "scope": "Independent credit-evidence renewal for opted-in exhausted Codex pool accounts.", + "documents": ["src/codex/auth-api/pool-mode-gate.ts"] + }, { "path": "providers/openai-accounts.md", "tier": 4, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index 809727ad810..04e3259b9ab 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -441,7 +441,7 @@ true for unknown usage, correctly for an unbound pick — would trade a warm pre account. `CODEX_UNKNOWN_USAGE_SCORE` is 101, so the second bar excludes an unobserved destination without a special case. -`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. +`src/codex/routing/failure-window.ts` keeps a 60-second sliding ratio beside the consecutive streak. Twenty or more terminal samples at a 25% transient-failure ratio mark the account degraded; it clears only after the ratio stays at or below 10% for 30 seconds. The ratio does not depend on completion order. Degraded accounts leave the unbound candidate list, so new threads move. A live thread binding is left alone, and a manual pin stays in place unless `codexPinnedTransientPolicy` is `detour-new-threads`. The default `hold` logs that the pin is degraded and keeps using it. `codexFailureWindow: false` leaves steering to the consecutive counter. Nothing here resends a turn that already started. The window is runtime state owned by the account id, so `purgeCodexAccountRuntimeState` forgets it with the rest of the alias-keyed rows: an account deleted and recreated under the same id, and a different physical login behind `__main__`, both start undegraded rather than inheriting the predecessor's samples or its pin-degraded warning. Terminal success and transient-failure samples carry pre-dispatch provenance: the stored Pool credential generation, or the owned main dispatch's identity generation and credential fences, including A-to-B-to-A transitions. Header materialization captures that proof; terminal and sidecar callbacks retain it rather than recapturing at arrival. Pool samples retain their generation through insertion and are revalidated from one current store snapshot per insertion or lookup, so cross-process replacement after an unlocked precheck cannot donate a sample. A retired generation's degraded state, recovery timer and pin warning reset when read; rejecting an old arrival never deletes the replacement's live window. A snapshot read or parse failure is explicitly unknown and preserves the samples, degraded flag, recovery timer, retained generation and pin warning; a genuinely absent or valid empty store still retires its old generations. An expired proof cannot refill the replacement's window or contribute a late success to its sample count. This fence applies only to window samples; existing account-health and probe-lease semantics retain their own owners. Movement is therefore bounded by the number of accounts rather than the number of turns. The rule narrows a preference and never a refusal: a 429/402 with no success since, a failover streak, pause, @@ -535,7 +535,7 @@ including failures, while preserving main-owner claims and per-credential WHAM d No requests means no new polling. For this reason only, stale observation proof bypasses aggregate quota-cache freshness after the existing attempt backoff. Main refresh keeps its owned lease and passive intent: cache bypass does not clear an inference reauthentication mark. Other prime reasons -retain their existing cache rules. The split config schema degrades malformed optional values to +retain their existing cache rules except the independently scheduled [pool credit renewal](pool-credit-renewal.md). The split config schema degrades malformed optional values to false. Exact-account and Direct routes are unchanged. ## Main-account policy observations diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 9631ceab68e..caffc0dd5bf 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -334,7 +334,7 @@ identity retries: WHAM success and terminal 401/403 responses never set or clear this mode. Native token preparation also preserves the traffic quarantine. Other callers retain the existing auth behavior. Failed or incomplete observations never renew the credit clock; refreshing usage does not redeem reset credits or validate pending accounts through inference. -Coverage: `tests/codex-integration/main-account-credit-renewal.test.ts`. +Coverage: `tests/codex-integration/main-account-credit-renewal.test.ts`. Pool accounts use the separate [pool credit renewal](pool-credit-renewal.md) contract; their fresh spendable-evidence requirement is identical. Credit parsing, expiry, partial updates and reset-ticket separation are covered in `tests/codex-integration/codex-quota-parser-parity.test.ts`; selection and bulk-pause behavior @@ -378,9 +378,13 @@ It blocks newly admitted identity-matched main-account requests. Pool alternativ explicit main selection and stored Direct substitution do not override it. It neither pauses the account nor clears upstream cooldown/reauth state, and management quota refresh remains available. Fresh valid usage below the effective threshold, including 0%, or validated WHAM absence retires a measured short block; -passing a reset timestamp alone does not. The minute sweep waits locally until the latest known blocking reset; -when no future reset is known or reads remain blocked, main recovery uses the same capped -5/10/20/40/60-minute delay calculation as usage-query failures. Skipped ticks do not extend it; +passing a reset timestamp alone does not. A future known reset is a scheduling hint (#7312): recovery +arms one base delay the first time an identity/credential sees the hold (including a hold that appears +during token preparation), then reads WHAM metadata on the same capped 5/10/20/40/60-minute delay +calculation as usage-query failures. The due time is pulled in to the latest known reset, including one +that moved earlier or just passed, but hold reads stay at least one base interval apart; a nonterminal +401/403 retry does not re-arm the first-hold delay. An early reset redeemed elsewhere is thus learned +without inference or reset-credit use. A paused or needs-reauth main gets no hold rechecks. Skipped ticks do not extend the delay; the physical bearer is reconciled before checking the delay, and late results cannot charge a replacement credential. A longer valid Retry-After from any main usage reader is checked for the current credential before the recovery worker takes a profile lease or prepares a token; a replacement diff --git a/structure/providers/pool-credit-renewal.md b/structure/providers/pool-credit-renewal.md new file mode 100644 index 00000000000..ad47b4ebef0 --- /dev/null +++ b/structure/providers/pool-credit-renewal.md @@ -0,0 +1,24 @@ +# Pool credit renewal + +`src/codex/auth-api/pool-mode-gate.ts` renews independently aged pool credit evidence on the +existing `codex-cooldown-recovery` after-tick hook, using `primeCodexPoolQuotas` with reason +`credit-renewal`. No extra timer or inference request is added. + +Only configured pool accounts opted into `creditCodexAccountIds` with a currently exhausted +included window qualify. Paused, reauthentication-required, below-limit and non-opted-in accounts +are skipped. A credit observation younger than three minutes is retained; due or missing evidence +uses the existing bounded-concurrency quota reader with cache bypass, because traffic headers can +keep aggregate usage fresh while the independent credit clock expires. + +The existing per-credential attempt map bounds retries to at least three minutes. Query backoff, +request admission and generation/configuration publication fences remain owned by the quota reader. +Only actual WHAM evidence advances the credit clock. A failed read leaves stale evidence unusable; +no credit freshness limit or spending consent is relaxed. Negative evidence remains discoverable. +The eligibility check runs again before each queued account read. The pool-only reason does not +claim, reconcile or read native main; its separate renewal owner is unchanged. + +Regression coverage: `tests/codex-integration/codex-quota-prime.test.ts` exercises pre-expiry and +expired evidence, admission after a fresh read, the sweep entry, no native-main work, exclusions +and failed-read backoff. Tests use fake metadata and private homes, not paid inference. + +> Decision record: [ADR-7377](../decisions/ADR-7377-pool-credit-renewal.md) diff --git a/tests/codex-integration/codex-account-delete-atomicity.test.ts b/tests/codex-integration/codex-account-delete-atomicity.test.ts index b1df61a9400..9ec8ae90591 100644 --- a/tests/codex-integration/codex-account-delete-atomicity.test.ts +++ b/tests/codex-integration/codex-account-delete-atomicity.test.ts @@ -32,6 +32,12 @@ import { getConfigPath, loadConfig, saveConfig } from "../../src/config"; import * as configModule from "../../src/config"; import { setCodexAccountAutoSwitchThresholdOverride } from "../../src/codex/account-auto-switch"; import { clearAllManualPreferences, manualPreferenceBlocks, resetCodexRoutingForManualSelection } from "../../src/codex/routing/active-account"; +import { + failureWindowSteersNewThreads, + forgetCodexFailureWindow, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { POOL_KEY_CODEX } from "../../src/oauth/pool-kernel"; import { prepareConfigObjectChildDeletionRebase } from "../../src/config/rebase-provenance"; import type { OcxConfig } from "../../src/types"; @@ -429,4 +435,56 @@ describe("Codex account delete persistence ordering", () => { clearAllManualPreferences(); } }); + + test("deleting the account forgets its failure-window degradation but keeps sibling state", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (accountId: string): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, accountId, true, now - index); + }; + degrade(ACCOUNT_ID); + degrade("sibling-account"); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(true); + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + try { + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + expect(isCodexFailureWindowDegraded(ACCOUNT_ID, now)).toBe(false); + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + // A different account's window is untouched by the deletion. + expect(isCodexFailureWindowDegraded("sibling-account", now)).toBe(true); + } finally { + forgetCodexFailureWindow(ACCOUNT_ID); + forgetCodexFailureWindow("sibling-account"); + } + }); + + test("deleting the account forgets its pinned-degrade warning", () => { + const config = seededConfig(); + const now = Date.now(); + const degrade = (): void => { + for (let index = 0; index < 24; index += 1) noteCodexFailureWindowSample(config, ACCOUNT_ID, true, now - index); + }; + forgetCodexFailureWindow(ACCOUNT_ID); + degrade(); + const warnSpy = spyOn(console, "warn").mockImplementation(() => {}); + try { + // The seeded config pins ACCOUNT_ID under the default "hold" policy, so steering + // logs the pin-degraded warning exactly once per process-lifetime of the state. + expect(failureWindowSteersNewThreads(config, ACCOUNT_ID, now)).toBe(false); + expect(warnSpy).toHaveBeenCalledTimes(1); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(deleteCodexAccount(config, ACCOUNT_ID)).toBe(true); + // Deletion clears the pin as well; re-pin to model a recreated same-id account the + // operator selects again. If the warned set survived the purge, a re-degraded + // same-id pinned account would stay silent forever; the forget lets it fire again. + config.activeCodexAccountPinned = ACCOUNT_ID; + degrade(); + failureWindowSteersNewThreads(config, ACCOUNT_ID, now); + expect(warnSpy).toHaveBeenCalledTimes(2); + } finally { + warnSpy.mockRestore(); + forgetCodexFailureWindow(ACCOUNT_ID); + } + }); }); diff --git a/tests/codex-integration/codex-auth-context.test.ts b/tests/codex-integration/codex-auth-context.test.ts index 9f1244eb43e..cdd00a433db 100644 --- a/tests/codex-integration/codex-auth-context.test.ts +++ b/tests/codex-integration/codex-auth-context.test.ts @@ -2431,3 +2431,20 @@ test("context Direct bearer admission uses real stored-main materialization and globalThis.fetch = originalFetch; } }); + +test("pool exhaustion never falls back to an observed paused main caller", async () => { + const cfg = config(); + cfg.codexAccounts = []; + cfg.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + cfg.codexMainAccountHardLock = false; + observeMainQuotaIdentity("paused-main-account"); + observeMainQuotaCredential("paused-main-bearer", "paused-main-account"); + const headers = new Headers({ authorization: "Bearer paused-main-bearer", "chatgpt-account-id": "paused-main-account" }); + await expect(resolveCodexAuthContext(headers, cfg, "pool", { + requestScopedMainCredential: true, + })).rejects.toBeInstanceOf(CodexMainProfileDrainingError); + // Direct remains caller-owned; the pause applies to Pool fallback, not unrelated callers. + expect(await resolveCodexAuthContext(headers, cfg, "direct", { + requestScopedMainCredential: true, + })).toMatchObject({ kind: "main", accountId: null }); +}); diff --git a/tests/codex-integration/codex-failure-window.test.ts b/tests/codex-integration/codex-failure-window.test.ts index 55e5e773b24..ee16af342df 100644 --- a/tests/codex-integration/codex-failure-window.test.ts +++ b/tests/codex-integration/codex-failure-window.test.ts @@ -1,8 +1,16 @@ -import { afterEach, beforeEach, expect, test } from "bun:test"; -import { mkdtempSync } from "node:fs"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; +import * as fs from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import * as accountStore from "../../src/codex/account-store"; +import { headersForCodexAuthContext, type CodexAuthContext } from "../../src/codex/auth-context"; +import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { captureMainQuotaWriter, clearMainAccountInfoCache } from "../../src/codex/main-account-cache"; +import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/account-id"; +import { captureConfigGeneration } from "../../src/lib/state-store-sweeper"; +import { codexForwardTerminalOutcomeRecorder, sidecarOutcomeRecorder } from "../../src/server/responses/core-codex-account"; import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../../src/codex/auth-api"; import { clearCodexUpstreamHealth, @@ -12,11 +20,12 @@ import { } from "../../src/codex/routing"; import { forgetCodexFailureWindows, + failureWindowSteersNewThreads, isCodexFailureWindowDegraded, noteCodexFailureWindowSample, } from "../../src/codex/routing/failure-window"; import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; -import type { OcxConfig } from "../../src/types"; +import type { OcxConfig, OcxProviderConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; const NOW = 1_700_000_000_000; @@ -47,13 +56,19 @@ function fill(accountId: string, failures: number, successes: number, now = NOW) let home = ""; let previousHome: string | undefined; +let previousCodexHome: string | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; home = mkdtempSync(join(tmpdir(), "ocx-window-")); setIcaclsRunnerForTests(() => ICACLS_OK); setAsyncIcaclsRunnerForTests(async () => ICACLS_OK); process.env.OPENCODEX_HOME = home; + process.env.CODEX_HOME = join(home, "codex"); + mkdirSync(process.env.CODEX_HOME, { recursive: true }); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); clearThreadAccountMap(); clearCodexUpstreamHealth(); clearAccountQuota(); @@ -75,11 +90,383 @@ afterEach(async () => { clearThreadAccountMap(); clearAccountQuota(); forgetCodexFailureWindows(); + clearMainAccountInfoCache(); + resetMainCodexAccountIdentityTrackingForTests(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; await removeTreeWithRetry(home); }); +const forwardProvider = { adapter: "openai-responses", authMode: "forward" } as OcxProviderConfig; + +function terminal(config: OcxConfig, context: CodexAuthContext) { + headersForCodexAuthContext(new Headers(), context, config); + return codexForwardTerminalOutcomeRecorder(config, context, forwardProvider)!; +} + +function mainTerminal(config: OcxConfig, identity: string, accessToken = `access-${identity}`) { + writeFileSync(join(process.env.CODEX_HOME!, "auth.json"), JSON.stringify({ tokens: { access_token: accessToken, account_id: identity } })); + reconcileMainCodexAccountRuntimeState(); + return terminal(config, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + mainQuotaWriter: captureMainQuotaWriter(identity), accessToken, chatgptAccountId: identity, fixedAccount: true, + }); +} + +function poolTerminal(config: OcxConfig) { + const record = readCodexAccountRecord("a")!; + return terminal(config, { + kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), generation: record.generation, + accessToken: record.credential!.accessToken, chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true, + }); +} + +test("late terminal outcomes cannot refill a replacement main failure window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const replacement = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) replacement("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + replacement("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a returned main identity does not revive the first login's terminal writer", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const first = mainTerminal(cfg, "physical-a"); + mainTerminal(cfg, "physical-b"); + const returned = mainTerminal(cfg, "physical-a"); + for (let index = 0; index < 20; index++) first("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) returned("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("late successful terminals cannot count toward the replacement main window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a"); + const current = mainTerminal(cfg, "physical-b"); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + old("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("sidecar recorder retains the old main dispatch even if its context is reused", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const context: CodexAuthContext = { kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, + writerGeneration: captureConfigGeneration(), mainQuotaWriter: captureMainQuotaWriter("physical-a"), + accessToken: "access-physical-a", chatgptAccountId: "physical-a" }; + headersForCodexAuthContext(new Headers(), context, cfg); + const old = sidecarOutcomeRecorder(cfg, context)!; + const current = mainTerminal(cfg, "physical-b"); + context.mainQuotaWriter = captureMainQuotaWriter("physical-b"); + context.accessToken = "access-physical-b"; + context.chatgptAccountId = "physical-b"; + headersForCodexAuthContext(new Headers(), context, cfg); + for (let index = 0; index < 20; index++) old(500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("a replaced main credential retires its terminal samples without an identity change", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + const old = mainTerminal(cfg, "physical-a", "old-access"); + const current = mainTerminal(cfg, "physical-a", "new-access"); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + current("incomplete"); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(true); +}); + +test("same-id pool recreation rejects the deleted credential's late terminal samples", () => { + const cfg = config(); + const old = poolTerminal(cfg); + deleteCodexAccount(cfg, "a"); + saveCodexAccountCredential("a", { accessToken: "replacement-access", refreshToken: "replacement-refresh", chatgptAccountId: "replacement-account", expiresAt: Date.now() + 60_000 }); + cfg.codexAccounts!.push({ id: "a", email: "replacement@test", isMain: false }); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("terminal pool samples keep the dispatched generation if the context is reused", () => { + const cfg = config(); + const record = readCodexAccountRecord("a")!; + const context: CodexAuthContext = { kind: "pool", accountId: "a", writerGeneration: captureConfigGeneration(), + generation: record.generation, accessToken: record.credential!.accessToken, + chatgptAccountId: record.credential!.chatgptAccountId, fixedAccount: true }; + const old = terminal(cfg, context); + context.generation = saveCodexAccountCredential("a", { ...record.credential!, accessToken: "new-access" }); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 20; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a main terminal without an owned dispatch proof cannot seed the window", () => { + const cfg = config({ activeCodexAccountId: MAIN_CODEX_ACCOUNT_ID }); + mainTerminal(cfg, "physical-a"); + const record = codexForwardTerminalOutcomeRecorder(cfg, { + kind: "main-pool", accountId: MAIN_CODEX_ACCOUNT_ID, writerGeneration: captureConfigGeneration(), + accessToken: "unobserved-access", chatgptAccountId: "physical-a", + }, forwardProvider)!; + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID)).toBe(false); +}); + +test("a cross-process replacement after the unlocked live check cannot donate a terminal sample", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.isCodexAccountGenerationLive; + let swapped = false; + const liveCheck = spyOn(accountStore, "isCodexAccountGenerationLive").mockImplementation((id, generation) => { + const live = original(id, generation); + if (id === "a" && live && !swapped) { + swapped = true; + const credential = readCodexAccountRecord(id)!.credential!; + saveCodexAccountCredential(id, { ...credential, accessToken: "replacement-after-read" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + liveCheck.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a replacement does not inherit degraded state or the prior pinned warning", () => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const old = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-access" }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + old("completed"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(2); + // Rejecting a stale writer must not erase the replacement's healthy or degraded window. + old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + } finally { + warn.mockRestore(); + } +}); + +test("a replacement after the insertion snapshot is filtered at window read time", () => { + const cfg = config(); + const old = poolTerminal(cfg); + const original = accountStore.beginCodexAccountGenerationLiveCheck; + let swapped = false; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(() => { + const live = original(); + if (!swapped) { + swapped = true; + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-snapshot" }); + } + return live; + }); + try { + old("completed"); + expect(swapped).toBe(true); + } finally { + snapshot.mockRestore(); + } + const current = poolTerminal(cfg); + for (let index = 0; index < 19; index++) current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + current("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test("a full tagged window loads one current credential snapshot per read", () => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 512; index++) { + recordCodexUpstreamOutcome(cfg, "a", 200, { failureWindowCredentialGeneration: generation, now: NOW }); + } + const original = accountStore.beginCodexAccountGenerationLiveCheck; + const snapshot = spyOn(accountStore, "beginCodexAccountGenerationLiveCheck").mockImplementation(original); + try { + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(1); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "next-full-window" }); + expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); + expect(snapshot).toHaveBeenCalledTimes(2); + } finally { + snapshot.mockRestore(); + } +}); + +function withFailedStoreSnapshot(kind: "read" | "parse", run: () => void): void { + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const original = fs.readFileSync; + const read = kind === "read" ? spyOn(fs, "readFileSync").mockImplementation((...args: Parameters) => { + if (String(args[0]) === path) throw Object.assign(new Error("fixture read unavailable"), { code: "EACCES" }); + return original(...args); + }) : undefined; + if (kind === "parse") writeFileSync(path, "{incomplete-json"); + try { + run(); + } finally { + read?.mockRestore(); + writeFileSync(path, bytes); + } +} + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves degraded samples and the pinned warning", kind => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + withFailedStoreSnapshot(kind, () => { + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + } +}); + +test.each(["read", "parse"] as const)("a temporary %s failure does not erase or append to nineteen retained samples", kind => { + const cfg = config(); + const record = poolTerminal(cfg); + for (let index = 0; index < 19; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + withFailedStoreSnapshot(kind, () => { + noteCodexFailureWindowSample(cfg, "a", true, Date.now()); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + }); + expect(isCodexFailureWindowDegraded("a")).toBe(false); + record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); +}); + +test.each(["read", "parse"] as const)("a temporary %s snapshot failure preserves the recovery timer and retained generation", kind => { + const cfg = config(); + const generation = readCodexAccountRecord("a")!.generation; + const record = (failed: boolean, now: number) => recordCodexUpstreamOutcome(cfg, "a", failed ? 500 : 200, + { failureWindowCredentialGeneration: generation, now }); + for (let index = 0; index < 20; index++) record(true, NOW); + for (let index = 0; index < 20; index++) record(false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + withFailedStoreSnapshot(kind, () => expect(isCodexFailureWindowDegraded("a", NOW + 75_000)).toBe(true)); + expect(isCodexFailureWindowDegraded("a", NOW + 90_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 91_000)).toBe(false); + const credential = readCodexAccountRecord("a")!.credential!; + saveCodexAccountCredential("a", { ...credential, accessToken: "replacement-after-failure" }); + const current = readCodexAccountRecord("a")!.generation; + for (let index = 0; index < 19; index++) recordCodexUpstreamOutcome(cfg, "a", 500, + { failureWindowCredentialGeneration: current, now: NOW + 92_000 }); + expect(isCodexFailureWindowDegraded("a", NOW + 92_000)).toBe(false); +}); + +test.each(["absent", "empty"] as const)("a valid %s snapshot still retires the previous generation", kind => { + const cfg = config(); + const old = poolTerminal(cfg); + for (let index = 0; index < 20; index++) old("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + const path = join(home, "codex-accounts.json"); + if (kind === "absent") unlinkSync(path); + else writeFileSync(path, "{}"); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("ok"); + expect(isCodexFailureWindowDegraded("a")).toBe(false); +}); + +test.each(["[]", "false", "0", "null", "\"\"", "\"text\""])("a wrong-shaped top-level snapshot %s preserves the failure window", contents => { + const cfg = config({ activeCodexAccountPinned: "a" }); + const record = poolTerminal(cfg); + const path = join(home, "codex-accounts.json"); + const bytes = readFileSync(path); + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + for (let index = 0; index < 20; index++) record("failed", 500); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, contents); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + expect(accountStore.beginCodexAccountGenerationLiveCheck().snapshotStatus).toBe("failed"); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + writeFileSync(path, bytes); + expect(isCodexFailureWindowDegraded("a")).toBe(true); + failureWindowSteersNewThreads(cfg, "a", Date.now()); + expect(warn).toHaveBeenCalledTimes(1); + } finally { + warn.mockRestore(); + writeFileSync(path, bytes); + } +}); + +test("a replacement does not inherit the previous generation's recovery timer", () => { + const cfg = config(); + const firstGeneration = readCodexAccountRecord("a")!.generation; + const record = (outcome: number, generation: number, now: number) => recordCodexUpstreamOutcome(cfg, "a", outcome, + { failureWindowCredentialGeneration: generation, now }); + const fillGeneration = (generation: number, failed: boolean, now: number) => { + for (let index = 0; index < 20; index++) record(failed ? 500 : 200, generation, now); + }; + fillGeneration(firstGeneration, true, NOW); + fillGeneration(firstGeneration, false, NOW + 61_000); + expect(isCodexFailureWindowDegraded("a", NOW + 61_000)).toBe(true); + const credential = readCodexAccountRecord("a")!.credential!; + const nextGeneration = saveCodexAccountCredential("a", { ...credential, accessToken: "next-access" }); + fillGeneration(nextGeneration, false, NOW + 62_000); + expect(isCodexFailureWindowDegraded("a", NOW + 62_000)).toBe(false); + fillGeneration(nextGeneration, true, NOW + 63_000); + expect(isCodexFailureWindowDegraded("a", NOW + 63_000)).toBe(true); + // Only G2 successes remain in-window; its own 30-second recovery starts here. + fillGeneration(nextGeneration, false, NOW + 124_000); + expect(isCodexFailureWindowDegraded("a", NOW + 153_999)).toBe(true); + expect(isCodexFailureWindowDegraded("a", NOW + 154_000)).toBe(false); +}); + test("fewer than 20 samples do not degrade an account", () => { fill("a", 19, 0); expect(isCodexFailureWindowDegraded("a", NOW)).toBe(false); diff --git a/tests/codex-integration/codex-main-rotation.test.ts b/tests/codex-integration/codex-main-rotation.test.ts index ffa98019c1e..bfc114c935d 100644 --- a/tests/codex-integration/codex-main-rotation.test.ts +++ b/tests/codex-integration/codex-main-rotation.test.ts @@ -21,6 +21,11 @@ import { reconcileMainCodexAccountRuntimeState, resetMainCodexAccountIdentityTrackingForTests, } from "../../src/codex/account-lifecycle"; +import { + forgetCodexFailureWindows, + isCodexFailureWindowDegraded, + noteCodexFailureWindowSample, +} from "../../src/codex/routing/failure-window"; import { MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "../../src/codex/main-account"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { @@ -243,6 +248,29 @@ describe("main account rotation (Option A)", () => { } }); + test("a physical main account switch forgets its failure-window degradation", () => { + const config = makeConfig(); + const now = Date.now(); + for (let index = 0; index < 24; index += 1) { + noteCodexFailureWindowSample(config, MAIN_CODEX_ACCOUNT_ID, true, now - index); + } + try { + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(true); + // First reconcile observes the fixture identity; the physical switch happens on the + // replacement write below and must drop the alias-keyed window like every other + // runtime row the purge owns. + expect(reconcileMainCodexAccountRuntimeState()).toBe(false); + writeFileSync( + join(CODEX_DIR, "auth.json"), + JSON.stringify({ tokens: { access_token: "replacement_access", account_id: "replacement_acct" } }), + ); + expect(reconcileMainCodexAccountRuntimeState()).toBe(true); + expect(isCodexFailureWindowDegraded(MAIN_CODEX_ACCOUNT_ID, now)).toBe(false); + } finally { + forgetCodexFailureWindows(); + } + }); + test("does not treat a transient missing auth file as an account switch", () => { expect(reconcileMainCodexAccountRuntimeState()).toBe(false); recordCodexUpstreamOutcome(makeConfig(), MAIN_CODEX_ACCOUNT_ID, 429, { retryAfter: "3600" }); diff --git a/tests/codex-integration/codex-quota-prime.test.ts b/tests/codex-integration/codex-quota-prime.test.ts index 75bd5fbfd12..f972cdeb24c 100644 --- a/tests/codex-integration/codex-quota-prime.test.ts +++ b/tests/codex-integration/codex-quota-prime.test.ts @@ -22,6 +22,9 @@ import { saveCodexAccountCredential, } from "../../src/codex/account-store"; import { resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; +import { isCodexAccountHeldForCredits } from "../../src/codex/account-credit-use"; +import * as sweeper from "../../src/lib/state-store-sweeper"; +import { registerCodexCooldownRecoveryProbeWorker } from "../../src/codex/auth-api/pool-mode-gate"; import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/main-account"; import { completeNativeMainRecovery, @@ -142,6 +145,116 @@ describe("primeCodexPoolQuotas", () => { if (existsSync(TEST_DIR)) removeTreeWithRetry(TEST_DIR); }); + test.each([180_000, 360_000])("credit renewal at %d ms reads WHAM despite fresh traffic headers and never touches native main (#7377)", async age => { + const config = makeConfig({ creditCodexAccountIds: ["p1"] }); + seedPoolAccount(config, "p1", "plus"); + const now = Date.now(); + setAccountQuotaFromParsed("p1", { shortPercent: 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - age } }); + expect(isCodexAccountHeldForCredits(config, "p1", getAccountQuota("p1"), "plus", now)).toBe(age > 300_000); + // A real response header updates usage, not the older credit observation. + updateAccountQuota("p1", 10); + const originalFetch = globalThis.fetch; + const probed: string[] = []; + try { + globalThis.fetch = (async (input, init) => { + expect(String(input)).toContain("/backend-api/wham/usage"); + probed.push(new Headers(init?.headers).get("chatgpt-account-id")!); + return Response.json({ plan_type: "plus", rate_limit: { + primary_window: { used_percent: 100, reset_at: Math.floor((now + 3_600_000) / 1000), limit_window_seconds: 18_000 }, + }, credits: { has_credits: true, balance: 5, overage_limit_reached: false }, spend_control: { reached: false } }); + }) as typeof fetch; + let mainAccesses = 0; + const noMain = () => { mainAccesses++; throw new Error("credit renewal must not access native main"); }; + await primeCodexPoolQuotas(config, "credit-renewal", { reconcileMainAccount: noMain, readMainTokens: noMain, fetchMainInfo: noMain }); + expect(mainAccesses).toBe(0); + expect(probed).toEqual(["acct-p1"]); + expect(getAccountQuota("p1")?.credits?.observedAt).toBeGreaterThanOrEqual(now); + expect(isCodexAccountHeldForCredits(config, "p1", getAccountQuota("p1"), "plus", Date.now())).toBe(false); + let afterTick: (() => void) | undefined; + const registration = spyOn(sweeper, "registerStateSweepAfterTick").mockImplementation(entry => { + afterTick = entry.afterTick; return () => {}; + }); + try { + // Stale credit evidence is independently renewable on the existing sweep. + clearCodexQuotaPrimeState(); + setAccountQuotaFromParsed("p1", { credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - age } }); + registerCodexCooldownRecoveryProbeWorker(config); + expect(afterTick).toBeDefined(); + afterTick!(); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(probed).toHaveLength(2); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(probed).toHaveLength(2); + } finally { registration.mockRestore(); } + } finally { globalThis.fetch = originalFetch; } + }); + + test("credit renewal excludes non-opted-in, paused, reauth, below-limit and fresh-credit accounts", async () => { + const config = makeConfig({ creditCodexAccountIds: ["paused", "reauth", "below", "fresh"], pausedCodexAccountIds: ["paused"] }); + const now = Date.now(); + for (const id of ["off", "paused", "reauth", "below", "fresh"]) { + seedPoolAccount(config, id, "plus"); + setAccountQuotaFromParsed(id, { shortPercent: id === "below" ? 50 : 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - (id === "fresh" ? 0 : 180_000) } }); + } + markAccountNeedsReauth("reauth"); + const originalFetch = globalThis.fetch; + let calls = 0; + try { + globalThis.fetch = (async () => { calls++; throw new Error("no eligible credit renewal"); }) as typeof fetch; + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(calls).toBe(0); + expect(isAccountNeedsReauth("reauth")).toBe(true); + } finally { globalThis.fetch = originalFetch; clearAccountNeedsReauth("reauth"); } + }); + + test("failed credit renewal backs off without making stale credits spendable", async () => { + const config = makeConfig({ creditCodexAccountIds: ["p1"] }); + seedPoolAccount(config, "p1", "plus"); + const now = Date.now(); + setAccountQuotaFromParsed("p1", { shortPercent: 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - 360_000 } }); + const originalFetch = globalThis.fetch; + let calls = 0; + try { + globalThis.fetch = (async () => { calls++; return new Response(null, { status: 503 }); }) as typeof fetch; + await primeCodexPoolQuotas(config, "credit-renewal"); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(calls).toBe(1); + expect(getAccountQuota("p1")?.credits?.observedAt).toBe(now - 360_000); + } finally { globalThis.fetch = originalFetch; } + }); + + test("credit renewal pacing is per credential generation and coalesces concurrent sweeps (#7290 intake)", async () => { + const config = makeConfig({ creditCodexAccountIds: ["p1"] }); + seedPoolAccount(config, "p1", "plus"); + const now = Date.now(); + setAccountQuotaFromParsed("p1", { shortPercent: 100, shortResetAt: now + 3_600_000, + shortObservedAt: now, credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - 360_000 } }); + const originalFetch = globalThis.fetch; + const tokens: string[] = []; + try { + globalThis.fetch = (async (input, init) => { + expect(String(input)).toContain("/backend-api/wham/usage"); + expect(init?.method ?? "GET").toBe("GET"); + tokens.push(new Headers(init?.headers).get("authorization") ?? ""); + return new Response(null, { status: 503 }); + }) as typeof fetch; + // Two sweeps racing on one generation share one metadata read. + await Promise.all([primeCodexPoolQuotas(config, "credit-renewal"), primeCodexPoolQuotas(config, "credit-renewal")]); + expect(tokens).toEqual(["Bearer access-p1"]); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(tokens).toHaveLength(1); + // Replacing the credential starts a new generation, which is not bound by the old failure. + saveCodexAccountCredential("p1", { accessToken: "access-p1-next", refreshToken: "refresh-p1-next", + expiresAt: Date.now() + 5 * 60_000, chatgptAccountId: "acct-p1" }); + await primeCodexPoolQuotas(config, "credit-renewal"); + expect(tokens).toEqual(["Bearer access-p1", "Bearer access-p1-next"]); + expect(isCodexAccountHeldForCredits(config, "p1", getAccountQuota("p1"), "plus", Date.now())).toBe(true); + } finally { globalThis.fetch = originalFetch; } + }); + test("startup may observe pending account quota without inference or validation", async () => { const config = makeConfig(); seedPoolAccount(config, "pending", "pro"); diff --git a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts index 14b91b46535..ddfa97d006d 100644 --- a/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts +++ b/tests/codex-integration/main-account-hard-lock-fallback-routing.test.ts @@ -3,8 +3,10 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { applyMainHardLockFallbackOverride } from "../../src/server/responses/main-hard-lock-routing"; +import { mainHardLockFallbackCallerCheck } from "../../src/server/responses/core-auth"; +import type { HandleResponsesOptions } from "../../src/server/responses/core-options"; import { resolveMainAccountHardLockFallback } from "../../src/codex/main-account-hard-lock"; -import { captureMainQuotaWriter, clearMainAccountInfoCache, observeMainQuotaIdentity } from "../../src/codex/main-account-cache"; +import { captureMainQuotaWriter, clearMainAccountInfoCache, observeMainQuotaCredential, observeMainQuotaIdentity } from "../../src/codex/main-account-cache"; import { clearAccountQuota, setAccountQuotaFromParsed, type StoredAccountQuota } from "../../src/codex/quota"; import { saveCodexAccountCredential } from "../../src/codex/account-store"; import { NATIVE_RESERVE_MODEL } from "../../src/codex/catalog/native-models"; @@ -57,7 +59,7 @@ function observe(quota: Omit): void { } function config(overrides: Partial = {}): OcxConfig { - return { providers: {}, codexAccounts: [], ...overrides } as OcxConfig; + return { providers: { deepseek: { adapter: "openai-responses", authMode: "key", baseUrl: "https://deepseek.example.test/v1" } }, codexAccounts: [], ...overrides } as OcxConfig; } function withPoolAccount(target: OcxConfig): OcxConfig { @@ -88,64 +90,164 @@ describe("main-account hard-lock fallback resolution", () => { }); }); -describe("applyMainHardLockFallbackOverride", () => { - test("a blocked lock with no eligible account rewrites the model and the effort", () => { +describe("Direct hard-lock fallback caller identity", () => { + const mainAccountId = "fallback-account-a"; + function tokenFor(accountId: string): string { + return `header.${Buffer.from(JSON.stringify({ + exp: Math.floor(Date.now() / 1000) + 3600, + "https://api.openai.com/auth": { chatgpt_account_id: accountId }, + })).toString("base64url")}.signature`; + } + const mainToken = tokenFor(mainAccountId); + const otherToken = tokenFor("fallback-account-b"); + function directConfig(): OcxConfig { + return config({ + defaultProvider: "openai", + autoSwitchThreshold: 0, + providers: { + openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", + baseUrl: "https://chatgpt.com/backend-api/codex" }, + rescue: { adapter: "openai-responses", authMode: "key", apiKey: "fixture-fallback-key", + baseUrl: "https://fallback.example/v1" }, + }, + codexMainAccountHardLockFallback: { model: "rescue/rescue-model" }, + } as Partial); + } + async function dispatch(callerToken: string, accountId: string, source?: "bearer" | "dedicated" | "x-api-key") { + const seen: Array<{ url: string; auth: string | null; model: string }> = []; + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")); + seen.push({ url: String(input), auth: new Headers(init?.headers).get("authorization"), model: body.model }); + return Response.json({ id: "resp_caller_fixture", status: "completed", model: body.model, + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "ok" }] }], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }); + }) as typeof fetch; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${callerToken}`, "chatgpt-account-id": accountId }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), directConfig(), { model: "", provider: "" }, source ? { + admission: { kind: "configured", keyId: "fixture-admission", source }, + } : {}); + expect(response.status).toBe(200); + await response.text(); + expect(seen).toHaveLength(1); + return seen[0]!; + } + beforeEach(() => { + observeMainQuotaCredential(mainToken, mainAccountId); + observe({ weeklyPercent: 100 }); + }); + test("another caller stays native while the observed main is blocked", async () => { + expect(await dispatch(otherToken, "fallback-account-b")).toEqual({ + url: "https://chatgpt.com/backend-api/codex/responses", auth: `Bearer ${otherToken}`, model: "gpt-5.3", + }); + }); + test("the same workspace with an unobserved bearer stays native", async () => { + const unobservedToken = mainToken.replace(".signature", ".other-signature"); + const seen = await dispatch(unobservedToken, mainAccountId); + expect(seen.url).toBe("https://chatgpt.com/backend-api/codex/responses"); + expect(seen.auth).toBe(`Bearer ${unobservedToken}`); + }); + test("a caller matching observed main uses the configured fallback", async () => { + expect(await dispatch(mainToken, mainAccountId)).toEqual({ + url: "https://fallback.example/v1/responses", auth: "Bearer fixture-fallback-key", model: "rescue-model", + }); + }); + test("proxy bearer admission preserves stored-main fallback without matching the proxy secret", async () => { + const seen = await dispatch("fixture-proxy-secret", "", "bearer"); + expect(seen.url).toBe("https://fallback.example/v1/responses"); + expect(seen.auth).toBe("Bearer fixture-fallback-key"); + }); + for (const source of ["dedicated", "x-api-key"] as const) { + test(`${source} admission preserves the separate caller's native credential`, async () => { + const seen = await dispatch(otherToken, "fallback-account-b", source); + expect(seen.url).toBe("https://chatgpt.com/backend-api/codex/responses"); + expect(seen.auth).toBe(`Bearer ${otherToken}`); + }); + } + test("an unblocked matching main stays native", async () => { + observe({ weeklyPercent: 10 }); + const seen = await dispatch(mainToken, mainAccountId); + expect(seen.url).toBe("https://chatgpt.com/backend-api/codex/responses"); + expect(seen.auth).toBe(`Bearer ${mainToken}`); + }); + test("a disabled original route keeps its normal 404 with fallback configured and hard lock off", async () => { + const cfg = directConfig(); + cfg.providers.openai!.disabled = true; + cfg.codexMainAccountHardLock = false; + let sends = 0; + globalThis.fetch = (async () => { sends++; throw new Error("disabled route must not send"); }) as typeof fetch; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json", authorization: `Bearer ${mainToken}`, + "chatgpt-account-id": mainAccountId }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), cfg, { model: "", provider: "" }); + expect(response.status).toBe(404); + const body = await response.json() as { error: { type: string } }; + expect(body.error.type).toBe("invalid_request_error"); + expect(sends).toBe(0); + }); +}); + +describe("applyMainHardLockFallbackOverride", async () => { + test("a blocked lock with no eligible account rewrites the model and the effort", async () => { observe({ weeklyPercent: 98 }); const body: Record = { model: "gpt-5.3", reasoning: { effort: "high" } }; - expect(applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))) + expect(await applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))) .toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); expect(body.model).toBe("deepseek/deepseek-chat"); expect(body.reasoning).toEqual({ effort: "low" }); }); - test("each blocking window governs on its own", () => { + test("each blocking window governs on its own", async () => { const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); observe({ shortPercent: 90 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ shortPercent: 0 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); observe({ weeklyPercent: 98 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ weeklyPercent: 0 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); }); - test("a monthly-only account governs on monthly; supplementary monthly data never blocks", () => { + test("a monthly-only account governs on monthly; supplementary monthly data never blocks", async () => { const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); // No short or weekly evidence at all: monthly is the account's only window. observe({ monthlyPercent: 99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ monthlyPercent: 50 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); // Supplementary monthly data on a 5h/weekly account never becomes a lock. observe({ shortPercent: 10, weeklyPercent: 10, monthlyPercent: 99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); }); - test("release is the same observation that admits: dropping below the threshold stops the rewrite", () => { + test("release is the same observation that admits: dropping below the threshold stops the rewrite", async () => { const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); observe({ weeklyPercent: 99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).not.toBeNull(); observe({ weeklyPercent: 97.99 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); }); - test("unknown usage, the opt-out, and a missing section all keep the refusal behavior", () => { - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); + test("unknown usage, the opt-out, and a missing section all keep the refusal behavior", async () => { + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); observe({ weeklyPercent: 100 }); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLock: false, codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); - expect(applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config())).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config({ codexMainAccountHardLock: false, codexMainAccountHardLockFallback: FALLBACK } as Partial))).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, config())).toBeNull(); }); - test("an eligible pool account keeps the request on Codex", () => { + test("an eligible pool account keeps the request on Codex", async () => { observe({ weeklyPercent: 100 }); const cfg = withPoolAccount(config({ codexMainAccountHardLockFallback: FALLBACK } as Partial)); const body: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); expect(body.model).toBe("gpt-5.3"); }); - test("a pool account in shared reset-derived cooldown or unused under Direct mode does not suppress the fallback", () => { + test("a pool account in shared reset-derived cooldown or unused under Direct mode does not suppress the fallback", async () => { observe({ weeklyPercent: 100 }); const cfg = withPoolAccount(config({ codexMainAccountHardLockFallback: FALLBACK } as Partial)); setScopedHealth("pool-a", "shared", { @@ -158,13 +260,14 @@ describe("applyMainHardLockFallbackOverride", () => { cooldownGeneration: 1, }); const cooledBody: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(cooledBody, cfg, now)) + expect(await applyMainHardLockFallbackOverride(cooledBody, cfg, now)) .toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); expect(cooledBody.model).toBe("deepseek/deepseek-chat"); deleteAllScopedHealth("pool-a"); const directCfg = withPoolAccount(config({ providers: { + deepseek: { adapter: "openai-responses", authMode: "key", baseUrl: "https://deepseek.example.test/v1" }, openai: { adapter: "openai-responses", authMode: "forward", @@ -175,11 +278,46 @@ describe("applyMainHardLockFallbackOverride", () => { codexMainAccountHardLockFallback: FALLBACK, } as Partial)); const directBody: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(directBody, directCfg, now)) + // Direct fails closed without the caller ownership proof supplied by request preparation. + expect(await applyMainHardLockFallbackOverride(directBody, directCfg, now)).toBeNull(); + expect(directBody.model).toBe("gpt-5.3"); + expect(await applyMainHardLockFallbackOverride(directBody, directCfg, now, async () => () => true)) .toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); }); - test("only bare Codex-native selectors qualify", () => { ++ test.each(["none", "identity", "bearer"] as const)("Direct ownership is rechecked after the async caller check (change: %s)", async change => { + observeMainQuotaCredential("fixture-caller-a", "fallback-account-a"); + observe({ weeklyPercent: 100 }); + const cfg = config({ + providers: { + deepseek: { adapter: "openai-responses", authMode: "key", baseUrl: "https://deepseek.example.test/v1" }, + openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + codexMainAccountHardLockFallback: FALLBACK, + } as Partial); + const req = new Request("http://127.0.0.1/v1/responses", { method: "POST", + headers: { authorization: "Bearer fixture-caller-a", "chatgpt-account-id": "fallback-account-a" } }); + const body = { model: "gpt-5.3", reasoning: { effort: "high" } }; + const pending = applyMainHardLockFallbackOverride(body, cfg, undefined, + mainHardLockFallbackCallerCheck(req, cfg, {} as HandleResponsesOptions)); + // The replacement is blocked too, so only the ownership recheck can refuse the reroute. + queueMicrotask(() => { + if (change === "identity") { + observeMainQuotaIdentity("fallback-account-b"); + observeMainQuotaCredential("fixture-caller-b", "fallback-account-b"); + setAccountQuotaFromParsed("__main__", { weeklyPercent: 100 }, undefined, captureMainQuotaWriter("fallback-account-b")!); + } else if (change === "bearer") observeMainQuotaCredential("fixture-caller-a-rotated", "fallback-account-a"); + }); + const result = await pending; + if (change === "none") { + expect(result).toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); + return; + } + expect(result).toBeNull(); + expect(body).toEqual({ model: "gpt-5.3", reasoning: { effort: "high" } }); + }); + + test("only bare Codex-native selectors qualify", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ codexMainAccountHardLockFallback: FALLBACK, @@ -194,40 +332,126 @@ describe("applyMainHardLockFallbackOverride", () => { } as Partial); for (const model of ["deepseek/deepseek-chat", "combo/x", "openai-codex/gpt-5.3", NATIVE_RESERVE_MODEL, "gpt-5.4"]) { const body: Record = { model }; - expect(applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); expect(body.model).toBe(model); } }); - test("a Codex-native fallback target never applies: the refusal is the honest answer", () => { + test("a Codex-native fallback target never applies: the refusal is the honest answer", async () => { observe({ weeklyPercent: 100 }); for (const target of ["gpt-5.4", "openai/gpt-5.4"]) { const body: Record = { model: "gpt-5.3" }; - expect(applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: { model: target } } as Partial))).toBeNull(); + expect(await applyMainHardLockFallbackOverride(body, config({ codexMainAccountHardLockFallback: { model: target } } as Partial))).toBeNull(); expect(body.model).toBe("gpt-5.3"); } }); - test("a fallback without an effort leaves the request's own reasoning untouched", () => { + test("a fallback without an effort leaves the request's own reasoning untouched", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ codexMainAccountHardLockFallback: { model: "deepseek/deepseek-chat" } } as Partial); const body: Record = { model: "gpt-5.3", reasoning: { effort: "high" } }; - expect(applyMainHardLockFallbackOverride(body, cfg)).toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); + expect(await applyMainHardLockFallbackOverride(body, cfg)).toEqual({ sourceModel: "gpt-5.3", targetModel: "deepseek/deepseek-chat" }); expect(body.model).toBe("deepseek/deepseek-chat"); expect(body.reasoning).toEqual({ effort: "high" }); }); - test("a body without a usable model selector is never touched", () => { + test("a body without a usable model selector is never touched", async () => { observe({ weeklyPercent: 100 }); const cfg = config({ codexMainAccountHardLockFallback: FALLBACK } as Partial); - expect(applyMainHardLockFallbackOverride(null, cfg)).toBeNull(); - expect(applyMainHardLockFallbackOverride([], cfg)).toBeNull(); - expect(applyMainHardLockFallbackOverride({ model: "" }, cfg)).toBeNull(); - expect(applyMainHardLockFallbackOverride({ model: 42 }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride(null, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride([], cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: "" }, cfg)).toBeNull(); + expect(await applyMainHardLockFallbackOverride({ model: 42 }, cfg)).toBeNull(); + }); + + test("unavailable chosen targets keep the blocked source and reasoning intact", async () => { + observe({ weeklyPercent: 100 }); + const gateway = { adapter: "openai-responses", authMode: "key", baseUrl: "https://gateway.example.test/v1" } as const; + for (const target of ["private/secret-model", "former/secret-model", "combo/removed", "policy/removed", "retired-account/gpt-5.3"]) { + for (const providers of [{ gateway }, {}]) { + const cfg = config({ providers, defaultProvider: "gateway", codexMainAccountHardLockFallback: { model: target, reasoningEffort: "low" } } as Partial); + const body = { model: "gpt-5.3", reasoning: { effort: "high" } }; + expect(await applyMainHardLockFallbackOverride(body, cfg)).toBeNull(); + expect(body).toEqual({ model: "gpt-5.3", reasoning: { effort: "high" } }); + } + } + const cfg = config({ providers: { gateway, private: { ...gateway, disabled: true } }, defaultProvider: "gateway", codexMainAccountHardLockFallback: { model: "private/secret-model" } } as Partial); + expect(await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg)).toBeNull(); + }); + + test("target admission is rechecked after provider removal, re-addition and alias changes", async () => { + observe({ weeklyPercent: 100 }); + const provider = { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1", alias: "chosen" } as const; + const cfg = config({ providers: { private: provider }, defaultProvider: "private", codexMainAccountHardLockFallback: { model: "chosen/secret-model" } } as Partial); + const apply = () => applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg); + expect(await apply()).not.toBeNull(); + delete cfg.providers.private; + expect(await apply()).toBeNull(); + cfg.providers.private = provider; + expect(await apply()).not.toBeNull(); + cfg.providers.private = { ...provider, alias: "renamed" }; + expect(await apply()).toBeNull(); + cfg.codexMainAccountHardLockFallback = { model: "private/secret-model" }; + expect(await apply()).not.toBeNull(); + }); + + test("bare fallback and explicitly declared slash model ids stay valid", async () => { + observe({ weeklyPercent: 100 }); + const provider = { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1", models: ["vendor/native-model"] } as const; + for (const model of ["bare-model", "vendor/native-model", "private/vendor/native-model"]) { + const cfg = config({ providers: { private: { ...provider, models: [...provider.models] } }, defaultProvider: "private", codexMainAccountHardLockFallback: { model } } as Partial); + expect((await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg))?.targetModel).toBe(model); + } + }); + + test("configured combo aliases, policy profiles and account namespaces keep their admission", async () => { + observe({ weeklyPercent: 100 }); + const cfg = config({ + providers: { private: { adapter: "openai-responses", authMode: "key", baseUrl: "https://chosen.example.test/v1" }, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" } }, + combos: { rescue: { alias: "chosen-combo", targets: [{ provider: "private", model: "secret-model" }] } }, + routingProfiles: { rescue: { alias: "chosen-policy", candidates: [{ provider: "private", model: "secret-model" }], require: { tools: true } } }, + codexAccountNamespaces: { pinned: "pool-a" }, + } as Partial); + for (const model of ["combo/rescue", "chosen-combo", "policy/rescue", "chosen-policy", "pinned/gpt-5.3"]) { + cfg.codexMainAccountHardLockFallback = { model }; + expect((await applyMainHardLockFallbackOverride({ model: "gpt-5.3" }, cfg))?.targetModel).toBe(model); + } + }); + + test("a stale qualified fallback never sends to the enabled default provider", async () => { + observeMainQuotaCredential("fixture-caller", "fallback-account-a"); + observe({ weeklyPercent: 100 }); + const cfg = config({ + providers: { + openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", baseUrl: "https://chatgpt.com/backend-api/codex" }, + gateway: { adapter: "openai-responses", authMode: "key", apiKey: "fixture-key", baseUrl: "https://gateway.example.test/v1" }, + }, + defaultProvider: "gateway", + codexMainAccountHardLockFallback: { model: "private/secret-model" }, + } as Partial); + let sends = 0; + const sentUrls: string[] = []; + globalThis.fetch = (async (input: unknown) => { + sends += 1; + sentUrls.push(String(input)); + return Response.json({ id: "resp_fixture", status: "completed", output: [] }); + }) as typeof fetch; + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer fixture-caller", "chatgpt-account-id": "fallback-account-a" }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "fixture" }), + }), cfg, { model: "", provider: "" }); + await response.text(); + expect(sentUrls).toEqual([]); + expect(sends).toBe(0); + expect(response.status).toBeGreaterThanOrEqual(400); }); test("handleResponses strips caller credentials, preserves requestedModel on combo fallback, and yields to shadow/compaction turns", async () => { observe({ weeklyPercent: 100 }); + // This caller is the observed main credential that actually owes the hard lock. + observeMainQuotaCredential("caller-chatgpt-token", "fallback-account-a"); const cfg = config({ defaultProvider: "gateway", providers: { @@ -284,7 +508,7 @@ describe("applyMainHardLockFallbackOverride", () => { "content-type": "application/json", "session_id": "lane-hard-lock-fallback", "authorization": "Bearer caller-chatgpt-token", - "chatgpt-account-id": "acct-caller", + "chatgpt-account-id": "fallback-account-a", }); const lane = sessionLaneIdFromRequest(headers); rememberComboForLane(lane, "prior", { provider: "gateway", model: "prior-model" }, "prior-model", captureConfigGeneration()); diff --git a/tests/codex-integration/main-account-hard-lock-recovery.test.ts b/tests/codex-integration/main-account-hard-lock-recovery.test.ts index 72f7bac7a25..9d22e29efd5 100644 --- a/tests/codex-integration/main-account-hard-lock-recovery.test.ts +++ b/tests/codex-integration/main-account-hard-lock-recovery.test.ts @@ -70,6 +70,13 @@ function usage(percent = 0): Response { } }); } +/** A metadata read that still reports the block with a future predicted reset. */ +function heldUsage(resetAt: number): Response { + return Response.json({ plan_type: "plus", rate_limit: { + primary_window: { used_percent: 99, limit_window_seconds: 18_000, reset_at: Math.floor(resetAt / 1000) }, + } }); +} + /** * Stub recovery HTTP calls, requiring a known metadata/token URL and an active native-main drain. * Return the captured URL list so tests can verify the requests made by background recovery. @@ -136,27 +143,156 @@ afterEach(async () => { }); describe("main hard-lock background recovery", () => { - test("known reset waits locally, then verifies recovery instead of unlocking by time", async () => { + test("known reset rechecks metadata on a capped schedule and never unlocks by time (#7312)", async () => { let now = Date.now(); const clock = spyOn(Date, "now").mockImplementation(() => now); try { - const deadline = now + 3_600_000; + const deadline = now + 6 * 3_600_000; setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, shortResetAt: deadline }, undefined, captureMainQuotaWriter(accountId)); - const calls = fetchWith(async () => usage(99)); - for (let tick = 0; tick < 60; tick++, now += 60_000) await runMainAccountHardLockRecovery(config()); + const readAt: number[] = []; + const calls = fetchWith(async (_url, init) => { + // Metadata only: a GET of WHAM usage, never inference or reset redemption. + expect(init?.method ?? "GET").toBe("GET"); + readAt.push(now); + return heldUsage(deadline); + }); + const start = now; + for (; now < deadline; now += 60_000) { + await runMainAccountHardLockRecovery(config()); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + } + expect(new Set(calls)).toEqual(new Set([whamUrl])); + // The block was just observed, so the first recheck waits one base interval. + expect(readAt[0]! - start).toBeGreaterThanOrEqual(5 * 60_000); + const gaps = readAt.slice(1).map((at, i) => at - readAt[i]!); + expect(gaps.every(gap => gap >= 5 * 60_000 && gap <= 60 * 60_000)).toBe(true); + expect(gaps).toEqual([...gaps].sort((a, b) => a - b)); + // Six hours of one-minute sweeps stay within the hourly cap plus the ramp. + expect(readAt.length).toBeGreaterThanOrEqual(6); + expect(readAt.length).toBeLessThanOrEqual(10); + // Reaching the predicted reset is not proof: a still-blocked read keeps the lock. + await runMainAccountHardLockRecovery(config()); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + now += 60 * 60_000; + const recovery = fetchWith(async () => usage(0)); + await runMainAccountHardLockRecovery(config()); + expect(recovery).toEqual([whamUrl]); + expect(getMainAccountHardLockStatus(config()).state).toBe("ready"); + } finally { clock.mockRestore(); } + }); + + test("an early reset redeemed elsewhere is learned from metadata before the predicted reset (#7312)", async () => { + let now = Date.now(); + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 5 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const calls = fetchWith(async () => usage(0)); + for (let tick = 0; tick < 5; tick++, now += 60_000) await runMainAccountHardLockRecovery(config()); expect(calls).toEqual([]); expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); await runMainAccountHardLockRecovery(config()); expect(calls).toEqual([whamUrl]); - now += 60_000; + expect(getMainAccountHardLockStatus(config()).state).toBe("ready"); + } finally { clock.mockRestore(); } + }); + + test.each(["paused", "needs-reauth"])("a %s main receives no predicted-reset rechecks (#7312)", async kind => { + let now = Date.now(); + const clock = spyOn(Date, "now").mockImplementation(() => now); + const cfg = { ...config(), ...(kind === "paused" ? { pausedCodexAccountIds: [MAIN] } : {}) }; + try { + const deadline = now + 3 * 3_600_000; + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: deadline }, undefined, captureMainQuotaWriter(accountId)); + if (kind === "needs-reauth") markAccountNeedsReauth(MAIN); + const calls = fetchWith(async () => usage(0)); + for (; now < deadline; now += 60_000) await runMainAccountHardLockRecovery(cfg); + expect(calls).toEqual([]); + expect(getMainAccountHardLockStatus(cfg).state).toBe("blocked"); + } finally { clock.mockRestore(); } + }); + ++ test("a predicted reset that moves earlier pulls the next hold read in, never under one base interval (#7312)", async () => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 6 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push((now - start) / 60_000); return heldUsage(start + 6 * 3_600_000); }); + for (; now - start <= 6 * 60_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5]); + // Same-identity evidence now predicts the reset at minute 8; the old reservation was minute 15. + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: start + 8 * 60_000 }, undefined, captureMainQuotaWriter(accountId)); + const recovered = fetchWith(async () => { readAt.push((now - start) / 60_000); return usage(0); }); + for (; now - start <= 12 * 60_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5, 10]); + expect(recovered).toEqual([whamUrl]); + expect(getMainAccountHardLockStatus(config()).state).toBe("ready"); + } finally { clock.mockRestore(); } + }); + ++ test("a revised reset that expires between sweeps still releases the old reservation (#7312)", async () => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 6 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push((now - start) / 60_000); return heldUsage(start + 6 * 3_600_000); }); await runMainAccountHardLockRecovery(config()); - expect(calls).toHaveLength(1); - now += 240_000; - const recovery = fetchWith(async () => usage(0)); + now = start + 5 * 60_000; await runMainAccountHardLockRecovery(config()); - expect(recovery).toEqual([whamUrl]); - expect(getMainAccountHardLockStatus(config()).state).toBe("ready"); + expect(readAt).toEqual([5]); + // Revised to minute 8 at minute 7, with no sweep until after that boundary has passed. + now = start + 7 * 60_000; + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: start + 8 * 60_000 }, undefined, captureMainQuotaWriter(accountId)); + now = start + 9 * 60_000; + await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5]); + now = start + 10 * 60_000; + await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5, 10]); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + } finally { clock.mockRestore(); } + }); + + test("a moving near-term reset keeps hold reads at least one base interval apart (#7312)", async () => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 2 * 60_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push(now - start); return heldUsage(now + 2 * 60_000); }); + for (; now - start < 3_600_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + const gaps = readAt.slice(1).map((at, i) => at - readAt[i]!); + expect(readAt.length).toBeGreaterThan(1); + expect(gaps.every(gap => gap >= 5 * 60_000)).toBe(true); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); + } finally { clock.mockRestore(); } + }); + + test.each([401, 403])("a nonterminal HTTP %s during a predicted-reset hold retries on the next sweep (#7312)", async status => { + let now = Date.now(); + const start = now; + const clock = spyOn(Date, "now").mockImplementation(() => now); + try { + setAccountQuotaFromParsed(MAIN, { shortPercent: 99, shortWindowSeconds: 18_000, + shortResetAt: now + 6 * 3_600_000 }, undefined, captureMainQuotaWriter(accountId)); + const readAt: number[] = []; + fetchWith(async () => { readAt.push((now - start) / 60_000); return Response.json({}, { status }); }); + for (; now - start <= 6 * 60_000; now += 60_000) await runMainAccountHardLockRecovery(config()); + expect(readAt).toEqual([5, 6]); + expect(isAccountNeedsReauth(MAIN)).toBe(false); + expect(getMainAccountHardLockStatus(config()).state).toBe("blocked"); } finally { clock.mockRestore(); } }); diff --git a/tests/codex-integration/main-account-policy-binding-wait.test.ts b/tests/codex-integration/main-account-policy-binding-wait.test.ts index 3e5c0d61fa5..67d8c7a8f89 100644 --- a/tests/codex-integration/main-account-policy-binding-wait.test.ts +++ b/tests/codex-integration/main-account-policy-binding-wait.test.ts @@ -22,6 +22,8 @@ import { clearAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/qu import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; import type { OcxConfig } from "../../src/types"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { handleResponses } from "../../src/server/responses"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; /** * The admission fence during an owned startup's main-policy binding (#5694 made it default-on). @@ -44,6 +46,7 @@ const previousHome = process.env.OPENCODEX_HOME; const previousCodexHome = process.env.CODEX_HOME; let releases: Array<() => Promise> = []; let openGates: Array<() => void> = []; +const originalFetch = globalThis.fetch; function bearerFor(id: string): string { const payload = Buffer.from(JSON.stringify({ @@ -162,6 +165,7 @@ beforeEach(() => { }); afterEach(async () => { + globalThis.fetch = originalFetch; // A case may end with the hold still up (the deadline case has to). Open every gate first: // `release` below awaits convergence, which cannot finish behind a closed barrier. for (const open of openGates.splice(0)) open(); @@ -178,6 +182,93 @@ afterEach(async () => { }); describe("a caller-owned request waits out the startup policy binding", () => { + for (const source of ["loopback", "dedicated"] as const) { + test(`handleResponses missing caller bearer keeps immediate 401 during binding under ${source} admission`, async () => { + const startup = await holdStartup(); + const cfg = directConfig(); + cfg.codexMainAccountHardLockFallback = { model: "rescue/rescue-model" }; + cfg.providers.rescue = { adapter: "openai-responses", authMode: "key", apiKey: "fixture-rescue", + baseUrl: "https://fallback.example/v1" }; + let sends = 0; + globalThis.fetch = (async () => { sends++; throw new Error("missing caller must not send"); }) as typeof fetch; + const request = handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), cfg, { model: "", provider: "" }, { + admission: source === "loopback" ? { kind: "loopback", source } + : { kind: "configured", keyId: "fixture-admission", source }, + }); + const settled = trackSettlement(request); + const outcome = request.then(response => ({ response }), error => ({ error })); + try { + await Bun.sleep(25); + expect(startup.pending()).toBe(true); + expect(settled()).toBe(true); + const result = await outcome; + if (!("response" in result)) throw result.error; + expect(result.response.status).toBe(401); + await result.response.text(); + expect(sends).toBe(0); + } finally { + startup.release(); + const result = await outcome; + if ("response" in result && !result.response.bodyUsed && !result.response.body?.locked) { + await result.response.body?.cancel(); + } + } + }); + } + + for (const [initialUsage, releaseBlocked] of [[100, true], [100, false], [undefined, true]] as const) { + test(`handleResponses rechecks fallback after binding settles from ${initialUsage ?? "unknown"} with blocked=${releaseBlocked}`, async () => { + const startup = await holdStartup(); + observeMainQuotaIdentity(accountId); + observeMainQuotaCredential(token, accountId); + const writer = captureMainQuotaWriter(accountId)!; + if (initialUsage !== undefined) setAccountQuotaFromParsed(MAIN, { weeklyPercent: initialUsage }, undefined, writer); + const cfg = directConfig(); + cfg.providers.rescue = { adapter: "openai-responses", authMode: "key", apiKey: "fixture-rescue", + baseUrl: "https://fallback.example/v1" }; + cfg.codexMainAccountHardLockFallback = { model: "rescue/rescue-model" }; + const seen: string[] = []; + const releaseSpend = acquireOwnedSpendHome(); + globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + seen.push(String(input)); + const body = JSON.parse(String(init?.body ?? "{}")); + return Response.json({ id: "resp_binding_fixture", status: "completed", model: body.model, + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "ok" }] }], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }); + }) as typeof fetch; + const request = handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: callerHeaders(), + body: JSON.stringify({ model: "gpt-5.3", stream: false, input: "hello" }), + }), cfg, { model: "", provider: "" }); + // Observe rejection too, so a failed assertion never leaves an unhandled request. + const outcome = request.then(response => ({ response }), error => ({ error })); + try { + await Bun.sleep(25); + expect(startup.pending()).toBe(true); + expect(seen).toEqual([]); + setAccountQuotaFromParsed(MAIN, { weeklyPercent: releaseBlocked ? 100 : 10 }, undefined, writer); + startup.release(); + const result = await outcome; + if (!("response" in result)) throw result.error; + expect(result.response.status).toBe(200); + await result.response.text(); + expect(seen).toEqual([releaseBlocked + ? "https://fallback.example/v1/responses" : "https://chatgpt.com/backend-api/codex/responses"]); + } finally { + startup.release(); + try { + const result = await outcome; + if ("response" in result && !result.response.bodyUsed && !result.response.body?.locked) { + await result.response.body?.cancel(); + } + } finally { releaseSpend(); } + } + }); + } + test("default config waits for convergence and then resolves an auth context", async () => { const startup = await holdStartup(); expect(startup.pending()).toBe(true);