From b64f0a3054cc0ee841632a285a230bef943f36bc Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 18 Oct 2026 04:07:08 +0800 Subject: [PATCH] fix(codex): refresh expired pool credits before account selection --- .../docs/reference/codex-retry-diagnostics.md | 11 +++ src/codex/account-credit-use.ts | 15 ++++ src/codex/auth-api/pool-mode-gate.ts | 19 ++++- src/codex/auth-api/pool-quota-probe.ts | 4 +- src/codex/auth-context.ts | 15 +++- src/codex/quota-types.ts | 16 +++- structure/providers/openai-accounts.md | 5 +- structure/providers/openai-tiers.md | 9 ++ .../codex-auth-context.test.ts | 17 ++++ .../codex-quota-prime.test.ts | 85 +++++++++++++++++++ 10 files changed, 184 insertions(+), 12 deletions(-) diff --git a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md index d2e2ee8c2e3..98a759fac3f 100644 --- a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md +++ b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md @@ -23,6 +23,17 @@ waiting visible: those items can contaminate history or imply work happened. Do not shorten the provider delay or provoke an extra request to advance the client's retry counter. +## Credits after the included limit + +An account at its included usage limit can continue only when **Use credits** is enabled for +that account and its spendable balance has a fresh observation. Usage headers and balance +observations have separate clocks. Before Pool selection, OpenCodex refreshes expired or missing +balance evidence for opted-in exhausted accounts, even if ordinary usage was just updated. +The refresh reads usage metadata; it does not run inference or redeem reset credits. Failed +observations stay blocked and are paced, rather than extending old balance evidence. A paused +main account is not a Pool fallback for its own forwarded credential. Enabling the main account's +credits is not required to refresh another account's balance. + ## Triage: `exceeded retry limit, last status: 429 Too Many Requests` The message is the client's retry budget summary, not a diagnosis. One refusal diff --git a/src/codex/account-credit-use.ts b/src/codex/account-credit-use.ts index 1c7e256694f..c75641eb0a4 100644 --- a/src/codex/account-credit-use.ts +++ b/src/codex/account-credit-use.ts @@ -6,6 +6,7 @@ import { TERMINAL_SHORT_WINDOW_FRESHNESS_MS, isTerminalShortWindow, hasSpendableCodexCredits, + hasFreshCodexCreditObservation, resetAtToMs, type StoredAccountQuota, } from "./quota-types"; @@ -105,3 +106,17 @@ export function isCodexAccountHeldForCredits( return !(codexAccountUsesCreditsAfterLimit(config, accountId) && hasSpendableCodexCredits(quota, now)) && isCodexUsageLimitReached(quota, plan, now); } + +/** Only opted-in, exhausted accounts need a balance observation before selection. */ +export function needsCodexCreditRefresh( + config: Pick, + accountId: string, + quota: StoredAccountQuota | null, + plan: unknown, + now = Date.now(), +): boolean { + return codexAccountUsesCreditsAfterLimit(config, accountId) + && !config.pausedCodexAccountIds?.includes(accountId) + && isCodexUsageLimitReached(quota, plan, now) + && !hasFreshCodexCreditObservation(quota, now); +} diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index 92b02e0ed2f..a385971eeeb 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -1,4 +1,4 @@ -import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt } from "../account-credit-use"; +import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt, needsCodexCreditRefresh } from "../account-credit-use"; import { nextCodexUsageQueryAt, nextQuotaQueryDelay, pruneRemovedCodexPoolUsageAccounts } from "../quota-query-backoff"; import { CODEX_PRIORITY_FAILBACK_REFRESH_MS } from "../account-priority"; import { codexQuotaHasFreshUsage } from "../quota-observation-freshness"; @@ -191,6 +191,8 @@ export function registerCodexCooldownRecoveryProbeWorker(config: OcxConfig): voi } export interface PrimeCodexPoolQuotasOptions { + /** An exact-account credit refresh must not inspect siblings or native main. */ + creditAccountId?: string; /** Test seams for proving fenced/recovery priming performs no native-main work. */ reconcileMainAccount?: typeof reconcileMainCodexAccountRuntimeState; readMainTokens?: typeof readCodexTokens; @@ -265,7 +267,15 @@ export async function primeCodexPoolQuotas( primeInFlight = trackBackgroundWork(`codex-pool-quota-prime:${reason}`, (async () => { const pool = (runtimeConfig.codexAccounts ?? []).filter(isSelectableCodexPoolAccount); const stale = pool.filter(a => { + if (reason === "credits-refresh" && options.creditAccountId !== undefined && a.id !== options.creditAccountId) return false; const q = getAccountQuota(a.id); + const creditStale = needsCodexCreditRefresh(runtimeConfig, a.id, q, a.plan); + if (creditStale) { + const last = poolQuotaPrimeAttemptedAt.get(a.id); + return !last || last.generation !== readCodexAccountRecord(a.id)?.generation + || Date.now() - last.at >= 60_000; + } + if (reason === "credits-refresh") return false; const observationStale = reason === "priority-failback" && q && !codexQuotaHasFreshUsage(q, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS); if (q && !observationStale) return Date.now() - q.updatedAt >= POOL_CACHE_TTL; @@ -308,14 +318,15 @@ export async function primeCodexPoolQuotas( }; try { await Promise.allSettled([ - primeMain(), + reason === "credits-refresh" ? Promise.resolve() : primeMain(), mapWithConcurrency(stale, POOL_QUOTA_REFRESH_CONCURRENCY, async a => { if (!getCodexAccountCredential(a.id)) return; let result: PoolQuotaResult; try { const quota = getAccountQuota(a.id); - const bypassCachedQuota = reason === "priority-failback" && quota !== null - && !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS); + const bypassCachedQuota = needsCodexCreditRefresh(runtimeConfig, a.id, quota, a.plan) + || (reason === "priority-failback" && quota !== null + && !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS)); result = await fetchPoolAccountQuota(a.id, bypassCachedQuota, a.plan, getValidPoolTokenForPrime); } catch (error) { // Local quota-flight saturation proves no WHAM request existed for this account. diff --git a/src/codex/auth-api/pool-quota-probe.ts b/src/codex/auth-api/pool-quota-probe.ts index 55d30da198c..671eceb1b58 100644 --- a/src/codex/auth-api/pool-quota-probe.ts +++ b/src/codex/auth-api/pool-quota-probe.ts @@ -1,3 +1,4 @@ +import { hasFreshCodexCreditObservation } from "../quota-types"; import { parseCodexCredits, rememberCodexCredits } from "../credits"; import { fetchCodexUsage } from "../quota-query-backoff"; import type { CodexUsageOwner } from "../quota-query-backoff"; @@ -498,7 +499,8 @@ export async function fetchPoolAccountQuota( explicitRefresh = false, ): Promise { const existing = getAccountQuota(accountId); - if (afterDispatchSequence === undefined && !forceRefresh && existing && Date.now() - existing.updatedAt < POOL_CACHE_TTL) { + if (afterDispatchSequence === undefined && !forceRefresh && existing && Date.now() - existing.updatedAt < POOL_CACHE_TTL + && (!existing.credits || hasFreshCodexCreditObservation(existing))) { return { quota: existing, needsReauth: false, diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index fbe8205f283..54ec61e4c5f 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1,4 +1,5 @@ import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt } from "./pool-credit-policy"; +import { needsCodexCreditRefresh } from "./account-credit-use"; import { hasSpendableCodexCredits } from "./quota-types"; import { nextQuotaQueryDelay } from "./quota-query-backoff"; import { noteMainAccountActivity } from "./main-account-external-usage"; @@ -1027,7 +1028,7 @@ export interface ResolveCodexAuthContextOptions { signal?: AbortSignal; /** Test seam: overrides the bounded startup policy-binding wait before the hard-lock fence. */ mainAccountPolicyBindingWaitMs?: number; - primeCodexPoolQuotas?: (config: OcxConfig, reason: string) => Promise; + primeCodexPoolQuotas?: (config: OcxConfig, reason: string, options?: { creditAccountId?: string }) => Promise; /** Test seam for account-gated native model discovery. */ resolveCodexModelEntitlements?: typeof resolveCodexModelEntitlements; /** Direct requests admitted with a proxy bearer substitute the stored native-main credential. */ @@ -1259,6 +1260,16 @@ async function resolveCodexAuthContextUnannotated( || (requestScopedMainCredential && fixedAccountId === MAIN_CODEX_ACCOUNT_ID)) { return resolveCallerOwnedMainContext(); } + // Usage headers cannot renew the balance clock. Observe expired credit evidence before + // selecting an account, rather than falling back to caller main while the pool is funded. + if ((config.codexAccounts ?? []).some(a => !a.isMain + && (fixedAccountId === undefined || fixedAccountId === a.id) + && needsCodexCreditRefresh(policy, a.id, getAccountQuota(a.id), a.plan))) { + options.signal?.throwIfAborted(); + const prime = options.primeCodexPoolQuotas ?? (await import("./auth-api")).primeCodexPoolQuotas; + await prime(config, "credits-refresh", { creditAccountId: fixedAccountId }).catch(() => {}); + options.signal?.throwIfAborted(); + } // A caller bearer can still accompany a request that selects a configured Pool account. Do not // let that request read, delete, or create a file-main affinity binding while deciding whether a // stored account is available; only the stored credential selected below may own Pool state. @@ -1427,6 +1438,7 @@ async function resolveCodexAuthContextUnannotated( if ( requestScopedMainCredential && fixedAccountId === undefined + && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID ) { return await resolveCallerOwnedMainContext(); @@ -1585,6 +1597,7 @@ async function resolveCodexAuthContextUnannotated( // alternate is eligible. A validated caller may still serve this request, // just as it can after an upstream rejection, without changing Pool state. if (requestScopedMainCredential && fixedAccountId === undefined + && !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID)) && options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID && !callerIsCooledPoolAccount(headers, config, accountId)) { return await resolveCallerOwnedMainContext(); diff --git a/src/codex/quota-types.ts b/src/codex/quota-types.ts index d3854172f8f..7ee5b3c2aac 100644 --- a/src/codex/quota-types.ts +++ b/src/codex/quota-types.ts @@ -35,15 +35,23 @@ export interface CodexSpendableCredits { observedAt: number; } +/** Freshness is independent of a positive/negative balance verdict and of usage headers. */ +export function hasFreshCodexCreditObservation( + quota: Pick | null, + now = Date.now(), +): boolean { + const observedAt = quota?.credits?.observedAt; + return typeof observedAt === "number" && Number.isFinite(observedAt) + && now >= observedAt && now - observedAt <= CODEX_CREDITS_FRESHNESS_MS; +} + export function hasSpendableCodexCredits( quota: Pick | null, now = Date.now(), ): boolean { const credits = quota?.credits; - if (!credits || !Number.isFinite(credits.observedAt)) return false; - const age = now - credits.observedAt; - if (age < 0 || age > CODEX_CREDITS_FRESHNESS_MS - || credits.allowed === false || credits.overageLimitReached === true || credits.hasCredits === false) return false; + if (!credits || !hasFreshCodexCreditObservation(quota, now)) return false; + if (credits.allowed === false || credits.overageLimitReached === true || credits.hasCredits === false) return false; return credits.unlimited === true || (credits.hasCredits === true && typeof credits.balance === "number" && Number.isFinite(credits.balance) && credits.balance > 0); } diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index bce47e46292..aa173f102bd 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -534,8 +534,9 @@ owns the five-minute cadence; `auth-api/pool-mode-gate.ts` bounds request-trigge including failures, while preserving main-owner claims and per-credential WHAM dispatch backoff. No requests means no new polling. For this reason only, stale observation proof bypasses aggregate quota-cache freshness after the existing attempt backoff. Main refresh keeps its owned lease and -passive intent: cache bypass does not clear an inference reauthentication mark. Other prime reasons -retain their existing cache rules. The split config schema degrades malformed optional values to +passive intent: cache bypass does not clear an inference reauthentication mark. Credit-expiry priming follows the independent balance clock described in the +[spendable credit contract](openai-tiers.md#spendable-codex-credits), bypasses fresh usage-only +caches, and performs no native-main discovery. Other prime reasons retain their existing cache rules. The split config schema degrades malformed optional values to false. Exact-account and Direct routes are unchanged. ## Main-account policy observations diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 9631ceab68e..22b0cb7d93e 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -298,6 +298,15 @@ refusal, so the two predicates can never both hold. ### Spendable Codex credits +Pool selection awaits a paced metadata refresh when an opted-in, unpaused account has an +exhausted included window and its balance observation is absent or older than five minutes. +Fresh usage headers do not renew that balance clock or suppress the refresh. Concurrent refreshes +share the existing prime flight; a dispatched unsuccessful observation waits at least one minute +before another request-triggered attempt for the same credential generation. No inference or reset +redemption is performed, and stale funds remain inadmissible if observation fails. Pool fallback +never uses an observed native-main caller credential while that main account is paused; unrelated +caller-owned Direct credentials retain their existing contract. + WHAM `credits` is spendable usage capacity, separate from manually redeemed `rate_limit_reset_credits`. The quota store keeps normalized flags, a finite nonnegative balance, and a separate observation clock. Explicit zero/null replaces earlier evidence; partial usage diff --git a/tests/codex-integration/codex-auth-context.test.ts b/tests/codex-integration/codex-auth-context.test.ts index 9f1244eb43e..cdd00a433db 100644 --- a/tests/codex-integration/codex-auth-context.test.ts +++ b/tests/codex-integration/codex-auth-context.test.ts @@ -2431,3 +2431,20 @@ test("context Direct bearer admission uses real stored-main materialization and globalThis.fetch = originalFetch; } }); + +test("pool exhaustion never falls back to an observed paused main caller", async () => { + const cfg = config(); + cfg.codexAccounts = []; + cfg.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID]; + cfg.codexMainAccountHardLock = false; + observeMainQuotaIdentity("paused-main-account"); + observeMainQuotaCredential("paused-main-bearer", "paused-main-account"); + const headers = new Headers({ authorization: "Bearer paused-main-bearer", "chatgpt-account-id": "paused-main-account" }); + await expect(resolveCodexAuthContext(headers, cfg, "pool", { + requestScopedMainCredential: true, + })).rejects.toBeInstanceOf(CodexMainProfileDrainingError); + // Direct remains caller-owned; the pause applies to Pool fallback, not unrelated callers. + expect(await resolveCodexAuthContext(headers, cfg, "direct", { + requestScopedMainCredential: true, + })).toMatchObject({ kind: "main", accountId: null }); +}); diff --git a/tests/codex-integration/codex-quota-prime.test.ts b/tests/codex-integration/codex-quota-prime.test.ts index 75bd5fbfd12..f82fe47ded7 100644 --- a/tests/codex-integration/codex-quota-prime.test.ts +++ b/tests/codex-integration/codex-quota-prime.test.ts @@ -21,6 +21,8 @@ import { readCodexAccountRecord, saveCodexAccountCredential, } from "../../src/codex/account-store"; +import { resolveCodexAuthContext } from "../../src/codex/auth-context"; +import { fetchPoolAccountQuota } from "../../src/codex/auth-api/pool-quota-probe"; import { resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle"; import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/main-account"; import { @@ -142,6 +144,89 @@ describe("primeCodexPoolQuotas", () => { if (existsSync(TEST_DIR)) removeTreeWithRetry(TEST_DIR); }); + test.each(["prime", "selection", "listing", "exact"])( + "%s refreshes expired credits even when usage was just observed", async path => { + const cfg = makeConfig({ creditCodexAccountIds: ["spender"], pausedCodexAccountIds: [MAIN_CODEX_ACCOUNT_ID], + autoSwitchThreshold: 0, activeCodexAccountId: "spender" }); + seedPoolAccount(cfg, "spender", "pro"); + const reset = Date.now() + 86_400_000; + setAccountQuotaFromParsed("spender", { weeklyPercent: 100, weeklyResetAt: reset, + credits: { hasCredits: true, balance: 20, observedAt: Date.now() - 300_001 } }); + updateAccountQuota("spender", 100, reset); + let reads = 0; + const originalFetch = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL) => { + expect(String(input)).toContain("/wham/usage"); + reads += 1; + return Response.json({ rate_limit: { secondary_window: { + used_percent: 100, limit_window_seconds: 604_800, reset_at: Math.floor(reset / 1000), + } }, credits: { has_credits: true, balance: "20" } }); + }) as typeof fetch; + try { + if (path === "selection" || path === "exact") { + expect(await resolveCodexAuthContext(new Headers(), cfg, "pool", { + modelId: "gpt-5.5", ...(path === "exact" ? { accountId: "spender" } : {}), + })) + .toMatchObject({ kind: "pool", accountId: "spender" }); + } else if (path === "listing") await fetchPoolAccountQuota("spender"); + else await primeCodexPoolQuotas(cfg, "credits-refresh"); + expect(reads).toBe(1); + expect(getAccountQuota("spender")?.credits?.observedAt).toBeGreaterThan(Date.now() - 5_000); + await primeCodexPoolQuotas(cfg, "credits-refresh"); + expect(reads).toBe(1); + } finally { globalThis.fetch = originalFetch; } + }, + ); + + test("fresh zero balances, paused accounts and credit opt-outs do not trigger credit discovery", async () => { + const cfg = makeConfig({ creditCodexAccountIds: ["empty", "paused"], pausedCodexAccountIds: ["paused"] }); + for (const id of ["empty", "paused", "optout"]) { + seedPoolAccount(cfg, id, "pro"); + setAccountQuotaFromParsed(id, { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000, + credits: { hasCredits: id !== "empty", balance: id === "empty" ? 0 : 20, + observedAt: Date.now() - (id === "empty" ? 0 : 300_001) } }); + } + const originalFetch = globalThis.fetch; + let reads = 0; + globalThis.fetch = (async () => { reads += 1; throw new Error("credit discovery must not run"); }) as typeof fetch; + try { + await primeCodexPoolQuotas(cfg, "credits-refresh"); + expect(reads).toBe(0); + expect(previewCodexAccountForRequest("no-permission", cfg)).toBeNull(); + } finally { globalThis.fetch = originalFetch; } + }); + + test("cancelled selection performs no credit observation", async () => { + const cfg = makeConfig({ creditCodexAccountIds: ["spender"] }); + seedPoolAccount(cfg, "spender", "pro"); + setAccountQuotaFromParsed("spender", { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000 }); + const controller = new AbortController(); + controller.abort(new Error("cancelled-before-observation")); + let primes = 0; + await expect(resolveCodexAuthContext(new Headers(), cfg, "pool", { + signal: controller.signal, primeCodexPoolQuotas: async () => { primes += 1; }, + })).rejects.toThrow("cancelled-before-observation"); + expect(primes).toBe(0); + }); + + test("failed credit observations are paced and never make stale funds spendable", async () => { + const cfg = makeConfig({ creditCodexAccountIds: ["spender"], pausedCodexAccountIds: [MAIN_CODEX_ACCOUNT_ID] }); + seedPoolAccount(cfg, "spender", "pro"); + setAccountQuotaFromParsed("spender", { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000, + credits: { hasCredits: true, balance: 20, observedAt: Date.now() - 300_001 } }); + const oldClock = getAccountQuota("spender")!.credits!.observedAt; + const originalFetch = globalThis.fetch; + let reads = 0; + globalThis.fetch = (async () => { reads += 1; return new Response("unavailable", { status: 503 }); }) as typeof fetch; + try { + await primeCodexPoolQuotas(cfg, "credits-refresh"); + await primeCodexPoolQuotas(cfg, "credits-refresh"); + expect(reads).toBe(1); + expect(getAccountQuota("spender")?.credits?.observedAt).toBe(oldClock); + expect(previewCodexAccountForRequest("still-held", cfg)).toBeNull(); + } finally { globalThis.fetch = originalFetch; } + }); + test("startup may observe pending account quota without inference or validation", async () => { const config = makeConfig(); seedPoolAccount(config, "pending", "pro");