Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs-site/src/content/docs/reference/codex-retry-diagnostics.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,17 @@ waiting visible: those items can contaminate history or imply work happened.
Do not shorten the provider delay or provoke an extra request to advance the
client's retry counter.

## Credits after the included limit

An account at its included usage limit can continue only when **Use credits** is enabled for
that account and its spendable balance has a fresh observation. Usage headers and balance
observations have separate clocks. Before Pool selection, OpenCodex refreshes expired or missing
balance evidence for opted-in exhausted accounts, even if ordinary usage was just updated.
The refresh reads usage metadata; it does not run inference or redeem reset credits. Failed
observations stay blocked and are paced, rather than extending old balance evidence. A paused
main account is not a Pool fallback for its own forwarded credential. Enabling the main account's
credits is not required to refresh another account's balance.

## Triage: `exceeded retry limit, last status: 429 Too Many Requests`

The message is the client's retry budget summary, not a diagnosis. One refusal
Expand Down
15 changes: 15 additions & 0 deletions src/codex/account-credit-use.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
TERMINAL_SHORT_WINDOW_FRESHNESS_MS,
isTerminalShortWindow,
hasSpendableCodexCredits,
hasFreshCodexCreditObservation,
resetAtToMs,
type StoredAccountQuota,
} from "./quota-types";
Expand Down Expand Up @@ -105,3 +106,17 @@ export function isCodexAccountHeldForCredits(
return !(codexAccountUsesCreditsAfterLimit(config, accountId) && hasSpendableCodexCredits(quota, now))
&& isCodexUsageLimitReached(quota, plan, now);
}

/** Only opted-in, exhausted accounts need a balance observation before selection. */
export function needsCodexCreditRefresh(
config: Pick<OcxConfig, "creditCodexAccountIds" | "pausedCodexAccountIds">,
accountId: string,
quota: StoredAccountQuota | null,
plan: unknown,
now = Date.now(),
): boolean {
return codexAccountUsesCreditsAfterLimit(config, accountId)
&& !config.pausedCodexAccountIds?.includes(accountId)
&& isCodexUsageLimitReached(quota, plan, now)
&& !hasFreshCodexCreditObservation(quota, now);
}
19 changes: 15 additions & 4 deletions src/codex/auth-api/pool-mode-gate.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt } from "../account-credit-use";
import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt, needsCodexCreditRefresh } from "../account-credit-use";
import { nextCodexUsageQueryAt, nextQuotaQueryDelay, pruneRemovedCodexPoolUsageAccounts } from "../quota-query-backoff";
import { CODEX_PRIORITY_FAILBACK_REFRESH_MS } from "../account-priority";
import { codexQuotaHasFreshUsage } from "../quota-observation-freshness";
Expand Down Expand Up @@ -191,6 +191,8 @@ export function registerCodexCooldownRecoveryProbeWorker(config: OcxConfig): voi
}

export interface PrimeCodexPoolQuotasOptions {
/** An exact-account credit refresh must not inspect siblings or native main. */
creditAccountId?: string;
/** Test seams for proving fenced/recovery priming performs no native-main work. */
reconcileMainAccount?: typeof reconcileMainCodexAccountRuntimeState;
readMainTokens?: typeof readCodexTokens;
Expand Down Expand Up @@ -265,7 +267,15 @@ export async function primeCodexPoolQuotas(
primeInFlight = trackBackgroundWork(`codex-pool-quota-prime:${reason}`, (async () => {
const pool = (runtimeConfig.codexAccounts ?? []).filter(isSelectableCodexPoolAccount);
const stale = pool.filter(a => {
if (reason === "credits-refresh" && options.creditAccountId !== undefined && a.id !== options.creditAccountId) return false;
const q = getAccountQuota(a.id);
const creditStale = needsCodexCreditRefresh(runtimeConfig, a.id, q, a.plan);
if (creditStale) {
const last = poolQuotaPrimeAttemptedAt.get(a.id);
return !last || last.generation !== readCodexAccountRecord(a.id)?.generation
|| Date.now() - last.at >= 60_000;
}
if (reason === "credits-refresh") return false;
const observationStale = reason === "priority-failback" && q
&& !codexQuotaHasFreshUsage(q, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS);
if (q && !observationStale) return Date.now() - q.updatedAt >= POOL_CACHE_TTL;
Expand Down Expand Up @@ -308,14 +318,15 @@ export async function primeCodexPoolQuotas(
};
try {
await Promise.allSettled([
primeMain(),
reason === "credits-refresh" ? Promise.resolve() : primeMain(),
mapWithConcurrency(stale, POOL_QUOTA_REFRESH_CONCURRENCY, async a => {
if (!getCodexAccountCredential(a.id)) return;
let result: PoolQuotaResult;
try {
const quota = getAccountQuota(a.id);
const bypassCachedQuota = reason === "priority-failback" && quota !== null
&& !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS);
const bypassCachedQuota = needsCodexCreditRefresh(runtimeConfig, a.id, quota, a.plan)
|| (reason === "priority-failback" && quota !== null
&& !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS));
result = await fetchPoolAccountQuota(a.id, bypassCachedQuota, a.plan, getValidPoolTokenForPrime);
} catch (error) {
// Local quota-flight saturation proves no WHAM request existed for this account.
Expand Down
4 changes: 3 additions & 1 deletion src/codex/auth-api/pool-quota-probe.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { hasFreshCodexCreditObservation } from "../quota-types";
import { parseCodexCredits, rememberCodexCredits } from "../credits";
import { fetchCodexUsage } from "../quota-query-backoff";
import type { CodexUsageOwner } from "../quota-query-backoff";
Expand Down Expand Up @@ -498,7 +499,8 @@ export async function fetchPoolAccountQuota(
explicitRefresh = false,
): Promise<PoolQuotaResult> {
const existing = getAccountQuota(accountId);
if (afterDispatchSequence === undefined && !forceRefresh && existing && Date.now() - existing.updatedAt < POOL_CACHE_TTL) {
if (afterDispatchSequence === undefined && !forceRefresh && existing && Date.now() - existing.updatedAt < POOL_CACHE_TTL
&& (!existing.credits || hasFreshCodexCreditObservation(existing))) {
return {
quota: existing,
needsReauth: false,
Expand Down
15 changes: 14 additions & 1 deletion src/codex/auth-context.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt } from "./pool-credit-policy";
import { needsCodexCreditRefresh } from "./account-credit-use";
import { hasSpendableCodexCredits } from "./quota-types";
import { nextQuotaQueryDelay } from "./quota-query-backoff";
import { noteMainAccountActivity } from "./main-account-external-usage";
Expand Down Expand Up @@ -1027,7 +1028,7 @@ export interface ResolveCodexAuthContextOptions {
signal?: AbortSignal;
/** Test seam: overrides the bounded startup policy-binding wait before the hard-lock fence. */
mainAccountPolicyBindingWaitMs?: number;
primeCodexPoolQuotas?: (config: OcxConfig, reason: string) => Promise<void>;
primeCodexPoolQuotas?: (config: OcxConfig, reason: string, options?: { creditAccountId?: string }) => Promise<void>;
/** Test seam for account-gated native model discovery. */
resolveCodexModelEntitlements?: typeof resolveCodexModelEntitlements;
/** Direct requests admitted with a proxy bearer substitute the stored native-main credential. */
Expand Down Expand Up @@ -1259,6 +1260,16 @@ async function resolveCodexAuthContextUnannotated(
|| (requestScopedMainCredential && fixedAccountId === MAIN_CODEX_ACCOUNT_ID)) {
return resolveCallerOwnedMainContext();
}
// Usage headers cannot renew the balance clock. Observe expired credit evidence before
// selecting an account, rather than falling back to caller main while the pool is funded.
if ((config.codexAccounts ?? []).some(a => !a.isMain
&& (fixedAccountId === undefined || fixedAccountId === a.id)
&& needsCodexCreditRefresh(policy, a.id, getAccountQuota(a.id), a.plan))) {
options.signal?.throwIfAborted();
const prime = options.primeCodexPoolQuotas ?? (await import("./auth-api")).primeCodexPoolQuotas;
await prime(config, "credits-refresh", { creditAccountId: fixedAccountId }).catch(() => {});
options.signal?.throwIfAborted();
}
// A caller bearer can still accompany a request that selects a configured Pool account. Do not
// let that request read, delete, or create a file-main affinity binding while deciding whether a
// stored account is available; only the stored credential selected below may own Pool state.
Expand Down Expand Up @@ -1427,6 +1438,7 @@ async function resolveCodexAuthContextUnannotated(
if (
requestScopedMainCredential
&& fixedAccountId === undefined
&& !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID))
&& options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID
) {
return await resolveCallerOwnedMainContext();
Expand Down Expand Up @@ -1585,6 +1597,7 @@ async function resolveCodexAuthContextUnannotated(
// alternate is eligible. A validated caller may still serve this request,
// just as it can after an upstream rejection, without changing Pool state.
if (requestScopedMainCredential && fixedAccountId === undefined
&& !(callerMatchesObservedMain(headers) && policy.pausedCodexAccountIds?.includes(MAIN_CODEX_ACCOUNT_ID))
&& options.excludeAccountId !== MAIN_CODEX_ACCOUNT_ID
&& !callerIsCooledPoolAccount(headers, config, accountId)) {
return await resolveCallerOwnedMainContext();
Expand Down
16 changes: 12 additions & 4 deletions src/codex/quota-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,23 @@ export interface CodexSpendableCredits {
observedAt: number;
}

/** Freshness is independent of a positive/negative balance verdict and of usage headers. */
export function hasFreshCodexCreditObservation(
quota: Pick<StoredAccountQuota, "credits"> | null,
now = Date.now(),
): boolean {
const observedAt = quota?.credits?.observedAt;
return typeof observedAt === "number" && Number.isFinite(observedAt)
&& now >= observedAt && now - observedAt <= CODEX_CREDITS_FRESHNESS_MS;
}

export function hasSpendableCodexCredits(
quota: Pick<StoredAccountQuota, "credits"> | null,
now = Date.now(),
): boolean {
const credits = quota?.credits;
if (!credits || !Number.isFinite(credits.observedAt)) return false;
const age = now - credits.observedAt;
if (age < 0 || age > CODEX_CREDITS_FRESHNESS_MS
|| credits.allowed === false || credits.overageLimitReached === true || credits.hasCredits === false) return false;
if (!credits || !hasFreshCodexCreditObservation(quota, now)) return false;
if (credits.allowed === false || credits.overageLimitReached === true || credits.hasCredits === false) return false;
return credits.unlimited === true || (credits.hasCredits === true
&& typeof credits.balance === "number" && Number.isFinite(credits.balance) && credits.balance > 0);
}
Expand Down
5 changes: 3 additions & 2 deletions structure/providers/openai-accounts.md
Original file line number Diff line number Diff line change
Expand Up @@ -534,8 +534,9 @@ owns the five-minute cadence; `auth-api/pool-mode-gate.ts` bounds request-trigge
including failures, while preserving main-owner claims and per-credential WHAM dispatch backoff.
No requests means no new polling. For this reason only, stale observation proof bypasses aggregate
quota-cache freshness after the existing attempt backoff. Main refresh keeps its owned lease and
passive intent: cache bypass does not clear an inference reauthentication mark. Other prime reasons
retain their existing cache rules. The split config schema degrades malformed optional values to
passive intent: cache bypass does not clear an inference reauthentication mark. Credit-expiry priming follows the independent balance clock described in the
[spendable credit contract](openai-tiers.md#spendable-codex-credits), bypasses fresh usage-only
caches, and performs no native-main discovery. Other prime reasons retain their existing cache rules. The split config schema degrades malformed optional values to
false. Exact-account and Direct routes are unchanged.

## Main-account policy observations
Expand Down
9 changes: 9 additions & 0 deletions structure/providers/openai-tiers.md
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,15 @@ refusal, so the two predicates can never both hold.

### Spendable Codex credits

Pool selection awaits a paced metadata refresh when an opted-in, unpaused account has an
exhausted included window and its balance observation is absent or older than five minutes.
Fresh usage headers do not renew that balance clock or suppress the refresh. Concurrent refreshes
share the existing prime flight; a dispatched unsuccessful observation waits at least one minute
before another request-triggered attempt for the same credential generation. No inference or reset
redemption is performed, and stale funds remain inadmissible if observation fails. Pool fallback
never uses an observed native-main caller credential while that main account is paused; unrelated
caller-owned Direct credentials retain their existing contract.

WHAM `credits` is spendable usage capacity, separate from manually redeemed
`rate_limit_reset_credits`. The quota store keeps normalized flags, a finite nonnegative balance,
and a separate observation clock. Explicit zero/null replaces earlier evidence; partial usage
Expand Down
17 changes: 17 additions & 0 deletions tests/codex-integration/codex-auth-context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2431,3 +2431,20 @@ test("context Direct bearer admission uses real stored-main materialization and
globalThis.fetch = originalFetch;
}
});

test("pool exhaustion never falls back to an observed paused main caller", async () => {
const cfg = config();
cfg.codexAccounts = [];
cfg.pausedCodexAccountIds = [MAIN_CODEX_ACCOUNT_ID];
cfg.codexMainAccountHardLock = false;
observeMainQuotaIdentity("paused-main-account");
observeMainQuotaCredential("paused-main-bearer", "paused-main-account");
const headers = new Headers({ authorization: "Bearer paused-main-bearer", "chatgpt-account-id": "paused-main-account" });
await expect(resolveCodexAuthContext(headers, cfg, "pool", {
requestScopedMainCredential: true,
})).rejects.toBeInstanceOf(CodexMainProfileDrainingError);
// Direct remains caller-owned; the pause applies to Pool fallback, not unrelated callers.
expect(await resolveCodexAuthContext(headers, cfg, "direct", {
requestScopedMainCredential: true,
})).toMatchObject({ kind: "main", accountId: null });
});
85 changes: 85 additions & 0 deletions tests/codex-integration/codex-quota-prime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import {
readCodexAccountRecord,
saveCodexAccountCredential,
} from "../../src/codex/account-store";
import { resolveCodexAuthContext } from "../../src/codex/auth-context";
import { fetchPoolAccountQuota } from "../../src/codex/auth-api/pool-quota-probe";
import { resetMainCodexAccountIdentityTrackingForTests } from "../../src/codex/account-lifecycle";
import { MAIN_CODEX_ACCOUNT_ID } from "../../src/codex/main-account";
import {
Expand Down Expand Up @@ -142,6 +144,89 @@ describe("primeCodexPoolQuotas", () => {
if (existsSync(TEST_DIR)) removeTreeWithRetry(TEST_DIR);
});

test.each(["prime", "selection", "listing", "exact"])(
"%s refreshes expired credits even when usage was just observed", async path => {
const cfg = makeConfig({ creditCodexAccountIds: ["spender"], pausedCodexAccountIds: [MAIN_CODEX_ACCOUNT_ID],
autoSwitchThreshold: 0, activeCodexAccountId: "spender" });
seedPoolAccount(cfg, "spender", "pro");
const reset = Date.now() + 86_400_000;
setAccountQuotaFromParsed("spender", { weeklyPercent: 100, weeklyResetAt: reset,
credits: { hasCredits: true, balance: 20, observedAt: Date.now() - 300_001 } });
updateAccountQuota("spender", 100, reset);
let reads = 0;
const originalFetch = globalThis.fetch;
globalThis.fetch = (async (input: RequestInfo | URL) => {
expect(String(input)).toContain("/wham/usage");
reads += 1;
return Response.json({ rate_limit: { secondary_window: {
used_percent: 100, limit_window_seconds: 604_800, reset_at: Math.floor(reset / 1000),
} }, credits: { has_credits: true, balance: "20" } });
}) as typeof fetch;
try {
if (path === "selection" || path === "exact") {
expect(await resolveCodexAuthContext(new Headers(), cfg, "pool", {
modelId: "gpt-5.5", ...(path === "exact" ? { accountId: "spender" } : {}),
}))
.toMatchObject({ kind: "pool", accountId: "spender" });
} else if (path === "listing") await fetchPoolAccountQuota("spender");
else await primeCodexPoolQuotas(cfg, "credits-refresh");
expect(reads).toBe(1);
expect(getAccountQuota("spender")?.credits?.observedAt).toBeGreaterThan(Date.now() - 5_000);
await primeCodexPoolQuotas(cfg, "credits-refresh");
expect(reads).toBe(1);
} finally { globalThis.fetch = originalFetch; }
},
);

test("fresh zero balances, paused accounts and credit opt-outs do not trigger credit discovery", async () => {
const cfg = makeConfig({ creditCodexAccountIds: ["empty", "paused"], pausedCodexAccountIds: ["paused"] });
for (const id of ["empty", "paused", "optout"]) {
seedPoolAccount(cfg, id, "pro");
setAccountQuotaFromParsed(id, { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000,
credits: { hasCredits: id !== "empty", balance: id === "empty" ? 0 : 20,
observedAt: Date.now() - (id === "empty" ? 0 : 300_001) } });
}
const originalFetch = globalThis.fetch;
let reads = 0;
globalThis.fetch = (async () => { reads += 1; throw new Error("credit discovery must not run"); }) as typeof fetch;
try {
await primeCodexPoolQuotas(cfg, "credits-refresh");
expect(reads).toBe(0);
expect(previewCodexAccountForRequest("no-permission", cfg)).toBeNull();
} finally { globalThis.fetch = originalFetch; }
});

test("cancelled selection performs no credit observation", async () => {
const cfg = makeConfig({ creditCodexAccountIds: ["spender"] });
seedPoolAccount(cfg, "spender", "pro");
setAccountQuotaFromParsed("spender", { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000 });
const controller = new AbortController();
controller.abort(new Error("cancelled-before-observation"));
let primes = 0;
await expect(resolveCodexAuthContext(new Headers(), cfg, "pool", {
signal: controller.signal, primeCodexPoolQuotas: async () => { primes += 1; },
})).rejects.toThrow("cancelled-before-observation");
expect(primes).toBe(0);
});

test("failed credit observations are paced and never make stale funds spendable", async () => {
const cfg = makeConfig({ creditCodexAccountIds: ["spender"], pausedCodexAccountIds: [MAIN_CODEX_ACCOUNT_ID] });
seedPoolAccount(cfg, "spender", "pro");
setAccountQuotaFromParsed("spender", { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000,
credits: { hasCredits: true, balance: 20, observedAt: Date.now() - 300_001 } });
const oldClock = getAccountQuota("spender")!.credits!.observedAt;
const originalFetch = globalThis.fetch;
let reads = 0;
globalThis.fetch = (async () => { reads += 1; return new Response("unavailable", { status: 503 }); }) as typeof fetch;
try {
await primeCodexPoolQuotas(cfg, "credits-refresh");
await primeCodexPoolQuotas(cfg, "credits-refresh");
expect(reads).toBe(1);
expect(getAccountQuota("spender")?.credits?.observedAt).toBe(oldClock);
expect(previewCodexAccountForRequest("still-held", cfg)).toBeNull();
} finally { globalThis.fetch = originalFetch; }
});

test("startup may observe pending account quota without inference or validation", async () => {
const config = makeConfig();
seedPoolAccount(config, "pending", "pro");
Expand Down
Loading