fix(responses): rotate a pooled API key on a native passthrough 429

src/server/responses/passthrough-dispatch.ts

@@ -1,5 +1,6 @@
 import { recoverAutoPausedAccount } from "./account-auto-pause-recovery";
 import { adoptCodexWsStage, capturePassthroughAffinity } from "./passthrough-observations";
+import { rotatePassthroughKeyOn429 } from "./passthrough-key-rotation";
 import { spendLedgerStorageErrorResponse } from "./spend-storage-error";
 import { requestPacingOverloadResponse } from "./pacing-overload";
 import { recordDistributedResponseFailure } from "../../providers/api-key-response-ownership";
@@ -1606,6 +1607,8 @@ export async function preparePassthroughExchange(
         return transportFailureResponse(err);
       }
     }
+    const keyRotation = sendBudgetExhausted(transientSendAttempts()) ? undefined : await rotatePassthroughKeyOn429(config, route, parsed, upstreamResponse, transportState.adapter.name, () => rebuildAndRefetch("key-429"));
+    if (keyRotation) { if ("failed" in keyRotation) return keyRotation.failed; upstreamResponse = keyRotation; continue passthroughRecovery; }
     const captureAffinityResponse = (
       response: Response,
       captureAuthCtx: CodexAuthContext = admissionState.authCtx,

src/server/responses/passthrough-key-rotation.ts

@@ -0,0 +1,35 @@
+import { isNonReplayableResponse } from "../../lib/upstream-retry";
+import { hasKeyPoolFailover, rotateProviderTransportOn429 } from "../../providers/key-failover";
+import type { RouteResult } from "../../router";
+import type { OcxConfig, OcxParsedRequest } from "../../types";
+import { bindRouteReasoningReplayScope } from "./core-replay";
+
+/**
+ * Key-pool failover for a native passthrough 429 before any output. This wire returns before the
+ * adapter loop's key-429 arm, so without it a pooled key's 429 reached the client while the same
+ * refusal rotates on the adapter and chat-native wires. Distributed (round-robin) selection already
+ * cools the key on the send and deliberately answers without a replay. A synthesized replay refusal
+ * is not a provider rate limit and must not cool a key. The caller checks the send budget first so
+ * a spent budget keeps the real 429 body. Returns undefined when nothing rotated.
+ */
+export async function rotatePassthroughKeyOn429<T>(
+  config: OcxConfig,
+  route: Pick<RouteResult, "provider" | "providerName">,
+  parsed: OcxParsedRequest,
+  response: Response,
+  adapterName: string,
+  refetch: () => Promise<T>,
+): Promise<T | undefined> {
+  if (response.status !== 429 || isNonReplayableResponse(response) || !hasKeyPoolFailover(route.provider)
+    || route.provider._apiKeyAttempt?.distributed) return;
+  const rotated = rotateProviderTransportOn429(config, route.providerName, route.provider, {
+    retryAfter: response.headers.get("retry-after"),
+    attemptedKey: route.provider.apiKey,
+    promptCacheKey: parsed.options.promptCacheKey,
+  });
+  if (!rotated) return;
+  try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ }
+  route.provider = rotated;
+  bindRouteReasoningReplayScope({ parsed, providerName: route.providerName, provider: rotated, adapterName });
+  return refetch();
+}

structure/transports/responses-failover.md

@@ -156,7 +156,7 @@ The shared Responses path follows the [bounded multipart recovery contract](../s
 
 ## Upstream key attempt accounting
 
-Key identity is sealed at guarded physical dispatch after queued selections are rebuilt. Distributed selections retain their request-local entry and revision. Permanent refusal auto-pause can disable that exact entry independently of the active key; recovery selects a healthy sibling under the same pacing and physical-send bounds. A rebuild after selection drift releases the old per-key lease and reacquires admission for the refreshed key; the send guard checks again after waiting. Turn-scoped adapters follow the same rule before invocation. Native Chat, Messages and keyed compact all rebuild stale queued selections before dispatch, including a manual active-key switch under per-key limits; they release the old lease, readmit the new key and honor cancellation while its capacity is occupied. Direct compact and Images record genuine distributed 401/429 cooldowns without replaying paid POSTs; synthetic replay refusals never cool a key. Native Messages revisits mixed distributed 401/429 outcomes with its existing counters. Native Chat and Messages require the physical outgoing credential to match the selected distributed key before pooling a rejection; Messages honors the configured `x-api-key` or Bearer transport while rejecting a conflicting credential header; configured credential overrides cannot cool unused entries. Replacement admission pauses the response-header deadline and retains client cancellation; the header deadline restarts once admission completes. Native Responses captures distributed-key ownership after connection-policy rewrites in `src/providers/api-key-response-ownership.ts`; terminal 401/429 responses settle cooldown only after same-key retry policy, and a failed replay admission or synthetic replay refusal retains the last genuine refusal. Failure ownership accepts an already cooling, still-configured credential so concurrent late 401s extend earlier short cooldowns; physical-send eligibility still refuses cooling credentials. Reused responses, credential/destination overrides, stale revisions and synthetic refusals cannot confer ownership. Local pacing errors retain 429 and Retry-After at the passthrough catch boundary. Unpaced dispatch without a readmission owner fails closed. Replay notification errors release acquired slots before propagating, including pre-acquired-slot dispatch. Translated Anthropic billing recovery uses the [immutable physical-send ownership contract](../providers/anthropic-key-billing.md). Its initial send, credential hop and later mixed-class refusals share the request budget. A denied hop retains the upstream refusal, and a leased pacing body rewrap carries the existing marker rather than attributing the response to rebuilt headers.
+Key identity is sealed at guarded physical dispatch after queued selections are rebuilt. Distributed selections retain their request-local entry and revision. Permanent refusal auto-pause can disable that exact entry independently of the active key; recovery selects a healthy sibling under the same pacing and physical-send bounds. A rebuild after selection drift releases the old per-key lease and reacquires admission for the refreshed key; the send guard checks again after waiting. Turn-scoped adapters follow the same rule before invocation. Native Chat, Messages and keyed compact all rebuild stale queued selections before dispatch, including a manual active-key switch under per-key limits; they release the old lease, readmit the new key and honor cancellation while its capacity is occupied. Direct compact and Images record genuine distributed 401/429 cooldowns without replaying paid POSTs; synthetic replay refusals never cool a key. Native Messages revisits mixed distributed 401/429 outcomes with its existing counters. Native Chat and Messages require the physical outgoing credential to match the selected distributed key before pooling a rejection; Messages honors the configured `x-api-key` or Bearer transport while rejecting a conflicting credential header; configured credential overrides cannot cool unused entries. Replacement admission pauses the response-header deadline and retains client cancellation; the header deadline restarts once admission completes. Native Responses captures distributed-key ownership after connection-policy rewrites in `src/providers/api-key-response-ownership.ts`; terminal 401/429 responses settle cooldown only after same-key retry policy, and a failed replay admission or synthetic replay refusal retains the last genuine refusal. A non-distributed (active-key) pool rotates a pre-output native Responses 429 to the next healthy key through `src/server/responses/passthrough-key-rotation.ts`, within the request send budget. Failure ownership accepts an already cooling, still-configured credential so concurrent late 401s extend earlier short cooldowns; physical-send eligibility still refuses cooling credentials. Reused responses, credential/destination overrides, stale revisions and synthetic refusals cannot confer ownership. Local pacing errors retain 429 and Retry-After at the passthrough catch boundary. Unpaced dispatch without a readmission owner fails closed. Replay notification errors release acquired slots before propagating, including pre-acquired-slot dispatch. Translated Anthropic billing recovery uses the [immutable physical-send ownership contract](../providers/anthropic-key-billing.md). Its initial send, credential hop and later mixed-class refusals share the request budget. A denied hop retains the upstream refusal, and a leased pacing body rewrap carries the existing marker rather than attributing the response to rebuilt headers.
 Before a turn-scoped retry readmits a refreshed selection, it rejects a binding already recorded as failed for this request and returns the last key failure. Eligible replacement keys still release the old lease and reacquire their own pacing capacity before invocation, rechecking after every admission. CLI-backed turns carry a synchronous selection fence through asynchronous tool setup to the physical spawn boundary. Raw adapter terminal usage is recorded before continuation, search, or image loops merge it;
 repeated parsing of one physical response does not count it twice. Key changes preserve the
 previous attempt while retaining the active attempt object shared by streaming/combo callbacks.

tests/helpers/responses-core-source.ts

@@ -53,6 +53,7 @@ export const RESPONSES_CORE_MODULES = [
   "passthrough-execution.ts",
   "passthrough-dispatch.ts",
   "passthrough-observations.ts",
+  "passthrough-key-rotation.ts",
   // Reached from passthrough/adapter dispatch, compact and runTurn: local spend-ledger refusal mapping.
   "spend-storage-error.ts",
   // Reached from passthrough dispatch: local pacing refusals retain 429 and Retry-After.

tests/oauth/generic-oauth-failover.test.ts

@@ -571,8 +571,9 @@ describe("sidecar on429 wiring", () => {
     //                wire/pool remain unchanged.
     //   anthropic = 3: its existing HTTP recovery sites; the self-authenticated Claude
     //                  CLI does not authorize API-key recovery through runTurn.
-    //   key       = 7: hasKeyPoolFailover guards the three existing HTTP sites (two 429
-    //                  response loops and the pre-stream 401 recovery site), the pre-output
+    //   key       = 8: hasKeyPoolFailover guards the three existing HTTP sites (two 429
+    //                  response loops and the pre-stream 401 recovery site), the native
+    //                  Responses passthrough 429 arm for non-distributed pools, the pre-output
     //                  Anthropic credit-balance loop (#6958), plus runTurn preflight activation,
     //                  its explicitly permitted API-key recovery arm, and the streaming
     //                  preflight stall bound. The billing loop needs only the key rotator: an
@@ -586,7 +587,7 @@ describe("sidecar on429 wiring", () => {
     // updating the matching number. That decision is the thing this test exists to force.
     expect(counts.generic).toBe(6);
     expect(counts.anthropic).toBe(3);
-    expect(counts.key).toBe(7);
+    expect(counts.key).toBe(8);
   });
 
   test("the verify-account arm moves via auth-refusal rotation, never rate-limit", () => {

tests/responses/responses-native-key-cooldown.test.ts

@@ -225,3 +225,11 @@ test("a synthetic replay refusal retains the preceding genuine key rejection", a
   expect(getKeyCooldownUntil(NAME, "k1")).toBeGreaterThan(Date.now());
   unchanged(config);
 });
+test("native Responses rotates a non-distributed key pool on 429 before output", async () => {
+  const config = fixture(); delete config.providers[NAME]!.requestPacing; saveConfig(config);
+  intercept(headers => headers.get("authorization") === `Bearer ${KEYS[0]}`
+    ? Response.json({ error: { message: "fixture capacity" } }, { status: 429, headers: { "retry-after": "45" } }) : completed());
+  expect((await start(config)).response.status).toBe(200);
+  expect(seen).toEqual([`Bearer ${KEYS[0]}`, `Bearer ${KEYS[1]}`]);
+  expect(getKeyCooldownUntil(NAME, "k1")).toBeGreaterThan(Date.now());
+});