diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index c96244d1718..5637a1ecd88 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -45,6 +45,7 @@ import { } from "../../combos"; import { formatErrorResponse } from "../../bridge"; import { SEND_BUDGET_EXHAUSTED_CODE } from "../../lib/errors"; +import { sanitizeLogMetadataString } from "../../lib/redact"; import { expandPreviousResponseInput, previousResponseReplayFailure, @@ -1195,8 +1196,15 @@ export async function executeComboResponses( } return lastFailure; } + // A bare status made a revoked credential read as a provider outage. Log only the parsed + // code and message, never the raw envelope, as one redacted, bounded line. + const failureReason = sanitizeLogMetadataString( + [failure.upstreamCode, failure.upstreamMessage].filter(Boolean).join(": ").replace(/\s+/g, " "), + 200, + ); console.warn( - `[combo] ${comboId}: ${targetKey(pick.target)} failed with ${failure.response.status} after ${Date.now() - started}ms`, + `[combo] ${comboId}: ${targetKey(pick.target)} failed with ${failure.response.status}` + + ` after ${Date.now() - started}ms${failureReason ? `: ${failureReason}` : ""}`, ); const failureNow = Date.now(); const attemptedTargets = pick.attempted; diff --git a/tests/responses/responses-combo-exhausted-error.test.ts b/tests/responses/responses-combo-exhausted-error.test.ts index c0c1410004c..8a7a1a0e8c6 100644 --- a/tests/responses/responses-combo-exhausted-error.test.ts +++ b/tests/responses/responses-combo-exhausted-error.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { clearComboSelectionState, clearComboTargetCooldowns, coolComboTarget, snapshotComboQuotaCooldowns } from "../../src/combos"; import { clearKeyCooldowns } from "../../src/providers/key-failover"; import { clearCachedProviderQuotas, setCachedProviderQuotaForTests } from "../../src/providers/quota-routing-cache"; @@ -284,6 +284,35 @@ describe("combo exhaustion reports the primary's quota refusal", () => { expect(logCtx.attempts).toHaveLength(2); }); + test("the hop warning names the upstream refusal on one redacted line", async () => { + const key = "sk-proj-" + "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6"; + globalThis.fetch = (async () => Response.json( + { error: { code: "invalid_api_key", message: `Incorrect API key provided: ${key}.\nRotate it.` } }, + { status: 401 }, + )) as unknown as typeof fetch; + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + await loggedCombo(config, { model: "", provider: "" }); + const line = warn.mock.calls.map(call => String(call[0])).find(text => text.includes("primary/model-primary failed with")); + expect(line).toMatch(/failed with 401 after \d+ms: invalid_api_key: Incorrect API key provided: .* Rotate it\.$/); + expect(line).not.toContain(key); + } finally { + warn.mockRestore(); + } + }); + + test("the hop warning never logs an unparsed upstream body", async () => { + globalThis.fetch = (async () => new Response("upstream said: session=abc123 private detail", { status: 502 })) as unknown as typeof fetch; + const warn = spyOn(console, "warn").mockImplementation(() => {}); + try { + await loggedCombo(config, { model: "", provider: "" }); + const line = warn.mock.calls.map(call => String(call[0])).find(text => text.includes("primary/model-primary failed with")); + expect(line).toMatch(/failed with 502 after \d+ms$/); + } finally { + warn.mockRestore(); + } + }); + test("a fallback 5xx is still the answer", async () => { upstream(502, "bad gateway"); const response = await handleResponses(request(), config, { model: "", provider: "" } as RequestLogContext);