Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion src/server/responses/core-combo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import {
} from "../../combos";
import { formatErrorResponse } from "../../bridge";
import { SEND_BUDGET_EXHAUSTED_CODE } from "../../lib/errors";
import { sanitizeLogMetadataString } from "../../lib/redact";
import {
expandPreviousResponseInput,
previousResponseReplayFailure,
Expand Down Expand Up @@ -1195,8 +1196,15 @@ export async function executeComboResponses(
}
return lastFailure;
}
// A bare status made a revoked credential read as a provider outage. Log only the parsed
// code and message, never the raw envelope, as one redacted, bounded line.
const failureReason = sanitizeLogMetadataString(
[failure.upstreamCode, failure.upstreamMessage].filter(Boolean).join(": ").replace(/\s+/g, " "),
200,
);
console.warn(
`[combo] ${comboId}: ${targetKey(pick.target)} failed with ${failure.response.status} after ${Date.now() - started}ms`,
`[combo] ${comboId}: ${targetKey(pick.target)} failed with ${failure.response.status}`
+ ` after ${Date.now() - started}ms${failureReason ? `: ${failureReason}` : ""}`,
);
const failureNow = Date.now();
const attemptedTargets = pick.attempted;
Expand Down
31 changes: 30 additions & 1 deletion tests/responses/responses-combo-exhausted-error.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
import { clearComboSelectionState, clearComboTargetCooldowns, coolComboTarget, snapshotComboQuotaCooldowns } from "../../src/combos";
import { clearKeyCooldowns } from "../../src/providers/key-failover";
import { clearCachedProviderQuotas, setCachedProviderQuotaForTests } from "../../src/providers/quota-routing-cache";
Expand Down Expand Up @@ -284,6 +284,35 @@ describe("combo exhaustion reports the primary's quota refusal", () => {
expect(logCtx.attempts).toHaveLength(2);
});

test("the hop warning names the upstream refusal on one redacted line", async () => {
const key = "sk-proj-" + "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6";
globalThis.fetch = (async () => Response.json(
{ error: { code: "invalid_api_key", message: `Incorrect API key provided: ${key}.\nRotate it.` } },
{ status: 401 },
)) as unknown as typeof fetch;
const warn = spyOn(console, "warn").mockImplementation(() => {});
try {
await loggedCombo(config, { model: "", provider: "" });
const line = warn.mock.calls.map(call => String(call[0])).find(text => text.includes("primary/model-primary failed with"));
expect(line).toMatch(/failed with 401 after \d+ms: invalid_api_key: Incorrect API key provided: .* Rotate it\.$/);
expect(line).not.toContain(key);
} finally {
warn.mockRestore();
}
});

test("the hop warning never logs an unparsed upstream body", async () => {
globalThis.fetch = (async () => new Response("upstream said: session=abc123 private detail", { status: 502 })) as unknown as typeof fetch;
const warn = spyOn(console, "warn").mockImplementation(() => {});
try {
await loggedCombo(config, { model: "", provider: "" });
const line = warn.mock.calls.map(call => String(call[0])).find(text => text.includes("primary/model-primary failed with"));
expect(line).toMatch(/failed with 502 after \d+ms$/);
} finally {
warn.mockRestore();
}
});

test("a fallback 5xx is still the answer", async () => {
upstream(502, "bad gateway");
const response = await handleResponses(request(), config, { model: "", provider: "" } as RequestLogContext);
Expand Down
Loading