Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion src/adapters/kiro-refusal.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
/** Account-scoped Kiro refusals. Unknown data never convicts an account. */
import { BOUNDED_BODY_MAX_BYTES } from "../lib/bounded-body";
export type KiroRefusalKind = "rate" | "monthly_quota" | "suspended" | "other";
export type KiroRefusalKind = "rate" | "monthly_quota" | "suspended" | "profile" | "other";
export interface KiroRefusal { kind: KiroRefusalKind; resetAt?: number }

/** The vendor's own name for the body validator that rejects a request for this identity. */
const VALIDATION_EXCEPTION = "ValidationException";
/** Kiro's wording for that rejection; the code alone is also what a genuinely bad payload gets. */
const IMPROPER_REQUEST_WORD = /improperly formed request/i;

const SUSPENSION_WORDS = [
"temporarily suspended", "temporarily is suspended",
"locked your account", "locked it as a",
Expand All @@ -22,9 +27,44 @@ export function classifyKiroRefusal(status: number, bodyText: string): KiroRefus
} catch { /* Unrecognised bodies cannot establish an account verdict. */ }
if ((status === 400 || status === 429) && reason === "MONTHLY_REQUEST_COUNT")
return { kind: "monthly_quota" };
// A 400 whose own validator calls the body improperly formed is not proof that the request is
// impossible: Kiro answers this shape when the identity a request was routed under cannot serve
// it. Treating it as `other` leaves the account selected and the same 400 reaches the client.
// Both vendor tokens are required, so a bad payload or a caller echoing one token stays terminal.
if (status === 400 && typeof message === "string" && IMPROPER_REQUEST_WORD.test(message)
&& (reason === VALIDATION_EXCEPTION || message.includes(VALIDATION_EXCEPTION)))
return { kind: "profile" };
if (status === 403 && (reason === "TEMPORARILY_SUSPENDED"
|| (typeof message === "string" && SUSPENSION_WORDS.some(word => message.toLowerCase().includes(word)))))
return { kind: "suspended" };
if (status === 429) return { kind: "rate" };
return { kind: "other" };
}

/** The adapter's own prefix for a normalized 400 (`normalizedKiroErrorMessage` in kiro-errors.ts). */
const KIRO_INVALID_REQUEST_PREFIX = "Kiro invalid request: ";

/**
* The same verdict at the combo layer. There the text is either the adapter's normalized message
* ("Kiro invalid request: ValidationException: Improperly formed request", what a combo child's
* failure carries) or a raw JSON body, each possibly behind the "Provider error 400: " frame.
* JSON goes through the parsed `reason`/`message` rules of `classifyKiroRefusal` (tokens in
* unrelated fields never count); prose counts only behind the adapter's own prefix. Callers must
* also know the failed target is a Kiro adapter.
*/
export function isKiroAccountProfileRefusal(status: number, bodyText: string): boolean {
if (status !== 400 || Buffer.byteLength(bodyText, "utf8") > BOUNDED_BODY_MAX_BYTES) return false;
const text = bodyText.replace(/^Provider error \d{3}: /, "");
if (text.startsWith(KIRO_INVALID_REQUEST_PREFIX)) {
const detail = text.slice(KIRO_INVALID_REQUEST_PREFIX.length);
return detail.includes(VALIDATION_EXCEPTION) && IMPROPER_REQUEST_WORD.test(detail);
}
try {
const row: unknown = JSON.parse(text);
if (!row || typeof row !== "object" || Array.isArray(row)) return false;
const envelope = (row as Record<string, unknown>).error;
const body = envelope && typeof envelope === "object" && !Array.isArray(envelope) ? envelope : row;
const { reason, message } = body as Record<string, unknown>;
return classifyKiroRefusal(400, JSON.stringify({ reason, message })).kind === "profile";
} catch { return false; }
}
9 changes: 8 additions & 1 deletion src/combos/failover.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { parseResetCooldownMs } from "../codex/routing";
import { classifyError, isCyberPolicyCode } from "../lib/errors";
import { isNonReplayableUpstreamCode } from "../lib/upstream-retry";
import { isAnthropicBillingFailure } from "../providers/anthropic-billing-refusal";
import { isKiroAccountProfileRefusal } from "../adapters/kiro-refusal";
import type { OcxComboTarget } from "../types";
import { targetKey } from "./types";
import {
Expand Down Expand Up @@ -772,7 +773,7 @@ export function codexAccountModelRefusal(
export function comboFailureDecision(
status: number,
message: string,
options?: { code?: string | null; codexModelRefusal?: CodexAccountModelRefusal },
options?: { code?: string | null; codexModelRefusal?: CodexAccountModelRefusal; failedAdapter?: string },
): ComboFailureDecision {
if (status === 499) return "stop";
if (message.toLowerCase().includes("origin_rejected")) return "stop";
Expand Down Expand Up @@ -856,6 +857,12 @@ export function comboFailureDecision(
// could have served the request (#4903).
if (isResponseFormatCapabilityRefusal(status, message, options?.code)) return "hop";
if (isAnthropicBillingFailure(status, message, options?.code)) return "hop";
// Kiro's body validator rejects the request for the identity it was routed under. The account
// layer rotates logins first; this is the no-alternate case, so the next combo target is the
// only remaining chance to serve the turn. Only a caller that knows the failed target is a Kiro
// adapter grants it (the words alone prove nothing for another provider); it must precede the
// generic invalid_request_error stop.
if (options?.failedAdapter === "kiro" && isKiroAccountProfileRefusal(status, message)) return "hop";
if (["origin_rejected", "context_length_exceeded", "invalid_request_error"].includes(error.code ?? "")) {
return "stop";
}
Expand Down
1 change: 1 addition & 0 deletions src/server/responses/core-combo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1165,6 +1165,7 @@ export async function executeComboResponses(
: comboFailureDecision(failure.response.status, failure.classificationText, {
code: failure.upstreamCode,
codexModelRefusal: failure.codexModelRefusal,
failedAdapter: config.providers[pick.target.provider]?.adapter,
});
const wantsStream = (rawBody as { stream?: unknown } | null)?.stream === true;
// Local byte admission has its own diagnostic; do not relabel it as an upstream refusal.
Expand Down
9 changes: 8 additions & 1 deletion structure/providers/kiro.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,14 @@ observed ID still informs routing preference.

`src/adapters/kiro-refusal.ts` recognizes an exact monthly reason on HTTP 400/429 and a
confirmed suspension on HTTP 403; ordinary 400/403 remains an error without an account
verdict. `src/providers/kiro-usage.ts` records monthly exhaustion only for the sent
verdict. A 400 whose parsed body carries `ValidationException` (as `reason`, or inside `message`)
and the message "Improperly formed request" is the `profile` kind: the account layer rotates to a
sibling login like any other account-scoped refusal, and both tokens are required in the parsed
fields. `isKiroAccountProfileRefusal` applies the same rule to combo failure text (adapter-normalized
`Kiro invalid request: ...` prose or a JSON body, with or without the `Provider error 400: ` frame).
It is only consulted when the failed target's adapter is `kiro` (see
[combo failover](../transports/responses-failover.md)); coverage is
`tests/providers/kiro/kiro-refusal.test.ts` and `tests/server/server-kiro-refusal-e2e.test.ts`. `src/providers/kiro-usage.ts` records monthly exhaustion only for the sent
credential generation and login identity, independently of quota observation time. A
completed response from that same live credential clears an older verdict after disk
hydration. Suspension is a process-local quarantine; rate refusals use a short cooldown.
Expand Down
12 changes: 12 additions & 0 deletions structure/transports/responses-failover.md
Original file line number Diff line number Diff line change
Expand Up @@ -566,3 +566,15 @@ The resend charges the root workflow send counter once without a second request-
- **Attempt ownership:** Each attempt captures its adapter binding at the physical-send observation. Recovery passes that binding's key and `_apiKeyAttempt` to the existing API-key 401/429 helpers, so a late failure cannot cool or overwrite a newer manual selection. A replacement rebuilds the adapter and replay scope before dispatch. Failed credentials are not resent in the request; the initial pool size and shared budget bound attempts.
- **Cooldown:** Only an eligible 429 API-key recovery enters this cooldown path. When no reset time is known, the failed key receives the existing default cooldown. Nonrecoverable quota classifications do not create a key cooldown through this path; no daily reset is inferred.
- **Regression coverage:** `tests/providers/qoder-adapter.test.ts` covers classification; `tests/server/server-key-failover-e2e.test.ts` covers mocked/local key recovery and replay boundaries; `tests/oauth/generic-oauth-failover.test.ts` guards recovery-site accounting.

## Kiro account-profile hop

`comboFailureDecision` takes an optional `failedAdapter` (the adapter of the target that failed;
`executeComboResponses` passes `config.providers[target.provider].adapter`). Only `failedAdapter === "kiro"`
may hop on an HTTP 400 that `isKiroAccountProfileRefusal` recognises (`ValidationException` plus
"Improperly formed request" in the parsed Kiro body, reached after the account layer has no sibling login
left to rotate to). Any other provider, or a caller that does not pass the adapter, keeps the generic
terminal `invalid_request_error` stop, so an invalid payload is never resent to another declared target.
Coverage: `tests/codex-integration/combos.test.ts` (unit, with non-Kiro and unrelated-field negatives) and
`tests/server/server-kiro-refusal-e2e.test.ts` (account rotation, combo exhaustion hop, non-Kiro 400 stop,
malformed-payload stop).
16 changes: 16 additions & 0 deletions tests/codex-integration/combos.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -951,6 +951,22 @@ describe("combo failure policy and advancement", () => {
expect(comboFailureDecision(410, "resource is gone")).toBe("stop");
expect(comboFailureDecision(410, "The model has reached its end of life and is no longer available.")).toBe("hop");
expect(comboFailureDecision(410, "The model is scheduled for retirement.")).toBe("hop");
// A Kiro 400 that names its own validator rejection is account-scoped: after the account layer
// has no login left to rotate to, the next combo target is the only chance to serve the turn.
const improper = JSON.stringify({ message: "ValidationException: Improperly formed request" });
const kiro = { failedAdapter: "kiro" };
expect(comboFailureDecision(400, improper, kiro)).toBe("hop");
expect(comboFailureDecision(400, `Provider error 400: ${improper}`, kiro)).toBe("hop");
expect(comboFailureDecision(400, JSON.stringify({ message: "Improperly formed request" }), kiro)).toBe("stop");
expect(comboFailureDecision(400, "ValidationException: model not found", kiro)).toBe("stop");
// Tokens in unrelated fields, or a failed target that is not Kiro (or unknown), never hop.
expect(comboFailureDecision(400, JSON.stringify({ metadata: "ValidationException", message: "Improperly formed request" }), kiro)).toBe("stop");
expect(comboFailureDecision(400, JSON.stringify({ error: { message: "Kiro invalid request: ValidationException: Improperly formed request" } }), kiro)).toBe("hop");
expect(comboFailureDecision(400, "Kiro invalid request: ValidationException: Improperly formed request", kiro)).toBe("hop");
expect(comboFailureDecision(400, "Kiro invalid request: ValidationException: Improperly formed request", { failedAdapter: "openai-chat" })).toBe("stop");
expect(comboFailureDecision(400, improper)).toBe("stop");
expect(comboFailureDecision(400, improper, { failedAdapter: "openai-chat" })).toBe("stop");
expect(comboFailureDecision(400, improper, { failedAdapter: "anthropic" })).toBe("stop");
expect(comboFailureDecision(410, "gone", { code: "model_retired" })).toBe("hop");
expect(comboFailureDecision(499, "client cancelled")).toBe("stop");
expect(comboFailureDecision(422, "invalid_api_key")).toBe("hop");
Expand Down
42 changes: 41 additions & 1 deletion tests/providers/kiro/kiro-refusal.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
import { classifyKiroRefusal } from "../../../src/adapters/kiro-refusal";
import { classifyKiroRefusal, isKiroAccountProfileRefusal } from "../../../src/adapters/kiro-refusal";
import { kiroEvidenceIdentity } from "../../../src/providers/kiro-account-state-disk";
import type { ProviderAccount } from "../../../src/oauth/types";

Expand All @@ -23,6 +23,46 @@ describe("Kiro refusal classification", () => {
}
});

test("kiro refusal: a 400 the vendor's validator calls improperly formed convicts the account", () => {
const improper = "ValidationException: Improperly formed request";
for (const body of [
{ message: improper },
{ reason: "ValidationException", message: "Improperly formed request." },
]) expect(classified(400, body).kind).toBe("profile");
expect(classified(429, { message: improper }).kind).toBe("rate");
expect(classified(403, { message: improper }).kind).toBe("other");
// Both tokens are required, so a bad payload and a caller echoing one token stay terminal.
expect(classified(400, { message: "Improperly formed request" }).kind).toBe("other");
expect(classified(400, { message: "ValidationException: model not found" }).kind).toBe("other");
expect(classified(400, { message: improper, padding: "x".repeat(70_000) }).kind).toBe("other");
});

test("kiro refusal: the combo layer recognises the same shape through a wrapped frame", () => {
const improper = JSON.stringify({ message: "ValidationException: Improperly formed request" });
expect(isKiroAccountProfileRefusal(400, improper)).toBe(true);
expect(isKiroAccountProfileRefusal(400, `Provider error 400: ${improper}`)).toBe(true);
expect(isKiroAccountProfileRefusal(429, improper)).toBe(false);
expect(isKiroAccountProfileRefusal(400, "Improperly formed request")).toBe(false);
// The adapter's normalized envelope is what the combo layer actually receives.
const envelope = JSON.stringify({ error: { message: "Kiro invalid request: ValidationException: Improperly formed request",
type: "invalid_request_error", code: "invalid_request_error" } });
expect(isKiroAccountProfileRefusal(400, envelope)).toBe(true);
expect(isKiroAccountProfileRefusal(400, JSON.stringify({ error: { message: "Kiro invalid request: model not found" } }))).toBe(false);
expect(isKiroAccountProfileRefusal(400, JSON.stringify({ error: { metadata: "ValidationException", message: "Improperly formed request" } }))).toBe(false);
// The adapter's normalized prose, bare or framed; the same words without its prefix do not count.
const prose = "Kiro invalid request: ValidationException: Improperly formed request";
expect(isKiroAccountProfileRefusal(400, prose)).toBe(true);
expect(isKiroAccountProfileRefusal(400, `Provider error 400: ${prose}`)).toBe(true);
expect(isKiroAccountProfileRefusal(400, "Kiro invalid request: model not found")).toBe(false);
// Split-field case: both tokens present, but not in the fields the account layer reads.
const split = JSON.stringify({ metadata: "ValidationException", message: "Improperly formed request" });
expect(classified(400, JSON.parse(split)).kind).toBe("other");
expect(isKiroAccountProfileRefusal(400, split)).toBe(false);
expect(isKiroAccountProfileRefusal(400, `Provider error 400: ${split}`)).toBe(false);
// Unparseable prose that merely contains both tokens is not the vendor's body.
expect(isKiroAccountProfileRefusal(400, "upstream said ValidationException: Improperly formed request")).toBe(false);
});

test("kiro refusal: unknown, nested, malformed and oversized shapes never quarantine", () => {
for (const body of ["not-json", "{", JSON.stringify({ error: { reason: "TEMPORARILY_SUSPENDED" } }),
JSON.stringify({ reason: "INVALID_MODEL_ID" }),
Expand Down
Loading
Loading