Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion scripts/test-layout/layout.json
Original file line number Diff line number Diff line change
Expand Up @@ -1587,6 +1587,7 @@
"responses-translated-reset.test.ts": "responses",
"responses-routed-web-search-fields.test.ts": "responses",
"responses-self-named-namespace-scrub.test.ts": "responses",
"responses-combo-bridge-usage.test.ts": "responses",
"responses-combo-exhausted-error.test.ts": "responses",
"responses-combo-run-turn-classified-failure.test.ts": "responses",
"responses-send-budget-counts.test.ts": "responses",
Expand Down Expand Up @@ -1674,7 +1675,8 @@
"server-jev-combo-e2e.test.ts": "server", "combo-requested-effort.test.ts": "server",
"server-kiro-completion-e2e.test.ts": "server",
"server-kiro-device-login.test.ts": "server", "server-kiro-oauth-401-replay.test.ts": "server",
"server-kiro-refusal-e2e.test.ts": "server", "server-live-existing-call.test.ts": "server",
"server-kiro-refusal-e2e.test.ts": "server", "server-kiro-usage-log-e2e.test.ts": "server",
"server-live-existing-call.test.ts": "server",
"server-live-frame-log.test.ts": "server",
"server-live-realtime-fixtures.test.ts": "server",
"server-live.test.ts": "server",
Expand Down
1 change: 1 addition & 0 deletions src/adapters/kiro-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ function parseTokenUsage(eventType: string, value: unknown): OcxUsage | undefine
totalTokens,
...(cacheRead !== undefined ? { cachedInputTokens: cacheRead, cacheReadInputTokens: cacheRead } : {}),
...(cacheWrite !== undefined ? { cacheCreationInputTokens: cacheWrite } : {}),
upstreamReported: true,
};
}

Expand Down
1 change: 1 addition & 0 deletions src/adapters/kiro/stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ function mergeKiroUsage(
...(sumOptional("providerCredits") !== undefined ? { providerCredits: sumOptional("providerCredits") } : {}),
...(sumOptional("reasoningOutputTokens") !== undefined ? { reasoningOutputTokens: sumOptional("reasoningOutputTokens") } : {}),
...(first.estimated || second.estimated ? { estimated: true } : {}),
...(first.upstreamReported && second.upstreamReported ? { upstreamReported: true as const } : {}),
};
}

Expand Down
6 changes: 5 additions & 1 deletion src/server/request-log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1623,7 +1623,10 @@ function finalizedUsage(
const usageFallback = !finalUsage && estimate !== undefined
? { inputTokens: estimate, outputTokens: 0, estimated: true }
: undefined;
const combinedInputTokens = finalUsage && estimate !== undefined
// The estimate is a floor under a count the adapter could only guess. A count the adapter read
// off the upstream's own usage frame stays exact: max() against a larger local estimate would
// replace it and mark it estimated.
const combinedInputTokens = finalUsage && !finalUsage.upstreamReported && estimate !== undefined
? Math.max(finalUsage.inputTokens, estimate)
: undefined;
const loggedUsage = finalUsage && combinedInputTokens !== undefined
Expand Down Expand Up @@ -1905,6 +1908,7 @@ export function aggregateAttemptUsage(
...(reasoningOutputTokens !== undefined ? { reasoningOutputTokens } : {}),
...(providerCredits !== undefined ? { providerCredits } : {}),
...(status === "estimated" ? { estimated: true } : {}),
...(usages.every(usage => usage.upstreamReported) ? { upstreamReported: true as const } : {}),
};
return { usage: aggregate, status, totalTokens };
}
Expand Down
19 changes: 19 additions & 0 deletions src/server/responses/core-combo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -931,6 +931,19 @@ export async function executeComboResponses(
finishRequestAttempt(attempt, 499, Date.now() - started, childLog.usage);
attemptRetained = true;
};
// The row is written from `logCtx`, which copies `childLog` only when the child commits. A
// bridge child binds its usage later, at the terminal frame, so the binding is forwarded here
// after that point. Left on `childLog`, the row would re-read the bridged wire instead of the
// adapter's usage, and that wire has no providerCredits and a context-derived input count.
// A binding that arrives before the commit waits for it, and a child that never commits
// forwards nothing: both flags belong to this target only.
let childCommitted = false;
let childUsageBound = false;
const forwardChildUsage = (): void => {
if (!childCommitted || !childUsageBound) return;
logCtx.usageFromBridge = true;
if (childLog.usage) logCtx.usage = childLog.usage;
};
const completedTarget = { provider: pick.target.provider, model: pick.target.model };
const writerGeneration = pick.writerGeneration;
let consumedChildFailure: ConsumedComboFailure | undefined;
Expand Down Expand Up @@ -1007,6 +1020,10 @@ export async function executeComboResponses(
comboDispatchPermit: hopDecision?.allowed ? hopDecision.permit : undefined,
comboReplaySnapshot,
deferCodexResetDerivedCooldown,
onBridgeUsageBound: () => {
childUsageBound = true;
forwardChildUsage();
},
// Attempt-relative TTFT is recorded HERE (not via childLog.firstOutputMs — a later
// Object.assign(logCtx, childLog) would overwrite the request-relative value).
onFirstOutput: () => {
Expand Down Expand Up @@ -1093,6 +1110,8 @@ export async function executeComboResponses(
options.onCodexAuthContextResolved?.(resolvedAuth);
options.setTerminalOutcomeRecorder?.(terminalRecorder);
callbackGate.commit();
childCommitted = true;
forwardChildUsage();
return response;
}

Expand Down
7 changes: 7 additions & 0 deletions src/server/responses/core-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,13 @@ export interface HandleResponsesOptions {
callerDirectAuth?: CallerDirectAuth | null;
/** Internal recursion guard; callers outside this module must not set it. */
comboAttempt?: boolean;
/**
* Set by the combo loop for each child. A child logs into its own context, which the combo
* copies once when the child commits, and the bridge reports usage after that, at the terminal
* frame. The transport calls this when it binds that usage on the child's context, so the combo
* can carry the binding to the context that writes the row.
*/
onBridgeUsageBound?: () => void;
/** Child-owned prepaid initial send; capacity remains charged until dispatch or unsent release. */
comboInitialSend?: { permit: SingleUseDispatchPermit; producerOwned?: boolean };
/** Exact externally booked combo hop, used for its child's spend preflight and send reports. */
Expand Down
5 changes: 2 additions & 3 deletions src/server/responses/request-transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -522,12 +522,11 @@ export async function prepareResponsesTransport(
logCtx.usageFromBridge = true;
if (usesApiKeyAccount(route.provider)) {
logCtx.usage = logCtx.activeAttempt?.usage;
return;
}
if (usage) {
} else if (usage) {
logCtx.usage = usage;
if (logCtx.activeAttempt) logCtx.activeAttempt.usage = usage;
}
options.onBridgeUsageBound?.();
};
const selectionIsCurrent = (binding: DispatchBinding | undefined): boolean => {
if (!anthropicAdmissionIsCurrent()) return false;
Expand Down
7 changes: 7 additions & 0 deletions src/types/request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,13 @@ export interface OcxUsage {
cacheCreation1hInputTokens?: number;
reasoningOutputTokens?: number;
estimated?: boolean;
/**
* The adapter read these counts off the upstream's own usage frame. Only an adapter whose
* provider is estimated by default (usageForFinalLog) sets it, and a merge keeps it only when
* every leg had it. A figure taken off a bridged wire or guessed never carries it, so the
* provider default still marks those as estimated.
*/
upstreamReported?: true;
/**
* The raw upstream usage object for Responses-shaped upstreams (openai/codex#41980 parity):
* codex-rs preserves the complete `response.usage` object through its own pipeline, so fields
Expand Down
8 changes: 7 additions & 1 deletion src/usage/log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,10 @@ export function usageTotalTokens(usage: OcxUsage | undefined): number | undefine
* Providers whose adapters can only estimate usage (no authoritative per-turn frame).
* Callers should pass the route ADAPTER when available; the name-prefix match is a
* fallback for paths that only know the configured provider name (e.g. "cursor-mykey").
*
* Kiro is on the list because most of what reaches the log is a guess: a figure taken off the
* bridged Responses wire derives its input from the context checkpoint. The adapter opts a
* measured figure out with `upstreamReported`, set where it reads Kiro's own tokenUsage frame.
*/
function isEstimatedUsageProvider(providerOrAdapter: string): boolean {
return providerOrAdapter === "kiro" || providerOrAdapter.startsWith("kiro-")
Expand All @@ -556,7 +560,9 @@ export function usageForFinalLog(
): OcxUsage | undefined {
if (!usage) return undefined;
if (locallyAnswered) return usage;
if (usage.estimated || isEstimatedUsageProvider(provider)) return { ...usage, estimated: true };
if (usage.estimated || (isEstimatedUsageProvider(provider) && !usage.upstreamReported)) {
return { ...usage, estimated: true };
}
return usage;
}

Expand Down
10 changes: 8 additions & 2 deletions structure/providers/kiro.md
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,14 @@ finite `contextUsagePercentage` (and a `tokenUsage` block) there as a fallback,
from `metadataEvent` is legitimate rather than impossible. Both feed the same field, and any
positive value overwrites an earlier one.

Spend arrives in `meteringEvent` as **credits, not tokens**. No captured response carried
`tokenUsage` on any event, which is why Kiro token usage stays estimated. The parser preserves
Spend arrives in `meteringEvent` as **credits, not tokens**. Kiro token usage stays estimated
unless a `metadataEvent` carries a `tokenUsage` block; no captured response did. That block is the
only frame that yields a measured count: the parser marks the usage `upstreamReported`,
`mergeKiroUsage` keeps the mark only when both legs of a two-leg turn have it, and
`usageForFinalLog` (`src/usage/log.ts`) then skips the provider-wide estimated mark while
`finalizedUsage` (`src/server/request-log.ts`) skips the local input floor. A turn without
`tokenUsage`, a leg without it, and a count read off the bridged Responses wire all keep the
estimated mark. The parser preserves
`meteringEvent` unit/usage (`amount` is an alias) and optional `unitPlural`; credit readings populate
`OcxUsage.providerCredits` independently of token metadata. The latest reading within a response
is a snapshot; separate completion-fallback responses add their credits. Missing metering stays
Expand Down
9 changes: 9 additions & 0 deletions structure/transports/responses-spend.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,15 @@ what those sends may cost, and it is the only bound here that survives a restart
caller is `request-spend.ts`, installed on the execution budget at genuine ingress in `core.ts`
and parked on the log context so `addFinalRequestLog` can settle it. Native Chat installs the same tracker before its independent physical-send ladder and charges it immediately before each dispatch, so that fast path cannot bypass root, identity or provider-pool ceilings.

The tracker settles with `logCtx.usage`, and so does the row. A combo copies its child's log
context into that one only when the child commits, while the bridge binds the adapter usage later,
at the terminal frame. `executeComboResponses` therefore passes each child `onBridgeUsageBound`
(`core-options.ts`), which `bindKeyUsageFromBridge` calls after it binds. After the child commits,
the combo sets `usageFromBridge` and `usage` on its own context; a binding that came earlier waits
for the commit, and a child that fails or never commits forwards nothing. Without that, the row
and the settlement would read the usage back off the bridged Responses wire, which carries no
`providerCredits` and derives its input count from the context checkpoint.

Unconfigured requests preserve the legacy counter observer: reservations increment the send
counter, refunds decrement it, and named reports reconcile prepaid sends. Enforced requests use
the seed and physical-start protocol below across Responses, native Messages and native Chat.
Expand Down
4 changes: 3 additions & 1 deletion tests/fixtures/test-layout-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@
"codex-shim-overlay-readiness.test.ts": "codex-integration",
"codex-shim-overlay-candidates.test.ts": "codex-integration",
"droid-normalized-reasoning-defaults.test.ts": "clients",
"responses-combo-bridge-usage.test.ts": "responses",
"responses-combo-exhausted-error.test.ts": "responses",
"responses-combo-run-turn-classified-failure.test.ts": "responses",
"cli-config-default-show.test.ts": "cli",
Expand Down Expand Up @@ -1155,7 +1156,8 @@
"server-jev-combo-e2e.test.ts": "server", "server-kiro-completion-e2e.test.ts": "server",
"combo-requested-effort.test.ts": "server",
"server-kiro-device-login.test.ts": "server", "server-kiro-oauth-401-replay.test.ts": "server",
"server-kiro-refusal-e2e.test.ts": "server", "server-live-existing-call.test.ts": "server",
"server-kiro-refusal-e2e.test.ts": "server", "server-kiro-usage-log-e2e.test.ts": "server",
"server-live-existing-call.test.ts": "server",
"server-live-frame-log.test.ts": "server",
"server-live-realtime-fixtures.test.ts": "server", "server-live.test.ts": "server",
"server-loopback-host-gate.test.ts": "server", "server-management-auth.test.ts": "server",
Expand Down
1 change: 1 addition & 0 deletions tests/providers/kiro/kiro-stream.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1538,6 +1538,7 @@ describe("kiro adapter — parseStream", () => {
cacheCreationInputTokens: 2,
outputTokens: 4,
totalTokens: 19,
upstreamReported: true,
});
// The whole-payload checkpoint must exceed this turn's own total, which is the point of
// reporting it separately.
Expand Down
164 changes: 164 additions & 0 deletions tests/responses/responses-combo-bridge-usage.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { clearComboSelectionState, clearComboTargetCooldowns } from "../../src/combos";
import { clearKeyCooldowns } from "../../src/providers/key-failover";
import { createTranslatorBudget } from "../../src/lib/translator-budget";
import { handleComboResponses } from "../../src/server/responses/core";
import { executeComboResponses } from "../../src/server/responses/core-combo";
import type { HandleResponsesOptions } from "../../src/server/responses/core-options";
import type { RequestLogContext } from "../../src/server/request-log";
import type { OcxConfig, OcxUsage } from "../../src/types";
import { acquireOwnedSpendHome } from "../helpers/owned-spend-home";

/**
* A combo child logs into its own context and the combo copies it into the request context when
* the child commits. The bridge binds the adapter usage on the child, possibly before that commit
* and normally after it. Only the child that commits may hand its usage to the request context.
*/
let releaseSpendHome: (() => void) | undefined;
const reset = (): void => {
clearComboSelectionState();
clearComboTargetCooldowns();
clearKeyCooldowns();
};
beforeEach(() => {
reset();
releaseSpendHome = acquireOwnedSpendHome();
});
afterEach(() => {
releaseSpendHome?.();
releaseSpendHome = undefined;
reset();
});

const provider = (name: string) => ({
adapter: "openai-chat",
baseUrl: `https://${name}.example/v1`,
authMode: "key",
apiKey: `sk-${name}`,
models: [`model-${name}`],
});
const config = {
defaultProvider: "primary",
providers: { primary: provider("primary"), fallback: provider("fallback") },
combos: {
fan: {
strategy: "failover",
targets: [
{ provider: "primary", model: "model-primary" },
{ provider: "fallback", model: "model-fallback" },
],
},
},
} as unknown as OcxConfig;

const usageOf = (credits: number): OcxUsage => ({
inputTokens: 18, outputTokens: 2, providerCredits: credits, upstreamReported: true,
});

interface Child {
provider: string;
/** The request context the combo writes the row from. */
parent: RequestLogContext;
log: RequestLogContext;
options: HandleResponsesOptions;
}

/** Stands in for the bridge: binds usage on the child's own context, as bindKeyUsageFromBridge does. */
function bind(child: Child, usage: OcxUsage): void {
child.log.usageFromBridge = true;
child.log.usage = usage;
if (child.log.activeAttempt) child.log.activeAttempt.usage = usage;
child.options.onBridgeUsageBound?.();
}

async function runCombo(
answer: (child: Child) => Response | Promise<Response>,
options: Partial<HandleResponsesOptions> = {},
): Promise<{ response: Promise<Response>; logCtx: RequestLogContext; children: Child[] }> {
const children: Child[] = [];
const logCtx: RequestLogContext = { model: "", provider: "" };
const req = new Request("http://localhost/v1/responses", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ model: "combo/fan", stream: false, input: "hello" }),
});
const response = executeComboResponses(req, await req.clone().json(), "fan", config, logCtx, {
translatorBudget: createTranslatorBudget(),
...options,
}, {
handleComboResponses,
handleResponses: async (_req, _config, childLog, childOptions) => {
const child: Child = { provider: childLog.provider, parent: logCtx, log: childLog, options: childOptions ?? {} };
children.push(child);
return answer(child);
},
});
return { response, logCtx, children };
}

const ok = (): Response => Response.json({ status: "completed" });
const refused = (): Response => Response.json(
{ error: { message: "weekly usage limit reached", type: "rate_limit_error" } },
{ status: 429 },
);

describe("combo hands the committed child's bridge usage to the request context", () => {
test("usage bound after the commit reaches the request context", async () => {
const run = await runCombo(() => ok());
expect((await run.response).status).toBe(200);
expect(run.logCtx.usageFromBridge).toBeUndefined();

const usage = usageOf(0.04);
bind(run.children[0]!, usage);
expect(run.logCtx.usageFromBridge).toBe(true);
expect(run.logCtx.usage).toBe(usage);
expect(run.logCtx.activeAttempt?.usage).toBe(usage);
});

test("usage bound before the commit is still handed over once the child commits", async () => {
const usage = usageOf(0.05);
const run = await runCombo(child => {
bind(child, usage);
// The commit has not happened yet, so nothing may reach the request context from here.
expect(child.parent.usage).toBeUndefined();
return ok();
});
expect((await run.response).status).toBe(200);
expect(run.logCtx.usageFromBridge).toBe(true);
expect(run.logCtx.usage).toBe(usage);
});

test("a refused first target never lends its usage; only the committed target does", async () => {
const run = await runCombo(child => child.provider === "primary" ? refused() : ok());
expect((await run.response).status).toBe(200);
expect(run.children.map(child => child.provider)).toEqual(["primary", "fallback"]);
const [primary, fallback] = run.children as [Child, Child];

// The refused target's stream can still report after the fallback has committed.
bind(primary, usageOf(9.99));
expect(run.logCtx.usage).toBeUndefined();
expect(run.logCtx.usageFromBridge).toBeUndefined();

const usage = usageOf(0.04);
bind(fallback, usage);
expect(run.logCtx.usage).toBe(usage);
expect(run.logCtx.usage?.providerCredits).toBe(0.04);
expect(run.logCtx.usageFromBridge).toBe(true);
});

test("a commit that throws forwards nothing for that child", async () => {
// The child reported its terminal before the commit, so the commit publishes it to the
// caller's callback. A callback that throws aborts the commit.
const run = await runCombo(child => {
child.options.onNativePassthroughTerminal?.("completed");
return ok();
}, {
onNativePassthroughTerminal: () => { throw new Error("terminal sink failed"); },
});
await expect(run.response).rejects.toThrow("terminal sink failed");

bind(run.children[0]!, usageOf(0.07));
expect(run.logCtx.usage).toBeUndefined();
expect(run.logCtx.usageFromBridge).toBeUndefined();
});
});
Loading
Loading