From bc2f0adc0f0be9587267cf444ca6ebabea29c07e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 13 Aug 2026 23:35:22 -0600 Subject: [PATCH 1/4] fix(supervise): price a bridge turn that carried no provider receipt A cli-bridge turn whose provider reported no billed dollars reached the dollar channel as a known $0. Across 292 fleet runs `spentTotal.usd > 0` was true on none of them, so the dollar channel reported nothing on runs that certainly spent money. Price such a turn from the model catalog against that turn's OWN token counts, using the response `model` the bridge already sends on every chunk. The event always carries `usdKnown: false`, and the priced part rides a new `usdEstimated` on both the cost event and `Spend`, so `usd - usdEstimated` is what a provider is known to have billed. `assertValidSpend` refuses an estimated part that exceeds `usd` or that claims `usdKnown: true`. Only a turn that billed NOTHING is priced. A turn holding a partial receipt already put real dollars on the channel, and a whole-turn catalog price on top would charge the same tokens twice. An unpriced model contributes no dollars and leaves the turn unknown rather than free. The catalog holds one input rate and one output rate per model and no cache-read rate, so a prompt prefix served from cache is priced at the full input rate. That overstates a cache-heavy turn, which is the correct direction: an invented discount would understate spend. A dollar cap is unaffected. `observe` and `reconcile` still refuse unknown dollars under a `maxUsd` root, and an estimate rides `usdKnown: false`. --- docs/architecture.md | 9 + src/durable/spawn-journal.ts | 4 +- src/runtime/personify/trajectory.ts | 12 +- src/runtime/supervise/bridge-executor.test.ts | 63 ++++++- src/runtime/supervise/budget.ts | 21 +++ src/runtime/supervise/cost-estimate.ts | 46 +++++ src/runtime/supervise/runtime.ts | 32 +++- src/runtime/supervise/supervisor.ts | 4 +- src/runtime/supervise/types.ts | 14 ++ src/runtime/util.ts | 20 +++ tests/kernel/dollar-estimate.test.ts | 164 ++++++++++++++++++ .../supervise-full-profile-bridge.test.ts | 7 +- 12 files changed, 383 insertions(+), 13 deletions(-) create mode 100644 src/runtime/supervise/cost-estimate.ts create mode 100644 tests/kernel/dollar-estimate.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 42279f4a..b421479f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -392,6 +392,15 @@ It accepts every wire spelling the sandbox, cli-bridge, Anthropic, OpenAI, and D A counter the provider did not report stays absent. A zero would assert the provider measured no cache, which is a different fact from a provider that reported nothing. +The dollar channel keeps a receipt and a price apart. +A provider receipt is a dollar figure the provider billed; cli-bridge sends one only with `cost_known: true` and `provider-receipt` or `billing-receipt` provenance, and the claude harness reports one for the whole `claude -p` invocation. +A cli-bridge turn that carries NO receipt is priced from the model catalog (`estimateCost` in agent-eval) against that turn's own token counts, because a zero there reads as a measured free turn and made a fleet-wide dollar total report `$0` on runs that certainly spent money. +The priced part is carried in `Spend.usdEstimated`, so `usd - usdEstimated` is what a provider is known to have billed, and it is admitted only with `usdKnown: false` — a catalog price approximates what a provider would bill and never measures what it did. +An unpriced model contributes no dollars and leaves the turn unknown rather than free. +The catalog holds one input rate and one output rate per model and no cache-read rate, so a prompt prefix the provider served from cache is priced at the full input rate. +That overstates a cache-heavy turn, which is the correct direction: a discount the catalog cannot support would be invented, and an invented discount understates spend. +A dollar cap is unaffected — `observe` and `reconcile` still refuse unknown dollars under a `maxUsd` root, and an estimate rides `usdKnown: false`. + Two facts make this the whole game: - `spawn` **reserves** from one root total and refunds the unspent remainder on settle. A nested driver partitions only its reserved allocation, then reconciles the whole subtree once, so `Σk(treatment) ≡ Σk(blind)` by construction — no arm can buy more compute (`supervise/budget.ts`). diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index 42965bed..861aaf79 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -39,7 +39,7 @@ import type { TreeView, } from '../runtime/supervise/types' import type { PendingWait } from '../runtime/supervise/wait' -import { addTokenUsage, cloneTokenUsage, zeroTokenUsage } from '../runtime/util' +import { addTokenUsage, cloneTokenUsage, usdEstimatedOf, zeroTokenUsage } from '../runtime/util' import { contentAddress } from './content-address' import { parseCommittedJsonLines, prepareJsonlAppend, writeAllBytes } from './jsonl-file' @@ -1136,6 +1136,7 @@ function cloneJournalSpend(spend: Spend): Spend { ...(spend.tokensKnown === false ? { tokensKnown: false } : {}), usd: spend.usd, ...(spend.usdKnown === false ? { usdKnown: false } : {}), + ...(spend.usdEstimated !== undefined ? { usdEstimated: spend.usdEstimated } : {}), ms: spend.ms, } } @@ -1176,6 +1177,7 @@ function addJournalSpend(a: Spend, b: Spend): Spend { ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), + ...usdEstimatedOf(a, b), ms: a.ms + b.ms, } } diff --git a/src/runtime/personify/trajectory.ts b/src/runtime/personify/trajectory.ts index 3b980e75..b9673818 100644 --- a/src/runtime/personify/trajectory.ts +++ b/src/runtime/personify/trajectory.ts @@ -31,7 +31,13 @@ import type { SpawnJournal, Spend, } from '../supervise/types' -import { addTokenUsage, chargedTokens, cloneTokenUsage, zeroTokenUsage } from '../util' +import { + addTokenUsage, + chargedTokens, + cloneTokenUsage, + usdEstimatedOf, + zeroTokenUsage, +} from '../util' import type { EqualKArm, EqualKOnCostOptions, @@ -306,6 +312,7 @@ function addNodeSpend(a: Spend, b: Spend): Spend { ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), + ...usdEstimatedOf(a, b), ms: a.ms + b.ms, } } @@ -317,6 +324,7 @@ function cloneSpend(spend: Spend): Spend { ...(spend.tokensKnown === false ? { tokensKnown: false } : {}), usd: spend.usd, ...(spend.usdKnown === false ? { usdKnown: false } : {}), + ...(spend.usdEstimated !== undefined ? { usdEstimated: spend.usdEstimated } : {}), ms: spend.ms, } } @@ -328,6 +336,8 @@ function addSpend(acc: Spend, delta: Spend): void { if (delta.tokensKnown === false) acc.tokensKnown = false acc.usd += delta.usd if (delta.usdKnown === false) acc.usdKnown = false + if (delta.usdEstimated !== undefined) + acc.usdEstimated = (acc.usdEstimated ?? 0) + delta.usdEstimated acc.ms += delta.ms } diff --git a/src/runtime/supervise/bridge-executor.test.ts b/src/runtime/supervise/bridge-executor.test.ts index 03033402..ae8cb222 100644 --- a/src/runtime/supervise/bridge-executor.test.ts +++ b/src/runtime/supervise/bridge-executor.test.ts @@ -1,5 +1,6 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import type { AddressInfo } from 'node:net' +import { estimateCost } from '@tangle-network/agent-eval' import { type AgentProfile, canonicalAgentProfileDigest, @@ -862,7 +863,7 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(requestBody?.messages).toEqual([{ role: 'user', content: 'design the experiment' }]) }) - it('marks dollar cost unknown when the bridge reports no price', async () => { + it('prices a turn the bridge reported no price for, and keeps the dollars unknown', async () => { const stub = await startBridgeStub( `data: ${JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2 } })}\n\ndata: [DONE]\n\n`, ) @@ -871,15 +872,57 @@ describe('bridgeExecutor upstream-error propagation', () => { const events = await drain( executor.execute('do the task', new AbortController().signal) as AsyncIterable, ) - expect(events).toContainEqual({ kind: 'cost', usd: 0, usdKnown: false }) - expect(spendFromUsageEvents(events).usdKnown).toBe(false) + // glm rates over this turn's own 3 in / 2 out. The dollars reach the channel instead of a + // zero, and they carry the marker that says the catalog priced them. + const priced = estimateCost(3, 2, 'pi/tangle-router/glm-5.2') + expect(priced).toBeGreaterThan(0) + expect(events).toContainEqual({ + kind: 'cost', + usd: priced, + usdKnown: false, + usdEstimated: priced, + }) + const spend = spendFromUsageEvents(events) + expect(spend.usdKnown).toBe(false) + expect(spend.usd).toBe(priced) + expect(spend.usdEstimated).toBe(priced) expect(executor.resultArtifact().spent).toMatchObject({ tokens: { input: 3, output: 2 }, - usd: 0, + usd: priced, + usdEstimated: priced, usdKnown: false, }) }) + it('reports no dollars for an unpriced model rather than inventing a rate', async () => { + const stub = await startBridgeStub( + `data: ${JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2 } })}\n\ndata: [DONE]\n\n`, + ) + server = stub.server + const executor = bridgeExecutor( + { + profile: { + name: 'bridge-test-worker', + harness: 'pi', + model: { provider: 'in-house', default: 'no-such-model-family' }, + }, + harness: null, + }, + { + signal: new AbortController().signal, + seams: { bridge: { bridgeUrl: stub.url, bridgeBearer: 'test-bearer' } }, + }, + ) + const events = await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + expect(events).toContainEqual({ kind: 'cost', usd: 0, usdKnown: false }) + const spend = spendFromUsageEvents(events) + expect(spend.usd).toBe(0) + expect(spend.usdKnown).toBe(false) + expect(spend.usdEstimated).toBeUndefined() + }) + it('lets a trusted terminal total supersede an earlier incomplete cost chunk', async () => { const body = [ `data: ${JSON.stringify({ @@ -1030,10 +1073,14 @@ describe('bridgeExecutor upstream-error propagation', () => { ) expect(requests).toBe(2) + // Turn 1 billed a real $0.01. Turn 2 sent no receipt, so only turn 2's own 4 in / 1 out is + // priced — the receipt and the estimate stay separable in the settled spend. + const turn2 = estimateCost(4, 1, 'pi/tangle-router/glm-5.2') expect(executor.resultArtifact().spent).toMatchObject({ iterations: 2, tokens: { input: 7, output: 3 }, - usd: 0.01, + usd: 0.01 + turn2, + usdEstimated: turn2, usdKnown: false, }) }) @@ -1280,12 +1327,16 @@ describe('bridgeExecutor upstream-error propagation', () => { content: expect.stringContaining('stop and use the corrected method'), }, ]) + // The interrupted turn presented 5 in / 2 out and billed nothing, so it is priced. The + // resumed turn carried a real $0.01 receipt and is not priced on top of it. + const interruptedTurn = estimateCost(5, 2, 'pi/tangle-router/glm-5.2') expect(executor.resultArtifact()).toMatchObject({ out: { content: 'corrected answer' }, spent: { iterations: 2, tokens: { input: 8, output: 3 }, - usd: 0.01, + usd: 0.01 + interruptedTurn, + usdEstimated: interruptedTurn, usdKnown: false, }, }) diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index 402e9014..ceca2b5e 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -193,6 +193,21 @@ function assertValidSpend(spend: Spend, label: string): void { throw new Error(`${label}.${field} must be a non-negative finite number`) } } + if (spend.usdEstimated !== undefined) { + if (!Number.isFinite(spend.usdEstimated) || spend.usdEstimated < 0) { + throw new Error(`${label}.usdEstimated must be a non-negative finite number`) + } + // The catalog-priced part is a part OF `usd`, not an addition to it. A larger value would let + // `usd - usdEstimated` report negative provider-billed dollars. + if (spend.usdEstimated > spend.usd) { + throw new Error(`${label}.usdEstimated must not exceed ${label}.usd`) + } + // A catalog price is never a measurement. Admitting one under `usdKnown: true` would let an + // estimate be read as billed spend. + if (spend.usdKnown !== false) { + throw new Error(`${label}.usdEstimated requires ${label}.usdKnown false`) + } + } } export interface BudgetPool { @@ -240,6 +255,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { const tokens = zeroTokenUsage() let tokensKnown = true let usd = 0 + let usdEstimated = 0 let usdKnown = true let iterations = 0 for (const ev of events) { @@ -248,6 +264,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { if (ev.tokensKnown === false) tokensKnown = false } else if (ev.kind === 'cost') { usd += ev.usd + usdEstimated += ev.usdEstimated ?? 0 if (ev.usdKnown === false) usdKnown = false } else { iterations += 1 @@ -259,6 +276,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), + ...(usdEstimated > 0 ? { usdEstimated } : {}), ms: 0, } } @@ -268,6 +286,7 @@ async function foldUsage(events: AsyncIterable | UsageEvent[]): Prom const tokens = zeroTokenUsage() let tokensKnown = true let usd = 0 + let usdEstimated = 0 let usdKnown = true let iterations = 0 for await (const ev of events) { @@ -276,6 +295,7 @@ async function foldUsage(events: AsyncIterable | UsageEvent[]): Prom if (ev.tokensKnown === false) tokensKnown = false } else if (ev.kind === 'cost') { usd += ev.usd + usdEstimated += ev.usdEstimated ?? 0 if (ev.usdKnown === false) usdKnown = false } else { iterations += 1 @@ -287,6 +307,7 @@ async function foldUsage(events: AsyncIterable | UsageEvent[]): Prom ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), + ...(usdEstimated > 0 ? { usdEstimated } : {}), ms: 0, } } diff --git a/src/runtime/supervise/cost-estimate.ts b/src/runtime/supervise/cost-estimate.ts new file mode 100644 index 00000000..faea9e42 --- /dev/null +++ b/src/runtime/supervise/cost-estimate.ts @@ -0,0 +1,46 @@ +/** + * Pricing for work that arrived without a provider receipt. + * + * A turn whose provider reported no billed dollars used to reach the dollar channel as a + * known `$0`, so a run that certainly spent money reported a dollar total of zero. A catalog + * price is not a receipt, so what this module produces is always marked, never promoted. + */ + +import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' +import type { UsageEvent } from './types' + +export interface UnreceiptedWork { + /** + * The provider's whole prompt total for this work, cache reads included. + * + * The catalog carries ONE input rate per model and no cache-read rate, so a prefix the + * provider served from cache is priced at the full input rate. That overstates a + * cache-heavy turn. It is the correct direction: a discount the catalog cannot support + * would be invented, and an invented discount understates spend. + */ + inputTokens: number + outputTokens: number + /** The model the provider reported for this work. An unpriced or absent id yields no dollars. */ + model: string | undefined +} + +/** + * Price one unit of work that no provider receipt covered. + * + * The event always carries `usdKnown: false`: a catalog price approximates what a provider + * WOULD bill, never measures what it did. The priced amount is repeated in `usdEstimated` so + * a consumer can subtract it from `usd` and recover the dollars a provider actually billed. + * + * A model with no catalog entry yields `usd: 0` with `usdKnown: false` and no `usdEstimated`. + * The turn then reads as unknown dollars, which is true, rather than as a free turn. + */ +export function priceUnreceiptedWork(work: UnreceiptedWork): Extract { + const unknown = { kind: 'cost', usd: 0, usdKnown: false } as const + const { inputTokens, outputTokens, model } = work + if (model === undefined || !isModelPriced(model)) return unknown + if (!Number.isFinite(inputTokens) || !Number.isFinite(outputTokens)) return unknown + if (inputTokens < 0 || outputTokens < 0) return unknown + const usd = estimateCost(inputTokens, outputTokens, model) + if (!Number.isFinite(usd) || usd <= 0) return unknown + return { kind: 'cost', usd, usdKnown: false, usdEstimated: usd } +} diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index c93653b3..5484e111 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -92,6 +92,7 @@ import type { Validator, } from '../types' import { addTokenUsage, cloneTokenUsage, zeroTokenUsage } from '../util' +import { priceUnreceiptedWork } from './cost-estimate' import { executableAgentProfileSnapshot, executableAgentSpecSnapshot } from './executable-spec' import { createInbox, type Inbox } from './inbox' import { @@ -2292,6 +2293,9 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable { @@ -2465,6 +2473,8 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable = { kind: 'tokens', input: chunk.usage.input, @@ -2575,9 +2585,24 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable 0 ? { usdEstimated: estimatedUsdCharged } : {}), ms: Date.now() - started, } const out = { diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index cd4ce5d7..5c862dd4 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -44,7 +44,7 @@ import { replaySpawnTree, } from '../../durable/spawn-journal' import { RuntimeRunStateError } from '../../errors' -import { addTokenUsage, cloneTokenUsage, zeroTokenUsage } from '../util' +import { addTokenUsage, cloneTokenUsage, usdEstimatedOf, zeroTokenUsage } from '../util' import { type BudgetPool, createBudgetPool } from './budget' import { armDeadlineTimer } from './deadline' import { runTree } from './finalizer' @@ -1325,6 +1325,7 @@ function accumulate(a: Spend, b: Spend): void { if (b.tokensKnown === false) a.tokensKnown = false a.usd += b.usd if (b.usdKnown === false) a.usdKnown = false + if (b.usdEstimated !== undefined) a.usdEstimated = (a.usdEstimated ?? 0) + b.usdEstimated a.ms += b.ms } @@ -1341,6 +1342,7 @@ function addSpend(a: Spend, b: Spend): Spend { ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), + ...usdEstimatedOf(a, b), ms: a.ms + b.ms, } } diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 4eb9e51c..8588a899 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -250,6 +250,15 @@ export type UsageEvent = /** Known dollar subtotal. When false, `usd` must not be treated as total cost. */ usdKnown?: false usd: number + /** + * The part of `usd` this runtime priced from a model catalog because no provider receipt + * covered the work. Requires `usdKnown: false` — a catalog price approximates what a + * provider would bill and never measures what it did. + * + * Absence means this runtime priced nothing here, NOT that `usd` is a receipt. `usdKnown` + * is what says whether a dollar figure is measured. + */ + usdEstimated?: number } | { kind: 'iteration' } @@ -479,6 +488,11 @@ export interface Spend { * when enforcing a dollar-denominated comparison or limit. */ usdKnown?: boolean usd: number + /** The part of `usd` priced from a model catalog because no provider receipt covered the work. + * `usd - usdEstimated` is what a provider is known to have billed. Present only with + * `usdKnown: false`; absence means nothing here was catalog-priced, not that `usd` is + * measured. */ + usdEstimated?: number ms: number } diff --git a/src/runtime/util.ts b/src/runtime/util.ts index 0b5c4e7f..fbb70d37 100644 --- a/src/runtime/util.ts +++ b/src/runtime/util.ts @@ -137,6 +137,26 @@ export function zeroTokenUsage(): LoopTokenUsage { return { input: 0, output: 0 } } +/** + * Sum the catalog-priced part of a dollar total across spends, as a field to spread. + * + * Returns nothing when no input carried one. A fold of pure provider receipts must not gain a + * `usdEstimated: 0`, which would read as "this runtime checked and priced none" on a path that + * never prices at all. + */ +export function usdEstimatedOf(...spends: ReadonlyArray<{ usdEstimated?: number }>): { + usdEstimated?: number +} { + let total = 0 + let priced = false + for (const spend of spends) { + if (spend.usdEstimated === undefined) continue + priced = true + total += spend.usdEstimated + } + return priced ? { usdEstimated: total } : {} +} + /** Copy a token subtotal without dropping optional provider cache telemetry. */ export function cloneTokenUsage(usage: LoopTokenUsage): LoopTokenUsage { const cacheBreakdownUnknown = diff --git a/tests/kernel/dollar-estimate.test.ts b/tests/kernel/dollar-estimate.test.ts new file mode 100644 index 00000000..d780d32e --- /dev/null +++ b/tests/kernel/dollar-estimate.test.ts @@ -0,0 +1,164 @@ +import { estimateCost, MODEL_PRICING } from '@tangle-network/agent-eval' +import { describe, expect, it } from 'vitest' +import { createBudgetPool, spendFromUsageEvents } from '../../src/runtime/supervise/budget' +import { priceUnreceiptedWork } from '../../src/runtime/supervise/cost-estimate' +import type { Spend, UsageEvent } from '../../src/runtime/supervise/types' +import { usdEstimatedOf } from '../../src/runtime/util' + +const MODEL = 'claude-sonnet-4-20250514' +const RATE = MODEL_PRICING[MODEL]! + +function spend(over: Partial = {}): Spend { + return { iterations: 1, tokens: { input: 0, output: 0 }, usd: 0, ms: 0, ...over } +} + +/** An uncapped pool — the shape 286 of 292 fleet runs use, since they set no `maxUsd`. */ +function uncappedPool() { + return createBudgetPool({ maxIterations: 100, maxTokens: 10_000_000 }, () => 0) +} + +describe('pricing work that carried no provider receipt', () => { + it('prices from the catalog and marks the dollars as not measured', () => { + const event = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + const expected = 10 * RATE.input + 2 * RATE.output + expect(event).toEqual({ + kind: 'cost', + usd: expected, + usdKnown: false, + usdEstimated: expected, + }) + expect(expected).toBeGreaterThan(0) + }) + + it('prices a cached prefix at the full input rate, because the catalog carries no cache rate', () => { + // The catalog entry is `{ input, output }` per model. There is no cache-read rate to apply, + // so a prefix the provider served from cache is charged at the full input rate. That + // overstates a cache-heavy turn, which is the correct direction: a discount the catalog + // cannot support would be invented, and an invented discount understates spend. + expect(Object.keys(RATE).sort()).toEqual(['input', 'output']) + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + expect(priced.usd).toBe(estimateCost(10_000, 2_000, MODEL)) + }) + + it('reports unknown dollars, not a free turn, for a model the catalog does not price', () => { + const event = priceUnreceiptedWork({ + inputTokens: 10_000, + outputTokens: 2_000, + model: 'in-house/no-such-model-family', + }) + expect(event).toEqual({ kind: 'cost', usd: 0, usdKnown: false }) + expect(event.usdEstimated).toBeUndefined() + }) + + it('reports unknown dollars when no model was observed', () => { + expect(priceUnreceiptedWork({ inputTokens: 10, outputTokens: 5, model: undefined })).toEqual({ + kind: 'cost', + usd: 0, + usdKnown: false, + }) + }) + + it('refuses to price a negative or non-finite token count', () => { + for (const inputTokens of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(priceUnreceiptedWork({ inputTokens, outputTokens: 5, model: MODEL })).toEqual({ + kind: 'cost', + usd: 0, + usdKnown: false, + }) + } + }) +}) + +describe('a dollar total built from estimates', () => { + it('reaches the pool as a lower bound rather than a measurement', () => { + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + const events: UsageEvent[] = [ + { kind: 'tokens', input: 10_000, output: 2_000 }, + priced, + { kind: 'iteration' }, + ] + const folded = spendFromUsageEvents(events) + expect(folded.usd).toBe(priced.usd) + expect(folded.usdEstimated).toBe(priced.usd) + expect(folded.usdKnown).toBe(false) + + const pool = uncappedPool() + pool.observe(folded) + // The dollars are recorded, and the pool refuses to call the total a measurement. + expect(pool.readout().usdKnown).toBe(false) + }) + + it('keeps a receipt and an estimate separable through the fold', () => { + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + const folded = spendFromUsageEvents([ + { kind: 'cost', usd: 0.25 }, + priced, + { kind: 'iteration' }, + ]) + expect(folded.usd).toBe(0.25 + priced.usd) + expect(folded.usdEstimated).toBe(priced.usd) + // The provider-billed part stays recoverable, which is the point of carrying both. + expect(folded.usd - folded.usdEstimated!).toBeCloseTo(0.25, 10) + expect(folded.usdKnown).toBe(false) + }) + + it('leaves a pure-receipt fold with no estimated part at all', () => { + const folded = spendFromUsageEvents([{ kind: 'cost', usd: 0.25 }, { kind: 'iteration' }]) + expect(folded.usd).toBe(0.25) + expect(folded.usdEstimated).toBeUndefined() + expect(folded.usdKnown).toBeUndefined() + }) + + it('sums the estimated part across merged spends, and reports none when nothing was priced', () => { + expect(usdEstimatedOf({ usdEstimated: 0.5 }, { usdEstimated: 0.25 })).toEqual({ + usdEstimated: 0.75, + }) + expect(usdEstimatedOf({ usdEstimated: 0.5 }, {})).toEqual({ usdEstimated: 0.5 }) + expect(usdEstimatedOf({}, {})).toEqual({}) + }) +}) + +describe('the estimated part may never be read as billed spend', () => { + it('refuses an estimated part on a spend that claims its dollars are known', () => { + const pool = uncappedPool() + expect(() => pool.observe(spend({ usd: 1, usdEstimated: 1 }))).toThrow( + /usdEstimated requires observed spend.usdKnown false/, + ) + expect(() => pool.observe(spend({ usd: 1, usdEstimated: 1, usdKnown: true }))).toThrow( + /usdEstimated requires observed spend.usdKnown false/, + ) + }) + + it('refuses an estimated part larger than the total it is a part of', () => { + expect(() => + uncappedPool().observe(spend({ usd: 1, usdEstimated: 1.5, usdKnown: false })), + ).toThrow(/usdEstimated must not exceed observed spend.usd/) + }) + + it('refuses a negative or non-finite estimated part', () => { + for (const usdEstimated of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => + uncappedPool().observe(spend({ usd: 10, usdEstimated, usdKnown: false })), + ).toThrow(/usdEstimated must be a non-negative finite number/) + } + }) + + it('accepts a well-formed estimated part', () => { + expect(() => + uncappedPool().observe(spend({ usd: 1, usdEstimated: 0.4, usdKnown: false })), + ).not.toThrow() + }) + + it('still refuses unknown dollars under a dollar-capped root', () => { + // Pricing an estimate does not open a dollar cap. A capped root refuses work whose dollars + // are not measured, exactly as before, because the estimate rides `usdKnown: false`. + const capped = createBudgetPool( + { maxIterations: 10, maxTokens: 1_000_000, maxUsd: 100 }, + () => 0, + ) + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + expect(() => + capped.observe(spend({ usd: priced.usd, usdEstimated: priced.usd, usdKnown: false })), + ).toThrow(/cannot observe unknown dollar cost under a dollar-capped budget/) + }) +}) diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts index b8220f66..d71b2f19 100644 --- a/tests/kernel/supervise-full-profile-bridge.test.ts +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -1396,7 +1396,12 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( if (result.reason === 'driver-failed') { expect(result.error.message).toMatch(/unknown dollar cost under a dollar-capped budget/) } - expect(result.spentTotal).toMatchObject({ usd: 0, usdKnown: false }) + // The turn carried no receipt, so its dollars are the catalog price of what it presented — + // recorded, and explicitly not a measurement. Pricing an estimate does not open the cap: the + // refusal above is unchanged. + expect(result.spentTotal.usdKnown).toBe(false) + expect(result.spentTotal.usd).toBeGreaterThan(0) + expect(result.spentTotal.usdEstimated).toBe(result.spentTotal.usd) }) it('records a manager with unknown token usage as unknown telemetry without ending the run', async () => { From 655b30d697f815459c5b641917d953048450f11e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 13 Aug 2026 23:50:48 -0600 Subject: [PATCH 2/4] docs(api): regenerate the API reference for the estimated-dollar field --- docs/api/index.md | 9 +++++++++ docs/api/runtime.md | 15 +++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 6d715ac7..bc27fc5b 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -8484,6 +8484,15 @@ Dollar accounting is known unless explicitly false. A false value must not be tr > **usd**: `number` +##### usdEstimated? + +> `optional` **usdEstimated?**: `number` + +The part of `usd` priced from a model catalog because no provider receipt covered the work. + `usd - usdEstimated` is what a provider is known to have billed. Present only with + `usdKnown: false`; absence means nothing here was catalog-priced, not that `usd` is + measured. + ##### ms > **ms**: `number` diff --git a/docs/api/runtime.md b/docs/api/runtime.md index cd77ba5d..955590e8 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -19938,7 +19938,7 @@ Resolve an external harness for one exact Runtime-owned manager identity. ### UsageEvent -> **UsageEvent** = \{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; `freshInput?`: `number`; `cacheRead?`: `number`; `cacheWrite?`: `number`; `cacheBreakdownKnown?`: `false`; \} \| \{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; \} \| \{ `kind`: `"iteration"`; \} +> **UsageEvent** = \{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; `freshInput?`: `number`; `cacheRead?`: `number`; `cacheWrite?`: `number`; `cacheBreakdownKnown?`: `false`; \} \| \{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; `usdEstimated?`: `number`; \} \| \{ `kind`: `"iteration"`; \} Normalized usage event — the single channel every executor reports through, so the conserved pool meters all runtimes identically. `tokens` carries `LoopTokenUsage`'s @@ -20004,7 +20004,7 @@ them is an upper bound. A counter the provider did not report is absent, never z ##### Type Literal -\{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; \} +\{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; `usdEstimated?`: `number`; \} ###### kind @@ -20020,6 +20020,17 @@ Known dollar subtotal. When false, `usd` must not be treated as total cost. > **usd**: `number` +###### usdEstimated? + +> `optional` **usdEstimated?**: `number` + +The part of `usd` this runtime priced from a model catalog because no provider receipt +covered the work. Requires `usdKnown: false` — a catalog price approximates what a +provider would bill and never measures what it did. + +Absence means this runtime priced nothing here, NOT that `usd` is a receipt. `usdKnown` +is what says whether a dollar figure is measured. + *** ##### Type Literal From 4cc0f3f819710edce9729dd6c8ae9902dd20384d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 13 Aug 2026 23:54:05 -0600 Subject: [PATCH 3/4] fix(budget): diagnose unmeasured dollars before comparing them to a ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `reconcile` compared a child's dollars against its reservation before it checked whether those dollars were measured at all. A catalog-priced turn can exceed the ceiling, so an unreceipted child was reported as spending more than it reserved — a claim that the child spent dollars a provider billed. Decide `unknownUnderCap` first. Dollars that are not measured have no business being compared to a dollar reservation, and the accurate diagnosis is the unknown-cost refusal. Both paths still throw and still close dollar admission, so no run outcome changes; the reason a caller reads does. --- src/runtime/supervise/budget.ts | 11 +++++++---- tests/kernel/dollar-estimate.test.ts | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index ceca2b5e..b96216e6 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -443,16 +443,19 @@ export function createBudgetPool( violation = `ticket ${ticket.id} spent ${spentTokens} tokens > reserved ${rTokens}` } else if (spent.iterations > rIterations) { violation = `ticket ${ticket.id} spent ${spent.iterations} iterations > reserved ${rIterations}` + } else if (unknownUnderCap) { + // Decided BEFORE the dollar comparison below. Dollars that are not measured may not be + // compared against a reservation as if they were billed: a catalog-priced turn can exceed + // the ceiling and would then be reported as an overspend the child never made. The known + // channels still settle, then the dollar channel is permanently tainted and admission + // closes. + violation = `ticket ${ticket.id} reported unknown dollar cost under a dollar-capped budget` } else if (usdCapped && usdBudgeted && spent.usd > rUsd) { // USD is conserved ONLY when the root declared a ceiling AND the child declared one to // be measured against. `maxUsd` is optional on both: when either is unset, usd is an // OBSERVED quantity (committed for accounting), never a budgeted constraint — an unset // ceiling must not behave as a hard $0 limit that fail-closes a real priced spend. violation = `ticket ${ticket.id} spent $${spent.usd} > reserved $${rUsd}` - } else if (unknownUnderCap) { - // The known channels still settle, then the dollar channel is permanently tainted and - // admission closes. - violation = `ticket ${ticket.id} reported unknown dollar cost under a dollar-capped budget` } // ── Settlement: unconditional, and the only place the ticket closes ─────────────── diff --git a/tests/kernel/dollar-estimate.test.ts b/tests/kernel/dollar-estimate.test.ts index d780d32e..3f3bad93 100644 --- a/tests/kernel/dollar-estimate.test.ts +++ b/tests/kernel/dollar-estimate.test.ts @@ -149,6 +149,24 @@ describe('the estimated part may never be read as billed spend', () => { ).not.toThrow() }) + it('diagnoses an unreceipted child as unknown dollars, not as overspending its ceiling', () => { + // A catalog price can exceed a child's declared dollar ceiling. Reporting that as an + // overspend would assert the child spent dollars a provider billed, which is the exact + // confusion this field exists to prevent. + const pool = createBudgetPool({ maxIterations: 10, maxTokens: 1_000_000, maxUsd: 100 }, () => 0) + const reserved = pool.reserve({ maxIterations: 2, maxTokens: 1_000, maxUsd: 0.001 }) + expect(reserved.ok).toBe(true) + if (!reserved.ok) return + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + expect(priced.usd).toBeGreaterThan(0.001) + expect(() => + pool.reconcile( + reserved.ticket, + spend({ usd: priced.usd, usdEstimated: priced.usd, usdKnown: false }), + ), + ).toThrow(/reported unknown dollar cost under a dollar-capped budget/) + }) + it('still refuses unknown dollars under a dollar-capped root', () => { // Pricing an estimate does not open a dollar cap. A capped root refuses work whose dollars // are not measured, exactly as before, because the estimate rides `usdKnown: false`. From 8a1c432205f26600c5d7e7ac5cf31468228fb59e Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 13 Aug 2026 23:57:24 -0600 Subject: [PATCH 4/4] test(bridge): pin the claude receipt wire against the dollar channel Neither repo covered the seam between them. This drives the bridge executor with the exact usage object cli-bridge emits for a claude turn carrying total_cost_usd, captured off deltaToOpenAIChunk: tokens and the receipt in ONE frame. The turn must settle as measured dollars with no estimated part. A receipt is a measurement, and a turn holding one is never catalog-priced on top. --- src/runtime/supervise/bridge-executor.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/runtime/supervise/bridge-executor.test.ts b/src/runtime/supervise/bridge-executor.test.ts index ae8cb222..530ea6cc 100644 --- a/src/runtime/supervise/bridge-executor.test.ts +++ b/src/runtime/supervise/bridge-executor.test.ts @@ -956,6 +956,39 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(executor.resultArtifact().spent.usdKnown).not.toBe(false) }) + it('takes a claude invocation receipt as measured dollars, with no estimated part', async () => { + // The EXACT usage object cli-bridge emits for a claude turn carrying `total_cost_usd` + // (drewstone/cli-bridge#159), captured off `deltaToOpenAIChunk`. Tokens and the receipt + // arrive in ONE frame, which is the shape that must not be re-priced. + const body = `data: ${JSON.stringify({ + choices: [{ delta: {}, finish_reason: 'stop' }], + usage: { + prompt_tokens: 12_000, + completion_tokens: 900, + total_tokens: 12_900, + cost: 0.0731, + cost_known: true, + cost_provenance: 'provider-receipt', + cost_scope: 'total', + }, + })}\n\ndata: [DONE]\n\n` + const stub = await startBridgeStub(body) + server = stub.server + const executor = makeExecutor(stub.url) + + const events = await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + expect(events).toContainEqual({ kind: 'cost', usd: 0.0731 }) + const spent = executor.resultArtifact().spent + expect(spent).toMatchObject({ tokens: { input: 12_000, output: 900 }, usd: 0.0731 }) + // A receipt is a measurement, and a turn holding one is never catalog-priced on top. + expect(spent.usdKnown).not.toBe(false) + expect(spent.usdEstimated).toBeUndefined() + expect(spendFromUsageEvents(events).usdEstimated).toBeUndefined() + }) + it('preserves absent prompt-cache fields instead of inventing zeroes', async () => { const body = `data: ${JSON.stringify({ usage: {