diff --git a/docs/api/index.md b/docs/api/index.md index 6d715ac7..bc27fc5b 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -8484,6 +8484,15 @@ Dollar accounting is known unless explicitly false. A false value must not be tr > **usd**: `number` +##### usdEstimated? + +> `optional` **usdEstimated?**: `number` + +The part of `usd` priced from a model catalog because no provider receipt covered the work. + `usd - usdEstimated` is what a provider is known to have billed. Present only with + `usdKnown: false`; absence means nothing here was catalog-priced, not that `usd` is + measured. + ##### ms > **ms**: `number` diff --git a/docs/api/runtime.md b/docs/api/runtime.md index cd77ba5d..955590e8 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -19938,7 +19938,7 @@ Resolve an external harness for one exact Runtime-owned manager identity. ### UsageEvent -> **UsageEvent** = \{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; `freshInput?`: `number`; `cacheRead?`: `number`; `cacheWrite?`: `number`; `cacheBreakdownKnown?`: `false`; \} \| \{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; \} \| \{ `kind`: `"iteration"`; \} +> **UsageEvent** = \{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; `freshInput?`: `number`; `cacheRead?`: `number`; `cacheWrite?`: `number`; `cacheBreakdownKnown?`: `false`; \} \| \{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; `usdEstimated?`: `number`; \} \| \{ `kind`: `"iteration"`; \} Normalized usage event — the single channel every executor reports through, so the conserved pool meters all runtimes identically. `tokens` carries `LoopTokenUsage`'s @@ -20004,7 +20004,7 @@ them is an upper bound. A counter the provider did not report is absent, never z ##### Type Literal -\{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; \} +\{ `kind`: `"cost"`; `usdKnown?`: `false`; `usd`: `number`; `usdEstimated?`: `number`; \} ###### kind @@ -20020,6 +20020,17 @@ Known dollar subtotal. When false, `usd` must not be treated as total cost. > **usd**: `number` +###### usdEstimated? + +> `optional` **usdEstimated?**: `number` + +The part of `usd` this runtime priced from a model catalog because no provider receipt +covered the work. Requires `usdKnown: false` — a catalog price approximates what a +provider would bill and never measures what it did. + +Absence means this runtime priced nothing here, NOT that `usd` is a receipt. `usdKnown` +is what says whether a dollar figure is measured. + *** ##### Type Literal diff --git a/docs/architecture.md b/docs/architecture.md index 42279f4a..b421479f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -392,6 +392,15 @@ It accepts every wire spelling the sandbox, cli-bridge, Anthropic, OpenAI, and D A counter the provider did not report stays absent. A zero would assert the provider measured no cache, which is a different fact from a provider that reported nothing. +The dollar channel keeps a receipt and a price apart. +A provider receipt is a dollar figure the provider billed; cli-bridge sends one only with `cost_known: true` and `provider-receipt` or `billing-receipt` provenance, and the claude harness reports one for the whole `claude -p` invocation. +A cli-bridge turn that carries NO receipt is priced from the model catalog (`estimateCost` in agent-eval) against that turn's own token counts, because a zero there reads as a measured free turn and made a fleet-wide dollar total report `$0` on runs that certainly spent money. +The priced part is carried in `Spend.usdEstimated`, so `usd - usdEstimated` is what a provider is known to have billed, and it is admitted only with `usdKnown: false` — a catalog price approximates what a provider would bill and never measures what it did. +An unpriced model contributes no dollars and leaves the turn unknown rather than free. +The catalog holds one input rate and one output rate per model and no cache-read rate, so a prompt prefix the provider served from cache is priced at the full input rate. +That overstates a cache-heavy turn, which is the correct direction: a discount the catalog cannot support would be invented, and an invented discount understates spend. +A dollar cap is unaffected — `observe` and `reconcile` still refuse unknown dollars under a `maxUsd` root, and an estimate rides `usdKnown: false`. + Two facts make this the whole game: - `spawn` **reserves** from one root total and refunds the unspent remainder on settle. A nested driver partitions only its reserved allocation, then reconciles the whole subtree once, so `Σk(treatment) ≡ Σk(blind)` by construction — no arm can buy more compute (`supervise/budget.ts`). diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index 42965bed..861aaf79 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -39,7 +39,7 @@ import type { TreeView, } from '../runtime/supervise/types' import type { PendingWait } from '../runtime/supervise/wait' -import { addTokenUsage, cloneTokenUsage, zeroTokenUsage } from '../runtime/util' +import { addTokenUsage, cloneTokenUsage, usdEstimatedOf, zeroTokenUsage } from '../runtime/util' import { contentAddress } from './content-address' import { parseCommittedJsonLines, prepareJsonlAppend, writeAllBytes } from './jsonl-file' @@ -1136,6 +1136,7 @@ function cloneJournalSpend(spend: Spend): Spend { ...(spend.tokensKnown === false ? { tokensKnown: false } : {}), usd: spend.usd, ...(spend.usdKnown === false ? { usdKnown: false } : {}), + ...(spend.usdEstimated !== undefined ? { usdEstimated: spend.usdEstimated } : {}), ms: spend.ms, } } @@ -1176,6 +1177,7 @@ function addJournalSpend(a: Spend, b: Spend): Spend { ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), + ...usdEstimatedOf(a, b), ms: a.ms + b.ms, } } diff --git a/src/runtime/personify/trajectory.ts b/src/runtime/personify/trajectory.ts index 3b980e75..b9673818 100644 --- a/src/runtime/personify/trajectory.ts +++ b/src/runtime/personify/trajectory.ts @@ -31,7 +31,13 @@ import type { SpawnJournal, Spend, } from '../supervise/types' -import { addTokenUsage, chargedTokens, cloneTokenUsage, zeroTokenUsage } from '../util' +import { + addTokenUsage, + chargedTokens, + cloneTokenUsage, + usdEstimatedOf, + zeroTokenUsage, +} from '../util' import type { EqualKArm, EqualKOnCostOptions, @@ -306,6 +312,7 @@ function addNodeSpend(a: Spend, b: Spend): Spend { ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), + ...usdEstimatedOf(a, b), ms: a.ms + b.ms, } } @@ -317,6 +324,7 @@ function cloneSpend(spend: Spend): Spend { ...(spend.tokensKnown === false ? { tokensKnown: false } : {}), usd: spend.usd, ...(spend.usdKnown === false ? { usdKnown: false } : {}), + ...(spend.usdEstimated !== undefined ? { usdEstimated: spend.usdEstimated } : {}), ms: spend.ms, } } @@ -328,6 +336,8 @@ function addSpend(acc: Spend, delta: Spend): void { if (delta.tokensKnown === false) acc.tokensKnown = false acc.usd += delta.usd if (delta.usdKnown === false) acc.usdKnown = false + if (delta.usdEstimated !== undefined) + acc.usdEstimated = (acc.usdEstimated ?? 0) + delta.usdEstimated acc.ms += delta.ms } diff --git a/src/runtime/supervise/bridge-executor.test.ts b/src/runtime/supervise/bridge-executor.test.ts index 03033402..530ea6cc 100644 --- a/src/runtime/supervise/bridge-executor.test.ts +++ b/src/runtime/supervise/bridge-executor.test.ts @@ -1,5 +1,6 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import type { AddressInfo } from 'node:net' +import { estimateCost } from '@tangle-network/agent-eval' import { type AgentProfile, canonicalAgentProfileDigest, @@ -862,7 +863,7 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(requestBody?.messages).toEqual([{ role: 'user', content: 'design the experiment' }]) }) - it('marks dollar cost unknown when the bridge reports no price', async () => { + it('prices a turn the bridge reported no price for, and keeps the dollars unknown', async () => { const stub = await startBridgeStub( `data: ${JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2 } })}\n\ndata: [DONE]\n\n`, ) @@ -871,15 +872,57 @@ describe('bridgeExecutor upstream-error propagation', () => { const events = await drain( executor.execute('do the task', new AbortController().signal) as AsyncIterable, ) - expect(events).toContainEqual({ kind: 'cost', usd: 0, usdKnown: false }) - expect(spendFromUsageEvents(events).usdKnown).toBe(false) + // glm rates over this turn's own 3 in / 2 out. The dollars reach the channel instead of a + // zero, and they carry the marker that says the catalog priced them. + const priced = estimateCost(3, 2, 'pi/tangle-router/glm-5.2') + expect(priced).toBeGreaterThan(0) + expect(events).toContainEqual({ + kind: 'cost', + usd: priced, + usdKnown: false, + usdEstimated: priced, + }) + const spend = spendFromUsageEvents(events) + expect(spend.usdKnown).toBe(false) + expect(spend.usd).toBe(priced) + expect(spend.usdEstimated).toBe(priced) expect(executor.resultArtifact().spent).toMatchObject({ tokens: { input: 3, output: 2 }, - usd: 0, + usd: priced, + usdEstimated: priced, usdKnown: false, }) }) + it('reports no dollars for an unpriced model rather than inventing a rate', async () => { + const stub = await startBridgeStub( + `data: ${JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2 } })}\n\ndata: [DONE]\n\n`, + ) + server = stub.server + const executor = bridgeExecutor( + { + profile: { + name: 'bridge-test-worker', + harness: 'pi', + model: { provider: 'in-house', default: 'no-such-model-family' }, + }, + harness: null, + }, + { + signal: new AbortController().signal, + seams: { bridge: { bridgeUrl: stub.url, bridgeBearer: 'test-bearer' } }, + }, + ) + const events = await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + expect(events).toContainEqual({ kind: 'cost', usd: 0, usdKnown: false }) + const spend = spendFromUsageEvents(events) + expect(spend.usd).toBe(0) + expect(spend.usdKnown).toBe(false) + expect(spend.usdEstimated).toBeUndefined() + }) + it('lets a trusted terminal total supersede an earlier incomplete cost chunk', async () => { const body = [ `data: ${JSON.stringify({ @@ -913,6 +956,39 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(executor.resultArtifact().spent.usdKnown).not.toBe(false) }) + it('takes a claude invocation receipt as measured dollars, with no estimated part', async () => { + // The EXACT usage object cli-bridge emits for a claude turn carrying `total_cost_usd` + // (drewstone/cli-bridge#159), captured off `deltaToOpenAIChunk`. Tokens and the receipt + // arrive in ONE frame, which is the shape that must not be re-priced. + const body = `data: ${JSON.stringify({ + choices: [{ delta: {}, finish_reason: 'stop' }], + usage: { + prompt_tokens: 12_000, + completion_tokens: 900, + total_tokens: 12_900, + cost: 0.0731, + cost_known: true, + cost_provenance: 'provider-receipt', + cost_scope: 'total', + }, + })}\n\ndata: [DONE]\n\n` + const stub = await startBridgeStub(body) + server = stub.server + const executor = makeExecutor(stub.url) + + const events = await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + expect(events).toContainEqual({ kind: 'cost', usd: 0.0731 }) + const spent = executor.resultArtifact().spent + expect(spent).toMatchObject({ tokens: { input: 12_000, output: 900 }, usd: 0.0731 }) + // A receipt is a measurement, and a turn holding one is never catalog-priced on top. + expect(spent.usdKnown).not.toBe(false) + expect(spent.usdEstimated).toBeUndefined() + expect(spendFromUsageEvents(events).usdEstimated).toBeUndefined() + }) + it('preserves absent prompt-cache fields instead of inventing zeroes', async () => { const body = `data: ${JSON.stringify({ usage: { @@ -1030,10 +1106,14 @@ describe('bridgeExecutor upstream-error propagation', () => { ) expect(requests).toBe(2) + // Turn 1 billed a real $0.01. Turn 2 sent no receipt, so only turn 2's own 4 in / 1 out is + // priced — the receipt and the estimate stay separable in the settled spend. + const turn2 = estimateCost(4, 1, 'pi/tangle-router/glm-5.2') expect(executor.resultArtifact().spent).toMatchObject({ iterations: 2, tokens: { input: 7, output: 3 }, - usd: 0.01, + usd: 0.01 + turn2, + usdEstimated: turn2, usdKnown: false, }) }) @@ -1280,12 +1360,16 @@ describe('bridgeExecutor upstream-error propagation', () => { content: expect.stringContaining('stop and use the corrected method'), }, ]) + // The interrupted turn presented 5 in / 2 out and billed nothing, so it is priced. The + // resumed turn carried a real $0.01 receipt and is not priced on top of it. + const interruptedTurn = estimateCost(5, 2, 'pi/tangle-router/glm-5.2') expect(executor.resultArtifact()).toMatchObject({ out: { content: 'corrected answer' }, spent: { iterations: 2, tokens: { input: 8, output: 3 }, - usd: 0.01, + usd: 0.01 + interruptedTurn, + usdEstimated: interruptedTurn, usdKnown: false, }, }) diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index 402e9014..b96216e6 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -193,6 +193,21 @@ function assertValidSpend(spend: Spend, label: string): void { throw new Error(`${label}.${field} must be a non-negative finite number`) } } + if (spend.usdEstimated !== undefined) { + if (!Number.isFinite(spend.usdEstimated) || spend.usdEstimated < 0) { + throw new Error(`${label}.usdEstimated must be a non-negative finite number`) + } + // The catalog-priced part is a part OF `usd`, not an addition to it. A larger value would let + // `usd - usdEstimated` report negative provider-billed dollars. + if (spend.usdEstimated > spend.usd) { + throw new Error(`${label}.usdEstimated must not exceed ${label}.usd`) + } + // A catalog price is never a measurement. Admitting one under `usdKnown: true` would let an + // estimate be read as billed spend. + if (spend.usdKnown !== false) { + throw new Error(`${label}.usdEstimated requires ${label}.usdKnown false`) + } + } } export interface BudgetPool { @@ -240,6 +255,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { const tokens = zeroTokenUsage() let tokensKnown = true let usd = 0 + let usdEstimated = 0 let usdKnown = true let iterations = 0 for (const ev of events) { @@ -248,6 +264,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { if (ev.tokensKnown === false) tokensKnown = false } else if (ev.kind === 'cost') { usd += ev.usd + usdEstimated += ev.usdEstimated ?? 0 if (ev.usdKnown === false) usdKnown = false } else { iterations += 1 @@ -259,6 +276,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), + ...(usdEstimated > 0 ? { usdEstimated } : {}), ms: 0, } } @@ -268,6 +286,7 @@ async function foldUsage(events: AsyncIterable | UsageEvent[]): Prom const tokens = zeroTokenUsage() let tokensKnown = true let usd = 0 + let usdEstimated = 0 let usdKnown = true let iterations = 0 for await (const ev of events) { @@ -276,6 +295,7 @@ async function foldUsage(events: AsyncIterable | UsageEvent[]): Prom if (ev.tokensKnown === false) tokensKnown = false } else if (ev.kind === 'cost') { usd += ev.usd + usdEstimated += ev.usdEstimated ?? 0 if (ev.usdKnown === false) usdKnown = false } else { iterations += 1 @@ -287,6 +307,7 @@ async function foldUsage(events: AsyncIterable | UsageEvent[]): Prom ...(tokensKnown ? {} : { tokensKnown: false }), usd, ...(usdKnown ? {} : { usdKnown: false }), + ...(usdEstimated > 0 ? { usdEstimated } : {}), ms: 0, } } @@ -422,16 +443,19 @@ export function createBudgetPool( violation = `ticket ${ticket.id} spent ${spentTokens} tokens > reserved ${rTokens}` } else if (spent.iterations > rIterations) { violation = `ticket ${ticket.id} spent ${spent.iterations} iterations > reserved ${rIterations}` + } else if (unknownUnderCap) { + // Decided BEFORE the dollar comparison below. Dollars that are not measured may not be + // compared against a reservation as if they were billed: a catalog-priced turn can exceed + // the ceiling and would then be reported as an overspend the child never made. The known + // channels still settle, then the dollar channel is permanently tainted and admission + // closes. + violation = `ticket ${ticket.id} reported unknown dollar cost under a dollar-capped budget` } else if (usdCapped && usdBudgeted && spent.usd > rUsd) { // USD is conserved ONLY when the root declared a ceiling AND the child declared one to // be measured against. `maxUsd` is optional on both: when either is unset, usd is an // OBSERVED quantity (committed for accounting), never a budgeted constraint — an unset // ceiling must not behave as a hard $0 limit that fail-closes a real priced spend. violation = `ticket ${ticket.id} spent $${spent.usd} > reserved $${rUsd}` - } else if (unknownUnderCap) { - // The known channels still settle, then the dollar channel is permanently tainted and - // admission closes. - violation = `ticket ${ticket.id} reported unknown dollar cost under a dollar-capped budget` } // ── Settlement: unconditional, and the only place the ticket closes ─────────────── diff --git a/src/runtime/supervise/cost-estimate.ts b/src/runtime/supervise/cost-estimate.ts new file mode 100644 index 00000000..faea9e42 --- /dev/null +++ b/src/runtime/supervise/cost-estimate.ts @@ -0,0 +1,46 @@ +/** + * Pricing for work that arrived without a provider receipt. + * + * A turn whose provider reported no billed dollars used to reach the dollar channel as a + * known `$0`, so a run that certainly spent money reported a dollar total of zero. A catalog + * price is not a receipt, so what this module produces is always marked, never promoted. + */ + +import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' +import type { UsageEvent } from './types' + +export interface UnreceiptedWork { + /** + * The provider's whole prompt total for this work, cache reads included. + * + * The catalog carries ONE input rate per model and no cache-read rate, so a prefix the + * provider served from cache is priced at the full input rate. That overstates a + * cache-heavy turn. It is the correct direction: a discount the catalog cannot support + * would be invented, and an invented discount understates spend. + */ + inputTokens: number + outputTokens: number + /** The model the provider reported for this work. An unpriced or absent id yields no dollars. */ + model: string | undefined +} + +/** + * Price one unit of work that no provider receipt covered. + * + * The event always carries `usdKnown: false`: a catalog price approximates what a provider + * WOULD bill, never measures what it did. The priced amount is repeated in `usdEstimated` so + * a consumer can subtract it from `usd` and recover the dollars a provider actually billed. + * + * A model with no catalog entry yields `usd: 0` with `usdKnown: false` and no `usdEstimated`. + * The turn then reads as unknown dollars, which is true, rather than as a free turn. + */ +export function priceUnreceiptedWork(work: UnreceiptedWork): Extract { + const unknown = { kind: 'cost', usd: 0, usdKnown: false } as const + const { inputTokens, outputTokens, model } = work + if (model === undefined || !isModelPriced(model)) return unknown + if (!Number.isFinite(inputTokens) || !Number.isFinite(outputTokens)) return unknown + if (inputTokens < 0 || outputTokens < 0) return unknown + const usd = estimateCost(inputTokens, outputTokens, model) + if (!Number.isFinite(usd) || usd <= 0) return unknown + return { kind: 'cost', usd, usdKnown: false, usdEstimated: usd } +} diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index c93653b3..5484e111 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -92,6 +92,7 @@ import type { Validator, } from '../types' import { addTokenUsage, cloneTokenUsage, zeroTokenUsage } from '../util' +import { priceUnreceiptedWork } from './cost-estimate' import { executableAgentProfileSnapshot, executableAgentSpecSnapshot } from './executable-spec' import { createInbox, type Inbox } from './inbox' import { @@ -2292,6 +2293,9 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable { @@ -2465,6 +2473,8 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable = { kind: 'tokens', input: chunk.usage.input, @@ -2575,9 +2585,24 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable 0 ? { usdEstimated: estimatedUsdCharged } : {}), ms: Date.now() - started, } const out = { diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index cd4ce5d7..5c862dd4 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -44,7 +44,7 @@ import { replaySpawnTree, } from '../../durable/spawn-journal' import { RuntimeRunStateError } from '../../errors' -import { addTokenUsage, cloneTokenUsage, zeroTokenUsage } from '../util' +import { addTokenUsage, cloneTokenUsage, usdEstimatedOf, zeroTokenUsage } from '../util' import { type BudgetPool, createBudgetPool } from './budget' import { armDeadlineTimer } from './deadline' import { runTree } from './finalizer' @@ -1325,6 +1325,7 @@ function accumulate(a: Spend, b: Spend): void { if (b.tokensKnown === false) a.tokensKnown = false a.usd += b.usd if (b.usdKnown === false) a.usdKnown = false + if (b.usdEstimated !== undefined) a.usdEstimated = (a.usdEstimated ?? 0) + b.usdEstimated a.ms += b.ms } @@ -1341,6 +1342,7 @@ function addSpend(a: Spend, b: Spend): Spend { ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), + ...usdEstimatedOf(a, b), ms: a.ms + b.ms, } } diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 4eb9e51c..8588a899 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -250,6 +250,15 @@ export type UsageEvent = /** Known dollar subtotal. When false, `usd` must not be treated as total cost. */ usdKnown?: false usd: number + /** + * The part of `usd` this runtime priced from a model catalog because no provider receipt + * covered the work. Requires `usdKnown: false` — a catalog price approximates what a + * provider would bill and never measures what it did. + * + * Absence means this runtime priced nothing here, NOT that `usd` is a receipt. `usdKnown` + * is what says whether a dollar figure is measured. + */ + usdEstimated?: number } | { kind: 'iteration' } @@ -479,6 +488,11 @@ export interface Spend { * when enforcing a dollar-denominated comparison or limit. */ usdKnown?: boolean usd: number + /** The part of `usd` priced from a model catalog because no provider receipt covered the work. + * `usd - usdEstimated` is what a provider is known to have billed. Present only with + * `usdKnown: false`; absence means nothing here was catalog-priced, not that `usd` is + * measured. */ + usdEstimated?: number ms: number } diff --git a/src/runtime/util.ts b/src/runtime/util.ts index 0b5c4e7f..fbb70d37 100644 --- a/src/runtime/util.ts +++ b/src/runtime/util.ts @@ -137,6 +137,26 @@ export function zeroTokenUsage(): LoopTokenUsage { return { input: 0, output: 0 } } +/** + * Sum the catalog-priced part of a dollar total across spends, as a field to spread. + * + * Returns nothing when no input carried one. A fold of pure provider receipts must not gain a + * `usdEstimated: 0`, which would read as "this runtime checked and priced none" on a path that + * never prices at all. + */ +export function usdEstimatedOf(...spends: ReadonlyArray<{ usdEstimated?: number }>): { + usdEstimated?: number +} { + let total = 0 + let priced = false + for (const spend of spends) { + if (spend.usdEstimated === undefined) continue + priced = true + total += spend.usdEstimated + } + return priced ? { usdEstimated: total } : {} +} + /** Copy a token subtotal without dropping optional provider cache telemetry. */ export function cloneTokenUsage(usage: LoopTokenUsage): LoopTokenUsage { const cacheBreakdownUnknown = diff --git a/tests/kernel/dollar-estimate.test.ts b/tests/kernel/dollar-estimate.test.ts new file mode 100644 index 00000000..3f3bad93 --- /dev/null +++ b/tests/kernel/dollar-estimate.test.ts @@ -0,0 +1,182 @@ +import { estimateCost, MODEL_PRICING } from '@tangle-network/agent-eval' +import { describe, expect, it } from 'vitest' +import { createBudgetPool, spendFromUsageEvents } from '../../src/runtime/supervise/budget' +import { priceUnreceiptedWork } from '../../src/runtime/supervise/cost-estimate' +import type { Spend, UsageEvent } from '../../src/runtime/supervise/types' +import { usdEstimatedOf } from '../../src/runtime/util' + +const MODEL = 'claude-sonnet-4-20250514' +const RATE = MODEL_PRICING[MODEL]! + +function spend(over: Partial = {}): Spend { + return { iterations: 1, tokens: { input: 0, output: 0 }, usd: 0, ms: 0, ...over } +} + +/** An uncapped pool — the shape 286 of 292 fleet runs use, since they set no `maxUsd`. */ +function uncappedPool() { + return createBudgetPool({ maxIterations: 100, maxTokens: 10_000_000 }, () => 0) +} + +describe('pricing work that carried no provider receipt', () => { + it('prices from the catalog and marks the dollars as not measured', () => { + const event = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + const expected = 10 * RATE.input + 2 * RATE.output + expect(event).toEqual({ + kind: 'cost', + usd: expected, + usdKnown: false, + usdEstimated: expected, + }) + expect(expected).toBeGreaterThan(0) + }) + + it('prices a cached prefix at the full input rate, because the catalog carries no cache rate', () => { + // The catalog entry is `{ input, output }` per model. There is no cache-read rate to apply, + // so a prefix the provider served from cache is charged at the full input rate. That + // overstates a cache-heavy turn, which is the correct direction: a discount the catalog + // cannot support would be invented, and an invented discount understates spend. + expect(Object.keys(RATE).sort()).toEqual(['input', 'output']) + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + expect(priced.usd).toBe(estimateCost(10_000, 2_000, MODEL)) + }) + + it('reports unknown dollars, not a free turn, for a model the catalog does not price', () => { + const event = priceUnreceiptedWork({ + inputTokens: 10_000, + outputTokens: 2_000, + model: 'in-house/no-such-model-family', + }) + expect(event).toEqual({ kind: 'cost', usd: 0, usdKnown: false }) + expect(event.usdEstimated).toBeUndefined() + }) + + it('reports unknown dollars when no model was observed', () => { + expect(priceUnreceiptedWork({ inputTokens: 10, outputTokens: 5, model: undefined })).toEqual({ + kind: 'cost', + usd: 0, + usdKnown: false, + }) + }) + + it('refuses to price a negative or non-finite token count', () => { + for (const inputTokens of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(priceUnreceiptedWork({ inputTokens, outputTokens: 5, model: MODEL })).toEqual({ + kind: 'cost', + usd: 0, + usdKnown: false, + }) + } + }) +}) + +describe('a dollar total built from estimates', () => { + it('reaches the pool as a lower bound rather than a measurement', () => { + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + const events: UsageEvent[] = [ + { kind: 'tokens', input: 10_000, output: 2_000 }, + priced, + { kind: 'iteration' }, + ] + const folded = spendFromUsageEvents(events) + expect(folded.usd).toBe(priced.usd) + expect(folded.usdEstimated).toBe(priced.usd) + expect(folded.usdKnown).toBe(false) + + const pool = uncappedPool() + pool.observe(folded) + // The dollars are recorded, and the pool refuses to call the total a measurement. + expect(pool.readout().usdKnown).toBe(false) + }) + + it('keeps a receipt and an estimate separable through the fold', () => { + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + const folded = spendFromUsageEvents([ + { kind: 'cost', usd: 0.25 }, + priced, + { kind: 'iteration' }, + ]) + expect(folded.usd).toBe(0.25 + priced.usd) + expect(folded.usdEstimated).toBe(priced.usd) + // The provider-billed part stays recoverable, which is the point of carrying both. + expect(folded.usd - folded.usdEstimated!).toBeCloseTo(0.25, 10) + expect(folded.usdKnown).toBe(false) + }) + + it('leaves a pure-receipt fold with no estimated part at all', () => { + const folded = spendFromUsageEvents([{ kind: 'cost', usd: 0.25 }, { kind: 'iteration' }]) + expect(folded.usd).toBe(0.25) + expect(folded.usdEstimated).toBeUndefined() + expect(folded.usdKnown).toBeUndefined() + }) + + it('sums the estimated part across merged spends, and reports none when nothing was priced', () => { + expect(usdEstimatedOf({ usdEstimated: 0.5 }, { usdEstimated: 0.25 })).toEqual({ + usdEstimated: 0.75, + }) + expect(usdEstimatedOf({ usdEstimated: 0.5 }, {})).toEqual({ usdEstimated: 0.5 }) + expect(usdEstimatedOf({}, {})).toEqual({}) + }) +}) + +describe('the estimated part may never be read as billed spend', () => { + it('refuses an estimated part on a spend that claims its dollars are known', () => { + const pool = uncappedPool() + expect(() => pool.observe(spend({ usd: 1, usdEstimated: 1 }))).toThrow( + /usdEstimated requires observed spend.usdKnown false/, + ) + expect(() => pool.observe(spend({ usd: 1, usdEstimated: 1, usdKnown: true }))).toThrow( + /usdEstimated requires observed spend.usdKnown false/, + ) + }) + + it('refuses an estimated part larger than the total it is a part of', () => { + expect(() => + uncappedPool().observe(spend({ usd: 1, usdEstimated: 1.5, usdKnown: false })), + ).toThrow(/usdEstimated must not exceed observed spend.usd/) + }) + + it('refuses a negative or non-finite estimated part', () => { + for (const usdEstimated of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => + uncappedPool().observe(spend({ usd: 10, usdEstimated, usdKnown: false })), + ).toThrow(/usdEstimated must be a non-negative finite number/) + } + }) + + it('accepts a well-formed estimated part', () => { + expect(() => + uncappedPool().observe(spend({ usd: 1, usdEstimated: 0.4, usdKnown: false })), + ).not.toThrow() + }) + + it('diagnoses an unreceipted child as unknown dollars, not as overspending its ceiling', () => { + // A catalog price can exceed a child's declared dollar ceiling. Reporting that as an + // overspend would assert the child spent dollars a provider billed, which is the exact + // confusion this field exists to prevent. + const pool = createBudgetPool({ maxIterations: 10, maxTokens: 1_000_000, maxUsd: 100 }, () => 0) + const reserved = pool.reserve({ maxIterations: 2, maxTokens: 1_000, maxUsd: 0.001 }) + expect(reserved.ok).toBe(true) + if (!reserved.ok) return + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + expect(priced.usd).toBeGreaterThan(0.001) + expect(() => + pool.reconcile( + reserved.ticket, + spend({ usd: priced.usd, usdEstimated: priced.usd, usdKnown: false }), + ), + ).toThrow(/reported unknown dollar cost under a dollar-capped budget/) + }) + + it('still refuses unknown dollars under a dollar-capped root', () => { + // Pricing an estimate does not open a dollar cap. A capped root refuses work whose dollars + // are not measured, exactly as before, because the estimate rides `usdKnown: false`. + const capped = createBudgetPool( + { maxIterations: 10, maxTokens: 1_000_000, maxUsd: 100 }, + () => 0, + ) + const priced = priceUnreceiptedWork({ inputTokens: 10_000, outputTokens: 2_000, model: MODEL }) + expect(() => + capped.observe(spend({ usd: priced.usd, usdEstimated: priced.usd, usdKnown: false })), + ).toThrow(/cannot observe unknown dollar cost under a dollar-capped budget/) + }) +}) diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts index b8220f66..d71b2f19 100644 --- a/tests/kernel/supervise-full-profile-bridge.test.ts +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -1396,7 +1396,12 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( if (result.reason === 'driver-failed') { expect(result.error.message).toMatch(/unknown dollar cost under a dollar-capped budget/) } - expect(result.spentTotal).toMatchObject({ usd: 0, usdKnown: false }) + // The turn carried no receipt, so its dollars are the catalog price of what it presented — + // recorded, and explicitly not a measurement. Pricing an estimate does not open the cap: the + // refusal above is unchanged. + expect(result.spentTotal.usdKnown).toBe(false) + expect(result.spentTotal.usd).toBeGreaterThan(0) + expect(result.spentTotal.usdEstimated).toBe(result.spentTotal.usd) }) it('records a manager with unknown token usage as unknown telemetry without ending the run', async () => {