diff --git a/packages/effect/README.md b/packages/effect/README.md index 6eadc94..9ec34a4 100644 --- a/packages/effect/README.md +++ b/packages/effect/README.md @@ -89,6 +89,30 @@ const deterministic = LanguageModel.generateText({ prompt: "Name a color." }).pi `CoderLanguageModel.fromModel` (the bridge core) adapts any AI SDK `LanguageModelV4`; unit tests use it to run without HTTP. +### Usage metadata + +The finish part's `usage` holds the normalized token counts. `effect/ai`'s +`Response.Usage` has no field for the provider's raw usage +(`LanguageModelV4Usage.raw`), so the bridge puts it in the finish part's +`metadata.coder.rawUsage` as plain JSON. The key is present only when the +provider reports raw usage. For the Coder Agents model, it is chatd's wire +usage (snake_case fields such as `input_tokens` and `context_limit`). The +`CoderFinishMetadata` type describes `metadata.coder`, and the same rule +applies on the generate and stream paths. + +```ts +import * as LanguageModel from "effect/ai/LanguageModel"; +import * as Effect from "effect/Effect"; +import type { CoderFinishMetadata } from "@coder/ai-sdk-effect"; + +const rawUsage = Effect.gen(function* () { + const response = yield* LanguageModel.generateText({ prompt: "hello" }); + const finish = response.content.find((part) => part.type === "finish"); + const coder = finish?.metadata.coder as CoderFinishMetadata | undefined; + return coder?.rawUsage; +}); +``` + ## `LanguageModel` over Coder Agents `CoderAgentModel.layer(settings)` implements `LanguageModel` on top of a Coder @@ -152,7 +176,7 @@ program.pipe( ## Typed error taxonomy `classifyError` maps an `AiError`, a raw AI SDK error, or a raw -`@coder/ai-sdk-agent` error to one of: `auth`, `rate-limit`, +`@coder/ai-sdk-agent` error to one of: `auth`, `rate-limit`, `quota`, `provider-unavailable`, `malformed-response`, `transport`, `timeout`, `unknown`. `isTransient` composes with `Effect.retry`: @@ -180,6 +204,10 @@ instead of adding its own error classes: - A `RateLimitError` from an AI Gateway response sets `retryAfter` from a valid `Retry-After` header (delay-seconds, or an HTTP-date). Read it as `error.retryAfter`, for example to pick a retry delay. +- HTTP 402 means the account's quota or billing limit is exhausted. It maps + to `QuotaExhaustedError` and the `quota` classification, which is not + transient: a retry cannot succeed until someone raises the limit. HTTP 429 + stays a transient `rate-limit`. - The reason's `metadata.coder` records the classification and, when the source error has them, the HTTP `status`, the Coder Agents `retryable` verdict, the chat error's `kind` and `provider`, and the stream's `chatId`. @@ -196,7 +224,8 @@ instead of adding its own error classes: | Source error | `AiError` reason | `classifyError` | `isTransient` | Reason `isRetryable` | | ----------------------------------------------------------------- | ---------------------------------------------------------------- | --------------------------------- | ------------- | ------------------------------- | | `APICallError` or `CoderApiError`, status 401 / 403 | `AuthenticationError` (`InvalidKey` / `InsufficientPermissions`) | `auth` | `false` | `false` | -| status 402 / 429 | `RateLimitError` | `rate-limit` | `true` | `true` | +| status 402 | `QuotaExhaustedError` | `quota` | `false` | `false` | +| status 429 | `RateLimitError` (`retryAfter` from `Retry-After`) | `rate-limit` | `true` | `true` | | status 5xx | `InternalProviderError` | `provider-unavailable` | `true` | `true` | | status 400 | `InvalidRequestError` | `unknown` | `false` | `false` | | any other status | `UnknownError` | `unknown` | `false` | `false` | @@ -306,6 +335,7 @@ no `effect/ai` equivalent are dropped. | Provider-defined tools (not expressible in `LanguageModelV4` call options) | `InvalidRequestError` | | Response parts of type `custom`, `reasoning-file`, `tool-approval-request`; file payloads that are not raw data (URL / provider-reference / inline-text) | Dropped | | Provider-executed tool calls and results for tools outside the call's toolkit (`effect/ai` types tool parts by toolkit) | Moved to the finish part's `metadata.coder.serverToolCalls` | +| Provider raw usage (`effect/ai`'s `Response.Usage` has no field for it) | Moved to the finish part's `metadata.coder.rawUsage` | | Tool approval parts in the prompt (`effect/ai` resolves and strips them before the call) | Not forwarded | | `Prompt` provider options (per-part metadata) | Not forwarded | | `ProviderOptions.span` (telemetry) | Not wired into request headers | diff --git a/packages/effect/src/errors.ts b/packages/effect/src/errors.ts index 0a0f240..aa0ed6a 100644 --- a/packages/effect/src/errors.ts +++ b/packages/effect/src/errors.ts @@ -40,7 +40,9 @@ import * as Schema from "effect/Schema"; * Coder-oriented classification of a gateway/model call failure. * * - `auth`: the credential was rejected (401/403) — Coder token or BYOK key. - * - `rate-limit`: the gateway or upstream throttled or exhausted quota (402/429). + * - `rate-limit`: the gateway or upstream throttled the request (429). + * - `quota`: the account's quota or billing limit is exhausted (402). Not + * transient: retrying cannot succeed until someone acts. * - `provider-unavailable`: the upstream or gateway failed server-side (5xx). * - `malformed-response`: the response could not be parsed or validated. * - `transport`: the request never produced an HTTP response (network error), @@ -52,6 +54,7 @@ import * as Schema from "effect/Schema"; export type ErrorReason = | "auth" | "rate-limit" + | "quota" | "provider-unavailable" | "malformed-response" | "transport" @@ -67,7 +70,7 @@ export type ErrorReason = * - `kind` / `provider`: from `CoderChatError`. * - `chatId`: from `CoderStreamError`. * - `message`: the source error's message, on reasons without a - * `description` field (`RateLimitError`). + * `description` field (`RateLimitError`, `QuotaExhaustedError`). * * A `RateLimitError` also sets `retryAfter` from the response's * `Retry-After` header when it is valid. @@ -76,6 +79,7 @@ const CoderErrorMetadata = Schema.Struct({ classification: Schema.Literals([ "auth", "rate-limit", + "quota", "provider-unavailable", "malformed-response", "transport", @@ -98,7 +102,8 @@ const decodeCoderMetadata = Schema.decodeUnknownOption(CoderErrorMetadata); /** Classify an HTTP status code into an {@link ErrorReason}. */ export const classifyStatus = (status: number): ErrorReason => { if (status === 401 || status === 403) return "auth"; - if (status === 402 || status === 429) return "rate-limit"; + if (status === 402) return "quota"; + if (status === 429) return "rate-limit"; if (status >= 500) return "provider-unavailable"; return "unknown"; }; @@ -135,8 +140,9 @@ const classifyReason = (reason: AiError.AiErrorReason): ErrorReason => { case "AuthenticationError": return "auth"; case "RateLimitError": - case "QuotaExhaustedError": return "rate-limit"; + case "QuotaExhaustedError": + return "quota"; case "InternalProviderError": return "provider-unavailable"; case "NetworkError": @@ -176,7 +182,8 @@ export const classifyError = (error: ClassifiableError): ErrorReason => { * `false` by the agent when replaying the prompt would duplicate server-side * effects; `CoderChatError.retryable` is chatd's own verdict). Otherwise * throttling, upstream outages, network errors, and turn timeouts are - * transient, while auth and malformed-response failures are terminal. + * transient, while auth, exhausted quota, and malformed-response failures + * are terminal. */ export const isTransient = (error: ClassifiableError): boolean => { const verdict = Option.getOrUndefined(coderMetadata(asAiError(error)))?.retryable; @@ -273,6 +280,12 @@ const reasonFor = ( metadata: { coder: { ...coder, message: description } }, }); break; + case "quota": + reason = new AiError.QuotaExhaustedError({ + http, + metadata: { coder: { ...coder, message: description } }, + }); + break; case "provider-unavailable": reason = new AiError.InternalProviderError(common); break; diff --git a/packages/effect/src/index.ts b/packages/effect/src/index.ts index 717b24e..618d902 100644 --- a/packages/effect/src/index.ts +++ b/packages/effect/src/index.ts @@ -10,7 +10,7 @@ export { export * as CoderAgentModel from "./agent-model.js"; export type { AgentModelSettings, AgentOptions } from "./agent-model.js"; export * as CoderLanguageModel from "./language-model.js"; -export type { GenerationOptions, ProviderSource } from "./language-model.js"; +export type { CoderFinishMetadata, GenerationOptions, ProviderSource } from "./language-model.js"; export { acquireSession, type AcquireSessionOptions, diff --git a/packages/effect/src/language-model.ts b/packages/effect/src/language-model.ts index b3413ee..6100b1d 100644 --- a/packages/effect/src/language-model.ts +++ b/packages/effect/src/language-model.ts @@ -19,6 +19,10 @@ * `effect/ai`, whose tool parts are typed by the toolkit. They are moved to * the finish part's `metadata.coder.serverToolCalls` instead; streaming still * emits their `tool-params-start` / `tool-params-end` markers. + * + * `effect/ai`'s `Response.Usage` has no slot for the provider's raw usage + * (`LanguageModelV4Usage.raw`), so it travels in the finish part's + * `metadata.coder.rawUsage` (see {@link CoderFinishMetadata}). */ import type { JSONValue, @@ -62,6 +66,28 @@ const MODULE = "CoderLanguageModel"; /** The model source: provider settings, or an already-constructed provider. */ export type ProviderSource = CoderProviderSettings | { readonly provider: CoderProvider }; +/** + * What the bridge records under the finish part's `metadata.coder`, as plain + * JSON. Each key is present only when there is something to report, and the + * finish part has no `metadata.coder` when neither is: + * + * - `serverToolCalls`: provider-executed tool activity for tools outside the + * call's toolkit (see the module docs), in call order. + * - `rawUsage`: the provider's raw usage (`LanguageModelV4Usage.raw`), present + * only when the provider reports it. Keys whose value is `undefined` are + * dropped. + */ +export type CoderFinishMetadata = { + readonly serverToolCalls?: ReadonlyArray<{ + readonly id: string; + readonly name: string; + readonly params?: Schema.Json; + readonly result?: Schema.Json; + readonly isFailure?: boolean; + }>; + readonly rawUsage?: { readonly [key: string]: Schema.Json }; +}; + /** * Generation controls forwarded to the underlying model. Options given at * construction apply to every call; {@link withGenerationOptions} overrides @@ -216,7 +242,7 @@ const generateText = ( if (serverTools.absorb(content)) continue; parts.push(...contentToParts(content)); } - parts.push(serverTools.attach(finishPart(result.finishReason, result.usage))); + parts.push(finishPart(result.finishReason, result.usage, serverTools)); return parts; }, catch: (error) => toAiError({ module: MODULE, method: "generateText", error }), @@ -259,9 +285,7 @@ const streamText = ( } try { if (serverTools.absorb(part)) return Stream.empty; - return Stream.fromIterable( - streamPartToParts(part).map((p) => (p.type === "finish" ? serverTools.attach(p) : p)), - ); + return Stream.fromIterable(streamPartToParts(part, serverTools)); } catch (error) { return Stream.fail(toAiError({ module: MODULE, method: "streamText", error })); } @@ -590,6 +614,7 @@ const sharedContentToParts = (content: SharedContent): Array const streamPartToParts = ( part: Exclude, + serverTools: ServerToolCalls, ): Array => { switch (part.type) { case "text-start": @@ -620,7 +645,7 @@ const streamPartToParts = ( case "response-metadata": return [responseMetadataPart(part)]; case "finish": - return [finishPart(part.finishReason, part.usage)]; + return [finishPart(part.finishReason, part.usage, serverTools)]; // `stream-start` (warnings) and `raw` chunks have no equivalent. case "stream-start": case "raw": @@ -641,7 +666,7 @@ interface ServerToolCall { /** * Collects provider-executed tool activity for tools outside the call's - * toolkit (see the module docs) and attaches it to the finish part. + * toolkit (see the module docs) for the finish part's metadata. */ class ServerToolCalls { readonly #toolNames: ReadonlySet; @@ -669,13 +694,13 @@ class ServerToolCalls { return false; } - attach(finish: Response.FinishPartEncoded): Response.FinishPartEncoded { - if (this.#calls.size === 0) return finish; + /** The recorded calls as plain JSON, or `undefined` when there are none. */ + toJson(): CoderFinishMetadata["serverToolCalls"] { + if (this.#calls.size === 0) return undefined; // Finish metadata must be plain JSON. Params come from `JSON.parse` and // results from the provider's JSON payload; the round trip drops the // `undefined` object values that the AI SDK's `JSONValue` allows. - const serverToolCalls: Schema.Json = JSON.parse(JSON.stringify([...this.#calls.values()])); - return { ...finish, metadata: { coder: { serverToolCalls } } }; + return JSON.parse(JSON.stringify([...this.#calls.values()])); } #entry(id: string, name: string): ServerToolCall { @@ -705,23 +730,34 @@ const responseMetadataPart = (metadata: { timestamp: metadata.timestamp?.toISOString(), }); +/** The finish part, with `metadata.coder` only when it has keys. */ const finishPart = ( finishReason: LanguageModelV4FinishReason, usage: LanguageModelV4Usage, -): Response.FinishPartEncoded => ({ - type: "finish", - reason: finishReason.unified, - usage: { - inputTokens: { - uncached: usage.inputTokens.noCache, - total: usage.inputTokens.total, - cacheRead: usage.inputTokens.cacheRead, - cacheWrite: usage.inputTokens.cacheWrite, - }, - outputTokens: { - total: usage.outputTokens.total, - text: usage.outputTokens.text, - reasoning: usage.outputTokens.reasoning, + serverTools: ServerToolCalls, +): Response.FinishPartEncoded => { + const part: Response.FinishPartEncoded = { + type: "finish", + reason: finishReason.unified, + usage: { + inputTokens: { + uncached: usage.inputTokens.noCache, + total: usage.inputTokens.total, + cacheRead: usage.inputTokens.cacheRead, + cacheWrite: usage.inputTokens.cacheWrite, + }, + outputTokens: { + total: usage.outputTokens.total, + text: usage.outputTokens.text, + reasoning: usage.outputTokens.reasoning, + }, }, - }, -}); + }; + const coder: { -readonly [K in keyof CoderFinishMetadata]: CoderFinishMetadata[K] } = {}; + const serverToolCalls = serverTools.toJson(); + if (serverToolCalls !== undefined) coder.serverToolCalls = serverToolCalls; + // The raw usage is a `JSONObject`; the round trip drops `undefined` values. + if (usage.raw !== undefined) coder.rawUsage = JSON.parse(JSON.stringify(usage.raw)); + if (Object.keys(coder).length === 0) return part; + return { ...part, metadata: { coder } }; +}; diff --git a/packages/effect/test/agent-model.test.ts b/packages/effect/test/agent-model.test.ts index c4bcc67..e681a2a 100644 --- a/packages/effect/test/agent-model.test.ts +++ b/packages/effect/test/agent-model.test.ts @@ -215,6 +215,10 @@ describe("CoderAgentModel", () => { const finish = parts.at(-1); expect(finish?.type === "finish" && finish.reason).toBe("stop"); expect(finish?.type === "finish" && finish.usage.inputTokens.total).toBe(12); + // chatd's wire usage (snake_case) is the provider's raw usage. + expect(finish?.metadata).toEqual({ + coder: { rawUsage: { input_tokens: 12, output_tokens: 4 } }, + }); expect(fake.created[0]?.system_prompt).toBe("Be brief."); expect(fake.interrupted).toEqual([]); expect(fake.openStreams).toBe(0); diff --git a/packages/effect/test/errors.test.ts b/packages/effect/test/errors.test.ts index fc2ce91..884e202 100644 --- a/packages/effect/test/errors.test.ts +++ b/packages/effect/test/errors.test.ts @@ -19,7 +19,7 @@ describe("classifyStatus", () => { it.each([ [401, "auth"], [403, "auth"], - [402, "rate-limit"], + [402, "quota"], [429, "rate-limit"], [500, "provider-unavailable"], [502, "provider-unavailable"], @@ -52,7 +52,7 @@ describe("toAiError", () => { it.each([ [400, "InvalidRequestError"], [401, "AuthenticationError"], - [402, "RateLimitError"], + [402, "QuotaExhaustedError"], [403, "AuthenticationError"], [404, "UnknownError"], [503, "InternalProviderError"], @@ -60,6 +60,18 @@ describe("toAiError", () => { expect(map(apiCallError(status)).reason._tag).toBe(tag); }); + it("maps HTTP 402 to QuotaExhaustedError, preserving details", () => { + const error = map(apiCallError(402)); + const reason = error.reason as AiError.QuotaExhaustedError; + expect(reason._tag).toBe("QuotaExhaustedError"); + expect(reason.isRetryable).toBe(false); + expect(reason.http?.response).toEqual({ status: 402, headers: { "x-request-id": "req_123" } }); + expect(reason.http?.body).toBe(`{"error":"failure"}`); + expect(reason.metadata).toEqual({ + coder: { classification: "quota", status: 402, message: "gateway failure (402)" }, + }); + }); + describe("RateLimitError.retryAfter", () => { const throttled = (headers: Record): AiError.AiError => map( @@ -145,6 +157,10 @@ describe("classifyError / isTransient", () => { expect(classifyError(throttled)).toBe("rate-limit"); expect(isTransient(throttled)).toBe(true); + const quota = toAiError({ module: "T", method: "m", error: apiCallError(402) }); + expect(classifyError(quota)).toBe("quota"); + expect(isTransient(quota)).toBe(false); + const upstream = toAiError({ module: "T", method: "m", error: apiCallError(503) }); expect(classifyError(upstream)).toBe("provider-unavailable"); expect(isTransient(upstream)).toBe(true); @@ -157,12 +173,32 @@ describe("classifyError / isTransient", () => { it("classifies raw AI SDK errors directly (documented contract)", () => { expect(classifyError(apiCallError(403))).toBe("auth"); expect(classifyError(apiCallError(429))).toBe("rate-limit"); + expect(classifyError(apiCallError(402))).toBe("quota"); expect(classifyError(apiCallError(undefined))).toBe("transport"); expect(classifyError(new JSONParseError({ text: "x", cause: new Error("y") }))).toBe( "malformed-response", ); expect(isTransient(apiCallError(503))).toBe(true); expect(isTransient(apiCallError(401))).toBe(false); + expect(isTransient(apiCallError(402))).toBe(false); + expect(isTransient(apiCallError(429))).toBe(true); + }); + + it("classifies effect/ai's own quota and rate-limit reasons by tag", () => { + const quota = AiError.make({ + module: "T", + method: "m", + reason: new AiError.QuotaExhaustedError({}), + }); + expect(classifyError(quota)).toBe("quota"); + expect(isTransient(quota)).toBe(false); + const throttled = AiError.make({ + module: "T", + method: "m", + reason: new AiError.RateLimitError({}), + }); + expect(classifyError(throttled)).toBe("rate-limit"); + expect(isTransient(throttled)).toBe(true); }); it("classifies malformed output as terminal", () => { @@ -208,6 +244,7 @@ describe("Coder Agents errors", () => { }); it.each([ + [402, "quota", false], [429, "rate-limit", true], [503, "provider-unavailable", true], [409, "unknown", false], @@ -250,6 +287,7 @@ describe("Coder Agents errors", () => { it.each([ [{ kind: "stream_closed", retryable: true }, "transport", true], [{ status_code: 429, retryable: true }, "rate-limit", true], + [{ status_code: 402 }, "quota", false], [{ status_code: 503, retryable: false }, "provider-unavailable", false], [{ status_code: 401 }, "auth", false], [{ kind: "generic" }, "unknown", false], diff --git a/packages/effect/test/language-model.test.ts b/packages/effect/test/language-model.test.ts index 100b0bf..2ddb8a7 100644 --- a/packages/effect/test/language-model.test.ts +++ b/packages/effect/test/language-model.test.ts @@ -23,6 +23,21 @@ const USAGE = { outputTokens: { total: 5, text: 5, reasoning: undefined }, }; +// Provider-shaped raw usage. `undefined` values are allowed by the AI SDK's +// `JSONObject` but are not JSON, so the finish metadata drops them. +const RAW_USAGE = { + input_tokens: 8, + cache_read_tokens: 2, + total_cost_micros: 1234, + details: { tier: "standard", unset: undefined }, +}; +const RAW_USAGE_JSON = { + input_tokens: 8, + cache_read_tokens: 2, + total_cost_micros: 1234, + details: { tier: "standard" }, +}; + const fakeModel = (overrides: Partial): LanguageModelV4 => ({ specificationVersion: "v4", provider: "coder", @@ -77,6 +92,9 @@ describe("generateText", () => { cacheWrite: undefined, }); expect(response.usage.outputTokens).toEqual({ total: 5, text: 5, reasoning: undefined }); + // No raw usage from the provider: no `metadata.coder` on the finish part. + const finish = response.content.find((part) => part.type === "finish"); + expect(finish?.metadata).toEqual({}); expect(captured?.prompt).toEqual([ { role: "user", content: [{ type: "text", text: "Say hello" }] }, ]); @@ -331,6 +349,47 @@ describe("generateText", () => { }); }); + it("carries the provider's raw usage as finish metadata.coder.rawUsage", async () => { + const model = fakeModel({ + doGenerate: async () => ({ + content: [ + { type: "text", text: "Listed the files." }, + { + type: "tool-call", + toolCallId: "srv_1", + toolName: "execute", + input: `{"command":"ls"}`, + providerExecuted: true, + }, + { type: "tool-result", toolCallId: "srv_1", toolName: "execute", result: "a.txt" }, + ], + finishReason: { unified: "stop", raw: undefined }, + usage: { ...USAGE, raw: RAW_USAGE }, + warnings: [], + }), + }); + + const response = await Effect.runPromise( + Effect.flatMap(service(model), (m) => m.generateText({ prompt: "ls" })), + ); + + const finish = response.content.find((part) => part.type === "finish"); + expect(finish?.metadata).toEqual({ + coder: { + rawUsage: RAW_USAGE_JSON, + serverToolCalls: [ + { + id: "srv_1", + name: "execute", + params: { command: "ls" }, + result: "a.txt", + isFailure: false, + }, + ], + }, + }); + }); + describe("tool choice", () => { const GetWeather = Tool.make("get_weather", { parameters: Schema.Struct({ city: Schema.String }), @@ -441,6 +500,7 @@ describe("generateText", () => { expect(classifyError(auth)).toBe("auth"); expect(classifyError(failure(await run(429)))).toBe("rate-limit"); + expect(classifyError(failure(await run(402)))).toBe("quota"); const network = failure(await run(undefined)); expect(network.reason._tag).toBe("NetworkError"); @@ -573,6 +633,33 @@ describe("streamText", () => { }); }); + it.each([ + ["present", { ...USAGE, raw: RAW_USAGE }, { coder: { rawUsage: RAW_USAGE_JSON } }], + ["absent", USAGE, {}], + ] as const)( + "carries raw usage on the finish part only when %s", + async (_label, usage, metadata) => { + const model = fakeModel({ + doStream: async () => ({ + stream: streamOf([ + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: "Hi" }, + { type: "text-end", id: "t1" }, + { type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage }, + ]), + }), + }); + + const parts = await Effect.runPromise( + Effect.flatMap(service(model), (m) => Stream.runCollect(m.streamText({ prompt: "hi" }))), + ); + + const finish = parts.at(-1); + expect(finish?.type).toBe("finish"); + expect(finish?.metadata).toEqual(metadata); + }, + ); + it("fails the stream with a typed AiError on error parts", async () => { const model = fakeModel({ doStream: async () => ({