diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 429f850b..ed44d8ce 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "source": { "source": "npm", "package": "@copilotkit/aimock", - "version": "^1.46.0" + "version": "^1.47.0" }, "description": "Fixture authoring skill for @copilotkit/aimock — LLM, multimedia (image/TTS/transcription/video), MCP, A2A, AG-UI, vector, embeddings, structured output, sequential responses, streaming physics, record/replay, agent loop patterns, and debugging" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 6295719e..d84cafe7 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "aimock", - "version": "1.46.0", + "version": "1.47.0", "description": "Fixture authoring guidance for @copilotkit/aimock — LLM, multimedia, MCP, A2A, AG-UI, vector, and service mocking", "author": { "name": "CopilotKit" diff --git a/CHANGELOG.md b/CHANGELOG.md index a6712c6f..d3986aa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,28 +1,17 @@ # @copilotkit/aimock -## [Unreleased] +## [1.47.0] - 2026-10-26 ### Added -- Fixture `citations` serve OpenAI Responses `url_citation` annotations; fixture `files` serve Gemini `inlineData` parts (#550 by @abhiaiyer91) -- Fixture `citations` also serve Chat Completions `url_citation` annotations (`message.annotations`, one streamed `delta.annotations` chunk), Anthropic `web_search_result_location` citations on text blocks (`citations_delta` when streaming), and Gemini `groundingMetadata` (`groundingChunks` + `groundingSupports`, byte offsets). A citation with `type: "file"` (`fileId`, `filename`, optional `index`, which defaults to the end of the text) serves an OpenAI Responses `file_citation` annotation on `output_text`, streamed through `response.output_text.annotation.added`; Chat Completions, Anthropic and Gemini skip it. Closes #547. -- Fixture `files` serve OpenAI Responses `image_generation_call` output items (`result` holds the base64 bytes), streamed with the `response.image_generation_call.*` events. Chat Completions and Anthropic Messages reject a fixture with `files` with a 500 carrying `aimock_unsupported_generated_file` (new `UnsupportedGeneratedFileError`). Closes #548. -- Malformed fixture `citations` or `files` (not an array, a missing or mistyped field, a non-integer offset, an offset outside the text, or citations on a turn with no text: tool calls only, `blocks` with no text block, or empty content) answer a 500 before any content is written: `aimock_invalid_fixture_citation` or `aimock_invalid_fixture_file`. A non-`image/*` file on the OpenAI Responses API answers a 500 with `aimock_unsupported_generated_file`. This is the same on every wire that reads these keys, with or without a misbehavior fault: the fixture is checked on its original turn before any fault applies (only an invalid misbehavior config answers its own 400 or 501 first). A valid fixture under a fault that changes the reply text is served without its `citations`. Only Chat Completions, Anthropic Messages, Gemini and OpenAI Responses read these keys; other wires ignore them. -- Fixture load validation warns about malformed `citations` and `files`, with the same problem description the 500 carries (the 500 message adds an `aimock: invalid fixture` prefix; the warning adds which 500 the requests answer). The fixture still loads. -- `InvalidFixtureOutputError` (the error behind `aimock_invalid_fixture_citation` and `aimock_invalid_fixture_file`) is exported from `@copilotkit/aimock`. -- Opt-in fixture `nonStreamingTimingAllProviders: true` delays non-streaming replies on OpenAI chat, Anthropic, Responses, Gemini/Vertex, Gemini Interactions, Cohere, Bedrock invoke/converse and Ollama; `nonStreamingTiming` is unchanged (OpenAI chat only); error replies are never delayed. On the OpenAI Responses API the wait also counts the stream frames that fixture `citations` and generated `files` add -- Raw passthrough fixtures: a `raw` response (`wire`, `events`, `json`, optional `status`, `headers`, and `done` on `openai-chat`) sends provider events or JSON exactly as written, with only the named wire's framing, on the HTTP LLM wires (OpenAI Responses and Chat Completions, Anthropic, Gemini/Vertex, Gemini Interactions, Cohere, Ollama `/api/chat` as `ollama` and `/api/generate` as `ollama-generate`, Bedrock invoke and converse). A raw fixture matches only its own wire. A request whose mode has no body answers that wire's 500 with `aimock_raw_mode_unavailable`; an invalid raw fixture added in code answers a 500 with `aimock_raw_fixture_invalid`, and `--validate-on-load` reports it at load. Streaming timing, truncation, disconnects and chaos apply; misbehavior faults skip raw fixtures. A non-streaming raw reply follows `nonStreamingTiming` like a typed fixture: `nonStreamingTiming: true` delays it on OpenAI Chat Completions only, `nonStreamingTimingAllProviders: true` on every provider; a raw reply with status 400 or above is never delayed, and with a `json` it answers JSON even to a streaming request. WebSocket Responses answers a raw fixture with an error event with the code `aimock_raw_mode_unavailable`. New exported types `RawResponse`, `RawWireId` and `WireId` ([#549](https://github.com/CopilotKit/aimock/issues/549)) +- Fixture `citations` serve sources on Responses, Chat Completions, Claude and Gemini (#550, #578) +- Fixture `files` serve generated images on Gemini and OpenAI Responses (#550, #578) +- Raw passthrough fixtures send provider SSE events or JSON exactly as written (#580) +- Opt-in `nonStreamingTimingAllProviders` delays non-streaming replies on every provider (#579) ### Fixed -- Misbehavior faults on Chat Completions, Anthropic Messages, Gemini and the OpenAI Responses API (HTTP and WebSocket): a fixture the wire rejects (generated `files` on Chat Completions or Anthropic, a non-`image/*` file on Responses, malformed `citations` or `files`) is never faulted. It takes the normal path, is journaled as a 500 that keeps its fixture, and consumes no fault `times` budget or seed ordinal; `aimock_misbehavior_total` does not count it. A non-stream Chat Completions fault is journaled before its timing wait. -- Journal accuracy: a client that hangs up during the timing wait of any non-stream Chat Completions reply (faulted or not) is journaled as interrupted, not as served. -- Client disconnects stop delivery (no more chunks or waits) on OpenAI chat, Anthropic, Responses, Gemini/Vertex, Gemini Interactions, Cohere, Bedrock, Ollama, transcription streams and the Responses, Realtime and Gemini Live WebSockets -- Client disconnects on those surfaces are journaled as `interrupted` with `interruptReason: "client aborted"` (HTTP `status: 0` when nothing was sent); aimock-caused teardowns record `"disconnect"`. Journal snapshots of hung-up requests now carry these fields -- WebSocket: a client TCP FIN without a close frame now closes the connection at once (1006) and ends the turn -- Daily drift: the OpenAI Responses K9 reasoning-exhaustion probes (HTTP and WebSocket) request `gpt-5.4-mini`. OpenAI retired `o4-mini` on 2026-10-23 (its `/v1/models` `shutdown_date`), and it has answered `404 model_not_found` since. The probe model is one constant, `OPENAI_K9_PROBE_MODEL`. The usage check accepts `output_tokens` equal to `output_tokens_details.reasoning_tokens` (gpt-5.4-mini spends the 32-token cap on reasoning; o4-mini reported 0). The event-order check is unchanged, and the retained o4-mini replay fixtures are still graded against o4-mini. Drift harness only; aimock behavior is unchanged (#584). -- Daily drift: a preflight checks every model the live drift probes pin (`src/__tests__/drift/probe-models.ts`) against the provider's own retirement data before the run. A retired model fails the run with `probe model X retired on DATE — update ` instead of looking like API drift; a shutdown date within 30 days is a warning in the job summary and in Slack. The Gemini embeddings canary now probes `gemini-embedding-001`; `text-embedding-004` answers 404. Drift harness only; aimock behavior is unchanged. -- Docker image: the base image is pinned to `node:22.23.4-alpine` by digest. Node 22.23.4 fixes CVE-2026-111497 (Brotli decompression memory bug); the 1.46.0 image shipped Node 22.23.3. +- Client disconnects stop and journal every LLM reply; Docker image on Node 22.23.4 (#579, #576) ## [1.46.0] - 2026-10-20 diff --git a/README.md b/README.md index 91aa5b2d..7dd36c1f 100644 --- a/README.md +++ b/README.md @@ -92,13 +92,16 @@ Run them all on one port with `npx @copilotkit/aimock --config aimock.json`, or - **[Native model catalogs and token counts](https://aimock.copilotkit.dev/claude-messages#model-catalog)** — Anthropic `GET /v1/models` (selected by `anthropic-version`, or `x-api-key` without `Authorization`; the list route needs `policy.modelCatalog: "native"` and otherwise stays the OpenAI list) and Gemini `GET /v1beta/models` list and detail routes with provider paging, built from fixture model IDs with synthetic metadata; `POST /v1/messages/count_tokens` and Gemini `:countTokens` return a deterministic estimate of about 1 token per 4 characters of messages (plus tool definitions), minimum 1. These routes never proxy or record - **Context-Based Fixture Routing** — `X-AIMock-Context` header scopes fixtures per integration; fixtures with `match.context` only match requests carrying that context, fixtures without it remain shared - **[Drift Detection](https://aimock.copilotkit.dev/drift-detection)** — Daily CI validation against real APIs -- **[Streaming Physics](https://aimock.copilotkit.dev/streaming-physics)** — Configurable `ttft`, `tps`, and `jitter` +- **[Streaming Physics](https://aimock.copilotkit.dev/streaming-physics)** — Configurable `ttft`, `tps`, and `jitter`; opt in with `nonStreamingTimingAllProviders: true` to give non-streaming replies the same timing on every LLM provider +- **[Client Disconnects](https://aimock.copilotkit.dev/error-injection#client-disconnects)** — When the client hangs up, every LLM provider (HTTP and WebSocket) stops sending and journals the request as `interrupted`, so you can test cancel and abort paths - **[WebSocket APIs](https://aimock.copilotkit.dev/websocket)** — OpenAI Realtime (GA protocol with models: gpt-realtime, gpt-realtime-2, gpt-realtime-1.5, gpt-realtime-mini; transcription/translation via gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1; image input; commentary phase), Responses WS, Gemini Live - **[GPT-Live](https://aimock.copilotkit.dev/websocket#openai-live)** — Mock, record, and replay primary WebSocket sessions at `/v1/live/sessions` in client and managed modes, with fixture-backed audio and backend tool continuation. Saved fixtures replay offline; aimock does not synthesize speech. This voice surface is separate from OpenAI text-model family support; WebRTC/SDP and sideband connections are outside this scope. - **[Prometheus Metrics](https://aimock.copilotkit.dev/metrics)** — Request counts, latencies, fixture match rates - **[Docker + Helm](https://aimock.copilotkit.dev/docker)** — Container image and Helm chart for CI/CD - **[Vitest & Jest Plugins](https://aimock.copilotkit.dev/test-plugins)** — Zero-config `useAimock()` with auto lifecycle and env patching - **[Response Overrides](https://aimock.copilotkit.dev/fixtures)** — Control `id`, `model`, `usage`, `finishReason` in fixture responses +- **[Citations and Generated Files](https://aimock.copilotkit.dev/fixtures#citations-and-files)** — Fixture `citations` serve source citations on OpenAI Responses (URL and file), Chat Completions, Claude and Gemini; fixture `files` serve generated images on the Responses API and Gemini. Each uses the provider's own format, streaming and non-streaming +- **[Raw Passthrough](https://aimock.copilotkit.dev/raw-passthrough)** — A `raw` fixture sends provider SSE events or JSON exactly as written on any HTTP LLM wire, for fields aimock does not model yet; timing, chaos and disconnects still apply - **[Ordered Blocks](https://aimock.copilotkit.dev/fixtures#ordered-blocks)** — A `blocks` array streams text and tool calls in any order (tool-first or interleaved); blocks-only fixtures are first-class, and the recorder captures order from genuinely tool-first/interleaved streams - **[Namespaced and Custom Tool Calls](https://aimock.copilotkit.dev/responses-api#namespaced-and-custom-tool-calls)** — Responses API fixtures can return namespaced function calls and custom tool calls (`custom_tool_call`) over HTTP streaming, non-streaming and WebSocket, and can match on `toolNamespace`. This lets you script OpenAI Codex CLI's namespaced MCP and sub-agent tools and its freeform `apply_patch` tool (with the `responsesTools: "extended"` server option) - **[Streaming Usage Chunks](https://aimock.copilotkit.dev/streaming-physics)** — `stream_options.include_usage` support emits a final chunk with token counts, matching OpenAI's streaming usage protocol diff --git a/charts/aimock/Chart.yaml b/charts/aimock/Chart.yaml index 1c9617b8..79fb52ef 100644 --- a/charts/aimock/Chart.yaml +++ b/charts/aimock/Chart.yaml @@ -3,4 +3,4 @@ name: aimock description: Mock infrastructure for AI application testing (OpenAI, Anthropic, Gemini, MCP, A2A, vector) type: application version: 0.1.0 -appVersion: "1.46.0" +appVersion: "1.47.0" diff --git a/docs/chat-completions/index.html b/docs/chat-completions/index.html index f9782f6e..96f355a5 100644 --- a/docs/chat-completions/index.html +++ b/docs/chat-completions/index.html @@ -59,6 +59,11 @@

OpenAI Chat Completions

raw passthrough fixture with wire: "openai-chat".

+

+ Fixture citations serve url_citation annotations. A fixture with + generated files answers a 500, because Chat Completions has no place for + them. See Citations and generated files. +

Endpoint

diff --git a/docs/claude-messages/index.html b/docs/claude-messages/index.html index 3e66e343..25e488d1 100644 --- a/docs/claude-messages/index.html +++ b/docs/claude-messages/index.html @@ -59,6 +59,12 @@

Anthropic Claude Messages API

raw passthrough fixture with wire: "anthropic".

+

+ Fixture citations serve web_search_result_location citations on + text blocks. A fixture with generated files answers a 500, because the + Messages API has no place for them. See + Citations and generated files. +

Endpoint

diff --git a/docs/docs/index.html b/docs/docs/index.html index 4ffa85cb..1bb9a53e 100644 --- a/docs/docs/index.html +++ b/docs/docs/index.html @@ -262,26 +262,47 @@

What's New

- Model Catalogs & Token Counts - New in 1.46.0 + Citations & Generated Images + New in 1.47.0

- List and look up Gemini and Anthropic models in their native formats (the Anthropic - list is opt-in), and estimate prompt size with count_tokens / - countTokens + Script source citations on OpenAI, Claude and Gemini replies, and generated images on + the Responses API and Gemini, in each provider's own format

- Docs → +
+ +
+ Raw Passthrough + New in 1.47.0 +

+ Send provider events or JSON exactly as written, for fields aimock does not model yet, + with timing, chaos and disconnects still applied +

+ Docs → +
+ +
+ Stream Realism + New in 1.47.0 +

+ Every LLM provider stops sending and journals the request when the client hangs up, + and non-streaming replies on every provider can opt in to realistic timing +

+ Docs →
- Native Auth Errors + Native Catalogs & Auth Errors New in 1.46.0

- Opt in to provider-native 401 bodies and required-header checks, so you can test how - your app handles a bad key or a missing anthropic-version + Gemini and Anthropic model lists, token counts, and opt-in provider-native 401s, so + you can test model pickers and bad-key handling

- Docs →
@@ -306,18 +327,6 @@

What's New

>
-
- Stream Realism - New in 1.46.0 -

- Anthropic streams stop and journal when the client disconnects, and non-streaming - OpenAI replies can opt in to streaming-profile timing -

- Docs → -
-
Responses API Tools

@@ -336,15 +345,6 @@

What's New

Docs →
- -
- GPT-Live Sessions -

- Mock, record, and replay OpenAI GPT-Live sessions, with message, audio, and lifetime - limits -

- Docs → -
diff --git a/docs/fixtures/index.html b/docs/fixtures/index.html index ca25fced..8aee4fb9 100644 --- a/docs/fixtures/index.html +++ b/docs/fixtures/index.html @@ -979,6 +979,23 @@

Response Override Fields

reason. Does not label reasoning or tool-call items + + + + + + + + + + @@ -1012,6 +1029,113 @@

Response Override Fields

citationsarray + Source citations for the reply text, served on OpenAI Responses, Chat Completions, + Claude and Gemini (see + Citations and generated files) +
filesarray + Model-generated files { mimeType, data }, served on Gemini and OpenAI + Responses (see Citations and generated files) +
systemFingerprint string
+

Citations and generated files

+

+ A text reply can carry source citations, and a reply can carry model-generated files such + as images. Add citations or files next to content, + and aimock serves them in each provider's own wire shape, streaming and non-streaming. Use + them to test how your app shows sources and renders generated images. +

+
+
citations.json json
+ 
{
+  "match": { "userMessage": "capital of France" },
+  "response": {
+    "content": "Paris is the capital of France.",
+    "citations": [
+      { "url": "https://en.wikipedia.org/wiki/Paris", "title": "Paris", "startIndex": 0, "endIndex": 5 },
+      { "type": "file", "fileId": "file-abc123", "filename": "notes.pdf" }
+    ]
+  }
+}
+
+

+ A URL citation needs url; title, startIndex and + endIndex are optional, and the span defaults to the whole text. Offsets are + JavaScript string positions in the reply text. A file citation (type: "file") + needs fileId and filename; its optional + index defaults to the end of the text. +

+
+
+ generated-image.json json +
+ 
{
+  "match": { "userMessage": "draw a cat" },
+  "response": {
+    "content": "Here is a cat.",
+    "files": [{ "mimeType": "image/png", "data": "iVBORw0KGgo..." }]
+  }
+}
+
+

+ data holds the base64 file bytes. Files are sent before the reply text and + tool calls. +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Wirecitationsfiles
OpenAI Responses (HTTP and WebSocket) + url_citation and file_citation annotations on + output_text, streamed with + response.output_text.annotation.added + + image_generation_call output items (image/* only), + streamed with the response.image_generation_call.* events +
Chat Completions (OpenAI, Azure, OpenRouter) + url_citation in message.annotations; one + delta.annotations chunk when streaming. File citations are skipped + 500 aimock_unsupported_generated_file
Claude Messages + web_search_result_location citations on text blocks; + citations_delta when streaming. File citations are skipped + 500 aimock_unsupported_generated_file
Gemini and Vertex AI + groundingMetadata (groundingChunks and + groundingSupports, byte offsets). File citations are skipped + inlineData parts
Every other wireIgnoredIgnored
+
+

+ Errors: a malformed citations or files value + (wrong type, missing field, an offset outside the text, or citations on a reply with no + text) answers a 500 with aimock_invalid_fixture_citation or + aimock_invalid_fixture_file before any content is sent. A non-image file on + the Responses API answers aimock_unsupported_generated_file. On the + Responses WebSocket these come as an error event. Fixture loading warns about the same + problems, and the fixture still loads. +

+
+

Fixture Options

@@ -1469,6 +1593,34 @@

Provider Support Matrix

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/gemini/index.html b/docs/gemini/index.html index 852a1ecb..d8046f93 100644 --- a/docs/gemini/index.html +++ b/docs/gemini/index.html @@ -58,6 +58,11 @@

Google Gemini

To send events or fields aimock does not model yet, use a raw passthrough fixture with wire: "gemini".

+

+ Fixture citations serve groundingMetadata, and fixture + files serve inlineData parts, such as generated images. See + Citations and generated files. +

Endpoints

— Yes
CitationsURLURL + fileURLURL—URL—URL——URL
Generated filesErrorImagesErrorYes—Yes—Error——Error
Custom Tool Calls‡ Error
diff --git a/docs/index.html b/docs/index.html index ec356212..95ab3e38 100644 --- a/docs/index.html +++ b/docs/index.html @@ -3010,7 +3010,7 @@ 

$ npx @copilotkit/aimock --config aimock.json
 
-⚡ aimock v1.46.0
+⚡ aimock v1.47.0
 
 ✓ LLM    mounted at  /v1/chat/completions
 ✓ LLM    mounted at  /v1/messages
@@ -3571,6 +3571,54 @@ 
LLM and protocol mocking

                   >
                 
               
+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/responses-api/index.html b/docs/responses-api/index.html index 41b26874..04ca811a 100644 --- a/docs/responses-api/index.html +++ b/docs/responses-api/index.html @@ -58,6 +58,12 @@

OpenAI Responses API

raw passthrough fixture with wire: "openai-responses".

+

+ Fixture citations serve url_citation and + file_citation annotations, and fixture files serve + image_generation_call output items. See + Citations and generated files. +

Endpoints

Provider-native source citations + Built-in ✓
1.47.0+ + 		manualmanualmanualNDNDmanualmanual
Model-generated images in replies + Built-in ✓
1.47.0+ + 		manualmanualmanualNDNDmanualmanual
Client disconnect stops the reply on every provider + Built-in ✓
1.47.0+ + 		NDNDNDpartialpartialNDND
Streaming physics Built-in ✓
diff --git a/package.json b/package.json index 7ce47091..5e0b918f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@copilotkit/aimock", - "version": "1.46.0", + "version": "1.47.0", "description": "Mock infrastructure for AI application testing — point your SDK at one local port and every provider, protocol, and service answers deterministically.", "license": "MIT", "keywords": [ diff --git a/packages/aimock-pytest/README.md b/packages/aimock-pytest/README.md index 967efe2b..f885a27c 100644 --- a/packages/aimock-pytest/README.md +++ b/packages/aimock-pytest/README.md @@ -95,7 +95,7 @@ These need aimock 1.45.0 or later. ``` --aimock-node PATH Path to node binary ---aimock-version VER aimock npm version (default: 1.46.0) +--aimock-version VER aimock npm version (default: 1.47.0) --aimock-api-key KEY Inbound API key for the aimock child process ``` diff --git a/packages/aimock-pytest/src/aimock_pytest/_version.py b/packages/aimock-pytest/src/aimock_pytest/_version.py index c4a7beb3..d82cb82f 100644 --- a/packages/aimock-pytest/src/aimock_pytest/_version.py +++ b/packages/aimock-pytest/src/aimock_pytest/_version.py @@ -11,4 +11,4 @@ a wheel. Keep it tracking npm releases. """ -AIMOCK_VERSION = "1.46.0" +AIMOCK_VERSION = "1.47.0" diff --git a/src/__tests__/competitive-matrix-homepage.test.ts b/src/__tests__/competitive-matrix-homepage.test.ts index 8b2a0d45..741a097d 100644 --- a/src/__tests__/competitive-matrix-homepage.test.ts +++ b/src/__tests__/competitive-matrix-homepage.test.ts @@ -921,6 +921,9 @@ describe("hand-reviewed homepage rows with no scan rule", () => { "Native model catalogs (Anthropic, Gemini)", "Token-count endpoints", "Native 401 auth errors", + "Provider-native source citations", + "Model-generated images in replies", + "Client disconnect stops the reply on every provider", ]; const MIGRATION_HAND_ROWS = [ "Native model catalogs", diff --git a/src/__tests__/docs-matrix-legend.test.ts b/src/__tests__/docs-matrix-legend.test.ts index 7ad9dcaa..bec6c8b6 100644 --- a/src/__tests__/docs-matrix-legend.test.ts +++ b/src/__tests__/docs-matrix-legend.test.ts @@ -85,7 +85,7 @@ describe("docs Not documented cells", () => { it("renders the homepage's Not documented cells as ND abbreviations", () => { const src = readFileSync(join(DOCS, "index.html"), "utf-8"); const tablesOnly = [...src.matchAll(//g)].map((m) => m[0]).join(""); - expect(tablesOnly.split(ND).length - 1).toBe(47); + expect(tablesOnly.split(ND).length - 1).toBe(56); }); it("styles the legend with one shared class on each stylesheet", () => {