diff --git a/docs-site/src/content/docs/guides/sidecars.md b/docs-site/src/content/docs/guides/sidecars.md index 1e353a4a48b..4df1c0cfcf7 100644 --- a/docs-site/src/content/docs/guides/sidecars.md +++ b/docs-site/src/content/docs/guides/sidecars.md @@ -138,6 +138,19 @@ failures after response headers have started are delivered as `response.failed` ## Vision sidecar +Mistral's `zai-glm-5-3` is enrolled by the provider preset as text-only. With the +vision sidecar enabled and available, OpenCodex describes attached images and +sends their text descriptions to GLM. The Codex catalog advertises image +attachments for this route. After an update, run `ocx sync`, then fully quit and +relaunch the Codex desktop app or reload the VS Code window. Sync without a +restart flag does not restart an existing Codex app-server, which can retain +the old model list. Alternatively, use `ocx sync --restart-codex` to sync and +restart matching Codex processes and the desktop app. Finish active work first: +this restart ends live conversations. See the [sync and restart reference](/reference/cli/lifecycle/#ocx-sync---restart-codex---restart-app-server-only). +After relaunch, select `mistral/zai-glm-5-3` and verify that image attachments +are available. No model modality override is needed. Mistral documents native input in its +[GLM 5.3 reference](https://docs.mistral.ai/models/zai-glm-5-3). + Image routing is capability-aware. Before an image-bearing upstream send, opencodex resolves the selected model's effective input modalities from runtime provider evidence, explicit operator declarations, backend/registry metadata, and generated vendor metadata. A target positively known to be text-only goes through the Vision Sidecar first; the image is described **before** the main call and replaced inline with text. A target positively known to support images receives the image directly. Unknown custom models keep the existing compatibility behavior rather than being guessed text-only. For the canonical ChatGPT Codex route, opencodex uses the `openai-codex` metadata bundle rather than public OpenAI API metadata, so backend-specific modality differences are respected. The native Chat fast path uses the same gate and cannot bypass a known text-only verdict. Without an available sidecar plan, raw images are stripped before a proven text-only backend. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index d9502640531..c2501cd4026 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1886,6 +1886,7 @@ "vision-reasoning-contract.test.ts": "vision", "vision-routed.test.ts": "vision", "vision-sidecar-e2e.test.ts": "vision", + "mistral-vision-sidecar.test.ts": "vision", "vision-sidecar-timeout-bounds.test.ts": "gui", "vision-text-only-predicate.test.ts": "vision", "volcengine-ark-assistant-content.test.ts": "providers", diff --git a/src/providers/registry/entries-extended.ts b/src/providers/registry/entries-extended.ts index 3756e01d285..b90e61f74d6 100644 --- a/src/providers/registry/entries-extended.ts +++ b/src/providers/registry/entries-extended.ts @@ -1020,7 +1020,12 @@ export const PROVIDER_REGISTRY_EXTENDED: readonly ProviderRegistryEntry[] = [ }, }, // FREEZE 2026-07-10: codestral-latest is unconfirmed behind auth. Evidence: devlog/_plan/260710_provider_hardening/003_research_aggregators.md. - { id: "mistral", label: "Mistral", baseUrl: "https://api.mistral.ai/v1", adapter: "openai-chat", authKind: "key", dashboardUrl: "https://console.mistral.ai/api-keys", defaultModel: "codestral-latest" }, + { + id: "mistral", label: "Mistral", baseUrl: "https://api.mistral.ai/v1", adapter: "openai-chat", authKind: "key", + dashboardUrl: "https://console.mistral.ai/api-keys", defaultModel: "codestral-latest", + // Native text-only input enrolls this model in the existing vision sidecar. + modelInputModalities: { "zai-glm-5-3": ["text"] }, + }, { id: "minimax", label: "MiniMax — Coding Plan", baseUrl: "https://api.minimax.io/v1", adapter: "openai-chat", authKind: "key", dashboardUrl: "https://platform.minimax.io", defaultModel: "MiniMax-M3", models: MINIMAX_MODELS, diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index a7526944ee6..718041a2cfc 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -1,4 +1,9 @@ # Providers And Adapters + +Mistral's registry entry in `src/providers/registry/entries-extended.ts` declares `zai-glm-5-3` native input as text-only. +Registry enrichment supplies that default to existing configurations: the catalog advertises image attachments +through the enabled vision sidecar and request preprocessing sends GLM descriptions. Explicit operator modalities retain precedence. + GCP JSON storage in `src/providers/api-keys.ts` and durable-row snapshot/post-commit cleanup in `src/providers/gcp-credential-cleanup.ts` follow [Vertex credential markers](providers/google.md#vertex-credential-markers); malformed blank marker accounts do not block cleanup of valid accounts. Combo target `metered` metadata and configured-key opt-in follow the [runtime consent contract](runtime.md#request-local-target-compatibility). Subscription-to-Cursor-to-key regression coverage in `tests/adapters/anthropic/anthropic-subscription-chain.test.ts` checks unchanged healthy subscription wire bodies, effort, key authentication and cache controls, and structured runTurn 429 propagation. OpenCode Go's exact Gemini pin and credential destination guard follow the [Go Gemini transport contract](providers/google.md#opencode-go-gemini-transport). Configured spend ceilings bind adapter inference executors to the [seed and physical-start contract](transports/responses-spend.md#historical-pool-continuity-and-rollback). Zed completion and auth-recovery replays claim separately; token acquisition stays outside inference accounting. Claude CLI, CodeBuddy and Qoder each count one invocation toward the frozen request limit, seed before spawn and retain a producer through completion. Child-internal retries/tool turns are not separately counted; reported actual usage settles in full, including amounts above the seed estimate. Devin combines only [consecutive same-ID tool results](adapters/registry.md#devin-consecutive-tool-results), using linear accumulation while preserving intervening message slots, image parts and error markers without mutating the parsed request. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 3a82672e452..acccdd23325 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1265,6 +1265,7 @@ "vision-custom-row-precedence.test.ts": "vision", "vision-eligibility.test.ts": "vision", "vision-fail-closed.test.ts": "vision", "vision-reasoning-contract.test.ts": "vision", "vision-routed.test.ts": "vision", "vision-sidecar-e2e.test.ts": "vision", + "mistral-vision-sidecar.test.ts": "vision", "vision-sidecar-timeout-bounds.test.ts": "gui", "vision-text-only-predicate.test.ts": "vision", "volcengine-ark-assistant-content.test.ts": "providers", "volcengine-providers.test.ts": "gui", "warmup-registration.test.ts": "ci-workflows", "warmup.test.ts": "codex-integration", diff --git a/tests/vision/mistral-vision-sidecar.test.ts b/tests/vision/mistral-vision-sidecar.test.ts new file mode 100644 index 00000000000..d6883474ad3 --- /dev/null +++ b/tests/vision/mistral-vision-sidecar.test.ts @@ -0,0 +1,174 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveConfig } from "../../src/config"; +import { buildCatalogEntries, gatherRoutedModels } from "../../src/codex/catalog"; +import { clearModelCache } from "../../src/codex/model-cache"; +import { routeModel } from "../../src/router"; +import { startServer } from "../../src/server"; +import type { OcxConfig } from "../../src/types"; +import { resetVisionDescriptionCache } from "../../src/vision"; +import { requiresVisionPreprocessing } from "../../src/vision/plan"; +import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; +import { withStubbedProviderFetch } from "../helpers/catalog-provider-fetch"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const GLM = "zai-glm-5-3"; +const NATIVE = "mistral-large-4-0"; +const IMAGE = "data:image/png;base64,aGVsbG8taW1hZ2UtYnl0ZXM="; +const CAPTION = "A red square with a white circle."; +const originalFetch = globalThis.fetch; +let previousHome: string | undefined; +let home = ""; +let codexHome: IsolatedCodexHome; +let sidecarCalls: Array>; +let mainCalls: Array>; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + codexHome = installIsolatedCodexHome("ocx-mistral-vision-codex-"); + home = mkdtempSync(join(tmpdir(), "ocx-mistral-vision-")); + process.env.OPENCODEX_HOME = home; + clearModelCache(); + resetVisionDescriptionCache(); + sidecarCalls = []; + mainCalls = []; + globalThis.fetch = (async (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.origin === "https://api.mistral.ai") { + if (url.pathname === "/v1/models") return Response.json({ data: [ + { id: GLM, capabilities: { vision: false }, max_context_length: 1_048_576 }, + { id: NATIVE, capabilities: { vision: true }, max_context_length: 1_048_576 }, + ] }); + if (url.pathname === "/v1/chat/completions") { + mainCalls.push(JSON.parse(String(init?.body))); + return Response.json({ + id: "chatcmpl-mistral-vision", object: "chat.completion", + choices: [{ index: 0, message: { role: "assistant", content: "A red square." }, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 4, total_tokens: 14 }, + }); + } + } + if (url.origin === "https://chatgpt.com" && url.pathname === "/backend-api/codex/responses") { + sidecarCalls.push(JSON.parse(String(init?.body))); + return new Response(`data: ${JSON.stringify({ type: "response.output_text.delta", delta: CAPTION })}\n\ndata: [DONE]\n\n`, { + headers: { "content-type": "text/event-stream" }, + }); + } + if (url.hostname === "127.0.0.1") return originalFetch(input, init); + throw new Error("Unexpected network destination in Mistral vision fixture"); + }) as typeof fetch; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + clearModelCache(); + resetVisionDescriptionCache(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome.restore(); + removeTreeWithRetry(home); +}); + +/** Build an isolated Mistral configuration with a caller-authenticated OpenAI vision helper. */ +function configuration(): OcxConfig { + return { + port: 0, hostname: "127.0.0.1", defaultProvider: "mistral", openaiProviderTierVersion: 2, + providers: { + mistral: { adapter: "openai-chat", baseUrl: "https://api.mistral.ai/v1", apiKey: "fixture-mistral-key" }, + openai: { + adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", + baseUrl: "https://chatgpt.com/backend-api/codex", + }, + }, + }; +} + +describe("Mistral GLM vision sidecar defaults", () => { + test("live discovery publishes GLM attachments without a saved modality override", async () => { + const config = configuration(); + const models = await gatherRoutedModels(withStubbedProviderFetch(config)); + expect(buildCatalogEntries(null, [], models).find(row => row.slug === `mistral/${GLM}`)?.input_modalities) + .toEqual(["text", "image"]); + const route = routeModel(config, `mistral/${GLM}`); + expect(requiresVisionPreprocessing(config, route.provider, GLM, "mistral")).toBe(true); + expect(config.providers.mistral!.modelInputModalities).toBeUndefined(); + expect(config.providers.mistral!.modelCapabilities).toBeUndefined(); + }); + + test("disabled vision sidecar leaves the GLM catalog text-only", async () => { + const config = { ...configuration(), visionSidecar: { enabled: false } }; + const models = await gatherRoutedModels(withStubbedProviderFetch(config)); + expect(models.find(row => row.provider === "mistral" && row.id === GLM)?.inputModalities).toEqual(["text"]); + }); + + test("native Mistral vision and explicit operator declarations keep their precedence", async () => { + const config = configuration(); + config.providers.mistral!.modelInputModalities = { [GLM]: ["text", "image"] }; + const models = await gatherRoutedModels(withStubbedProviderFetch(config)); + expect(models.find(row => row.provider === "mistral" && row.id === GLM)?.inputModalities).toEqual(["text", "image"]); + expect(models.find(row => row.provider === "mistral" && row.id === NATIVE)?.inputModalities).toEqual(["text", "image"]); + for (const id of [GLM, NATIVE]) { + const route = routeModel(config, `mistral/${id}`); + expect(requiresVisionPreprocessing(config, route.provider, id, "mistral")).toBe(false); + } + }); + + test("an exact capability override still wins over the GLM preset", () => { + const config = configuration(); + config.providers.mistral!.modelCapabilities = { [GLM]: { inputModalities: ["text", "image"] } }; + const route = routeModel(config, `mistral/${GLM}`); + expect(requiresVisionPreprocessing(config, route.provider, GLM, "mistral")).toBe(false); + }); + + test("the GLM enrollment does not apply to unknown Mistral models or another provider", () => { + const config = configuration(); + config.providers.other = { adapter: "openai-chat", baseUrl: "https://other.example/v1", apiKey: "fixture-other-key" }; + for (const selector of ["mistral/unknown-text-model", `other/${GLM}`]) { + const route = routeModel(config, selector); + expect(requiresVisionPreprocessing(config, route.provider, route.modelId, route.providerName)).toBe(false); + } + }); + + test.each(["sidecar", "disabled", "native"] as const)("image request uses the %s path", async mode => { + const config = configuration(); + if (mode === "disabled") config.visionSidecar = { enabled: false }; + saveConfig(config); + const server = startServer(0); + try { + const token = fakeChatGptJwt({ chatgpt_account_id: "acct-mistral-vision-fixture" }); + const response = await fetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { + "content-type": "application/json", authorization: `Bearer ${token}`, + "chatgpt-account-id": "acct-mistral-vision-fixture", + }, + body: JSON.stringify({ model: `mistral/${mode === "native" ? NATIVE : GLM}`, stream: false, + input: [{ role: "user", content: [ + { type: "input_text", text: "Describe this image." }, { type: "input_image", image_url: IMAGE }, + ] }], + }), + }); + expect(response.status).toBe(200); + await response.text(); + expect(mainCalls).toHaveLength(1); + expect(sidecarCalls).toHaveLength(mode === "sidecar" ? 1 : 0); + const forwarded = JSON.stringify(mainCalls[0]); + if (mode === "sidecar") { + expect(sidecarCalls[0]!.model).toBe("gpt-5.6-luna"); + expect(JSON.stringify(sidecarCalls[0])).toContain(IMAGE); + expect(forwarded).toContain(CAPTION); + } + if (mode === "native") expect(forwarded).toContain(IMAGE); + else { + expect(forwarded).not.toContain(IMAGE); + expect(forwarded).not.toContain("image_url"); + } + if (mode === "disabled") expect(forwarded).toContain("[image omitted:"); + } finally { + await server.stop(true); + } + }); +});