diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 3b419ea9e6a..942feb374b4 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -713,6 +713,25 @@ even when every target supports images — the catalog drops `image` from `input image-bearing requests are rejected with HTTP 400 before any target is called. `"auto"` (or omitting the field) keeps the automatic intersection. +In the dashboard, the **Image / multimodal** switch can be enabled whenever every target has +known image or text input capabilities. Native image members stay unchanged; text-only members are +automatically declared text-only on save (`modelCapabilities[model].inputModalities = ["text"]`) +so the [Vision Sidecar](/guides/sidecars/) describes their images. The switch's hint names the +members that will be enrolled. A member whose modalities are unknown or have no text input (for +example an audio-only model) cannot be covered by the sidecar; it keeps the switch unavailable and +is named in the hint. If the [Vision Sidecar](/guides/sidecars/) is disabled globally, enabling the +switch shows a warning linking to the dashboard: enrollment still saves, but images are replaced +with an omission note and requests continue without image descriptions until the sidecar is +re-enabled. Turning the switch off disables image input for the combo +but keeps those provider declarations, and `PUT /api/combos` accepts the same enrollment as an +optional top-level `visionSidecarTargets` array of exact `{ provider, model }` targets +(request-only; it is never stored on the combo and is rejected while `imageInput` is +`"disabled"` when the enrollment list is non-empty). Entries that already accept image input or +are declared without text are rejected with HTTP 400 rather than overwritten, entries the sidecar +already covers are skipped, and a save refused before publication restores the combo and every declaration together. +Removing a member from the combo does not remove its text-only declaration; clear it in the +provider editor if you no longer want it. + ## Encrypted v2 sub-agent tasks There is one important limitation for Codex v2 sub-agents ([issue #92](https://github.com/lidge-jun/opencodex/issues/92)). diff --git a/docs-site/src/content/docs/guides/sidecars.md b/docs-site/src/content/docs/guides/sidecars.md index 3d35989872b..09e52d28db2 100644 --- a/docs-site/src/content/docs/guides/sidecars.md +++ b/docs-site/src/content/docs/guides/sidecars.md @@ -148,6 +148,12 @@ allow attachments instead of blocking them before the sidecar runs. When use the `gpt-5.6-luna` fallback. Startup still migrates an explicitly persisted legacy `gpt-5.4-mini` value to `gpt-5.6-luna`; that migration applies to a stored value, not to an absent model field. +Saving a multimodal combo from the dashboard enrolls its non-image members automatically: the +Combos page sends them as `visionSidecarTargets` on `PUT /api/combos`, which writes the exact +text-only declaration on the member's provider, so no manual config edit and reload is needed. +Enrollment never overwrites an existing image-capable declaration, and members whose modalities are +unknown or have no text input are not enrolled — the dashboard names them and keeps the combo's +image switch unavailable. The first-party DeepSeek `deepseek-flash` model is native multimodal (`text` and `image`) and does not use this sidecar by default. Explicit `noVisionModels` or text-only declarations remain authoritative. First-party `deepseek-chat`, `deepseek-reasoner`, and `deepseek-v4-flash` remain diff --git a/gui/src/combo-capabilities.ts b/gui/src/combo-capabilities.ts index 32f91f66b98..587a4d631e5 100644 --- a/gui/src/combo-capabilities.ts +++ b/gui/src/combo-capabilities.ts @@ -1,14 +1,74 @@ import type { ComboTarget } from "./combo-workspace-data"; import type { ModelOption } from "./components/combo-workspace-types"; -/** Whether every selected target advertises image input (incomplete rows fail closed). */ +type ComboImageMemberKind = "vision" | "sidecar" | "blocked" | "missing"; + +/** + * One combo member's image story. Classification must survive a reload, so it + * reads `visionSidecarConsumer` — the runtime predicate's verdict served on the + * /api/models row — before falling back to the DECLARED modalities + * (`inputModalitiesDeclared`). A model covered only by legacy `noVisionModels` + * has no declaration and its advertised modalities are widened to image, so the + * flag is what keeps it a sidecar member after reload. Rows with no known + * modalities, or modalities without text, cannot be covered and block the combo. + */ +function imageMemberKind(target: ComboTarget, models: ModelOption[]): ComboImageMemberKind { + const provider = target.provider.trim(); + const modelId = target.model.trim(); + if (!provider || !modelId) return "missing"; + const model = models.find((row) => row.provider === provider && row.id === modelId); + if (!model) return "missing"; + if (model.visionSidecarConsumer === true) return "sidecar"; + const declared = model.inputModalitiesDeclared ?? model.inputModalities; + if (!declared || declared.length === 0) return "blocked"; + if (declared.includes("image")) return "vision"; + return declared.includes("text") ? "sidecar" : "blocked"; +} + +/** Whether images can be enabled: every target is known and either images natively or can be declared text-only. */ export function comboImagesSupported(targets: ComboTarget[], models: ModelOption[]): boolean { if (targets.length === 0) return false; return targets.every((target) => { - const provider = target.provider.trim(); - const modelId = target.model.trim(); - if (!provider || !modelId) return false; - const model = models.find((row) => row.provider === provider && row.id === modelId); - return !!model?.inputModalities?.includes("image"); + const kind = imageMemberKind(target, models); + return kind === "vision" || kind === "sidecar"; }); } + +/** + * Exact targets that need a text-only declaration so the Vision Sidecar covers + * them when the combo accepts images. Deduplicated in submission order. Empty + * while any member blocks images: the switch is forced off then, so a save must + * not declare the remaining members text-only behind the operator's back. + */ +export function comboVisionSidecarTargets( + targets: ComboTarget[], + models: ModelOption[], +): Array<{ provider: string; model: string }> { + if (!comboImagesSupported(targets, models)) return []; + const seen = new Set(); + const out: Array<{ provider: string; model: string }> = []; + for (const target of targets) { + if (imageMemberKind(target, models) !== "sidecar") continue; + const provider = target.provider.trim(); + const model = target.model.trim(); + const key = `${provider}/${model}`; + if (seen.has(key)) continue; + seen.add(key); + out.push({ provider, model }); + } + return out; +} + +/** + * Exact targets that cannot be covered by the Vision Sidecar: their row is + * known but its modalities are unknown or have no text input. Named in the + * hint so the operator knows which member blocks the image switch. + */ +export function comboImageBlockedTargets( + targets: ComboTarget[], + models: ModelOption[], +): Array<{ provider: string; model: string }> { + return targets + .filter((target) => imageMemberKind(target, models) === "blocked") + .map((target) => ({ provider: target.provider.trim(), model: target.model.trim() })); +} diff --git a/gui/src/combo-workspace-data.ts b/gui/src/combo-workspace-data.ts index f579a6f2b28..88c9b9b0528 100644 --- a/gui/src/combo-workspace-data.ts +++ b/gui/src/combo-workspace-data.ts @@ -504,9 +504,13 @@ export function draftEquals(a: ComboItem, b: ComboItem): boolean { }); } -export function toPutBody(item: ComboItem, options: { renameFrom?: string } = {}): { +export function toPutBody( + item: ComboItem, + options: { renameFrom?: string; visionSidecarTargets?: Array<{ provider: string; model: string }> } = {}, +): { id: string; renameFrom?: string; + visionSidecarTargets?: Array<{ provider: string; model: string }>; combo: { targets: ComboTarget[]; strategy: ComboStrategy; @@ -526,6 +530,7 @@ export function toPutBody(item: ComboItem, options: { renameFrom?: string } = {} return { id: item.id.trim(), ...(options.renameFrom ? { renameFrom: options.renameFrom } : {}), + ...(options.visionSidecarTargets?.length ? { visionSidecarTargets: options.visionSidecarTargets } : {}), combo: { targets: item.targets.map((target) => ({ provider: target.provider.trim(), diff --git a/gui/src/components/ComboWorkspace.tsx b/gui/src/components/ComboWorkspace.tsx index fa7ef50230d..1f513da07a6 100644 --- a/gui/src/components/ComboWorkspace.tsx +++ b/gui/src/components/ComboWorkspace.tsx @@ -26,6 +26,7 @@ export default function ComboWorkspace({ models, capabilityModels = models, cataloguedComboIds, + visionEnabled, loading, onRefresh, onSave, @@ -226,6 +227,7 @@ export default function ComboWorkspace({ providers={providers} models={models} capabilityModels={capabilityModels} + visionEnabled={visionEnabled} onBack={() => trySelect(null)} onSaved={(item) => { setDetailDirty(false); @@ -256,6 +258,7 @@ export default function ComboWorkspace({ providers={providers} models={models} capabilityModels={capabilityModels} + visionEnabled={visionEnabled} onSaved={(item) => { setDetailDirty(false); setSelectedId(item.id); diff --git a/gui/src/components/combo-workspace-controls.tsx b/gui/src/components/combo-workspace-controls.tsx index 50f8cfb4dc7..d9ad1f6b337 100644 --- a/gui/src/components/combo-workspace-controls.tsx +++ b/gui/src/components/combo-workspace-controls.tsx @@ -1,6 +1,6 @@ import { useState } from "react"; import type { ComboEffort, ComboStrategy, ComboTarget, ProviderQuotaStates } from "../combo-workspace-data"; -import { comboImagesSupported } from "../combo-capabilities"; +import { comboImageBlockedTargets, comboImagesSupported, comboVisionSidecarTargets } from "../combo-capabilities"; import { COMBO_EFFORTS, COMBO_STRATEGIES, @@ -8,6 +8,7 @@ import { newComboTarget, } from "../combo-workspace-data"; import { IconArrowDown, IconArrowUp, IconGrip, IconPlus, IconTrash } from "../icons"; +import { navigateHash } from "../hash-routing"; import { useT } from "../i18n/shared"; import { Switch } from "../ui"; import { formatProviderDisplayName } from "../provider-icons"; @@ -93,6 +94,7 @@ export function ComboCapabilities({ models, imageInput, reasoningEffortMode, + visionEnabled, disabled, onChange, }: { @@ -100,13 +102,30 @@ export function ComboCapabilities({ models: ModelOption[]; imageInput: "auto" | "disabled"; reasoningEffortMode: "strict" | "adaptive"; + /** Vision Sidecar enabled state; undefined = unknown, renders no warning. */ + visionEnabled?: boolean; disabled?: boolean; onChange: (patch: { imageInput?: "auto" | "disabled"; reasoningEffortMode?: "strict" | "adaptive" }) => void; }) { const t = useT(); const imagesSupported = comboImagesSupported(targets, models); + const blockedTargets = comboImageBlockedTargets(targets, models); // Default: checked (auto) when supported; force off when any target lacks image. const effectiveOn = imagesSupported && imageInput !== "disabled"; + const sidecarTargets = comboVisionSidecarTargets(targets, models); + // Enrollment wording only while image input is enabled: a disabled combo is not + // enrolling anything, so it gets the plain hint instead of save-time promises. + const imageHint = !imagesSupported + ? blockedTargets.length > 0 + ? t("cws.capability.imageInputBlockedHint", { + models: blockedTargets.map(({ provider, model }) => `${provider}/${model}`).join(", "), + }) + : t("cws.capability.imageInputUnavailable") + : sidecarTargets.length > 0 && imageInput !== "disabled" + ? t("cws.capability.imageInputSidecarHint", { + models: sidecarTargets.map(({ provider, model }) => `${provider}/${model}`).join(", "), + }) + : t("cws.capability.imageInputHint"); return (
@@ -115,8 +134,18 @@ export function ComboCapabilities({
{t("cws.capability.imageInput")}

- {imagesSupported ? t("cws.capability.imageInputHint") : t("cws.capability.imageInputUnavailable")} + {imageHint}

+ {sidecarTargets.length > 0 && effectiveOn && visionEnabled === false ? ( +

+ {t("cws.capability.imageInputSidecarDisabled", { + models: sidecarTargets.map(({ provider, model }) => `${provider}/${model}`).join(", "), + })}{" "} + +

+ ) : null}
void; onSaved: (item: ComboItem) => void; onRequestRemove?: () => void; @@ -411,6 +414,7 @@ export function DetailPanel({ models={capabilityModels} imageInput={draft.imageInput ?? "auto"} reasoningEffortMode={draft.reasoningEffortMode ?? "strict"} + visionEnabled={visionEnabled} disabled={busy} onChange={(patch) => updateDraft((d) => ({ ...d, ...patch }))} /> diff --git a/gui/src/components/combo-workspace-types.ts b/gui/src/components/combo-workspace-types.ts index 77d86a6f535..d2b613dd7f5 100644 --- a/gui/src/components/combo-workspace-types.ts +++ b/gui/src/components/combo-workspace-types.ts @@ -17,6 +17,10 @@ export type ModelOption = { namespaced?: string; reasoningEfforts?: string[]; inputModalities?: string[]; + /** Operator-declared modalities; beats the (sidecar-widened) catalog view on reload. */ + inputModalitiesDeclared?: string[]; + /** Authoritative runtime verdict from /api/models: the Vision Sidecar describes this model's images. */ + visionSidecarConsumer?: boolean; }; export type ComboAddIntent = "blank" | "jev-auto"; @@ -32,6 +36,8 @@ export interface ComboWorkspaceProps { capabilityModels?: ModelOption[]; /** Combo ids currently present in the live catalog (`provider === "combo"`). */ cataloguedComboIds?: ReadonlySet; + /** Vision Sidecar enabled state from /api/sidecar-settings; undefined = unknown, no warning. */ + visionEnabled?: boolean; loading?: boolean; onRefresh: () => void; onSave: (item: ComboItem, isCreate: boolean, renameFrom?: string) => Promise<{ ok: boolean; error?: string }>; diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 4973d9e3b4c..60adc47976d 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -3207,8 +3207,11 @@ export const de: Record = { "cws.field.defaultEffort": "Standard-Reasoning", "cws.field.defaultEffortNone": "Keine (Ziel-Standard)", "cws.field.defaultEffortHint": "Nur verwendet, wenn der Client keinen Reasoning-Aufwand sendet. Optionen sind die Schnittmenge der beworbenen Aufwände der gewählten Ziele.", - "cws.capability.imageInputUnavailable": "Erst verfügbar, wenn jedes gewählte Ziel Bildeingabe unterstützt.", + "cws.capability.imageInputUnavailable": "Wähle zuerst alle Ziele aus dem Katalog — unbekannte Modelle kann der Vision Sidecar nicht abdecken.", "cws.capability.imageInputHint": "Standardmäßig aktiv, wenn jedes Ziel Bilder unterstützt. Ausschalten für nur Text.", + "cws.capability.imageInputSidecarHint": "Standardmäßig aktiv. {models} wird beim Speichern als text-only deklariert und nutzt den Vision Sidecar für Bilder.", + "cws.capability.imageInputSidecarDisabled": "Der Vision Sidecar ist ausgeschaltet — an {models} gesendete Bilder werden durch einen Hinweistext ersetzt und die Anfrage läuft ohne sie weiter. Aktiviere ihn in den Dashboard-Einstellungen, um Bildbeschreibungen wiederherzustellen.", + "cws.capability.imageInputBlockedHint": "{models} kann vom Vision Sidecar nicht abgedeckt werden: Eingabemodalitäten sind unbekannt oder ohne Text.", "cws.capability.imageInput": "Bild / multimodal", "cws.capability.adaptiveEffort": "Adaptive Denkstufen", "cws.capability.adaptiveEffortHint": "Aus: Ziele ohne Denkstufen-Regelung blenden die Auswahl für die gesamte Kombination aus. An: Solche Ziele bleiben nutzbar, und die Auswahl zeigt weiterhin die Stufen der übrigen Ziele.", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index 29e27e4c18a..42acd763e7a 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -3321,8 +3321,11 @@ export const en = { "cws.field.defaultEffort": "Default reasoning", "cws.field.defaultEffortNone": "None (target default)", "cws.field.defaultEffortHint": "Used only when the client omits reasoning effort. Options are the intersection of the selected targets' advertised efforts; targets without catalog effort metadata offer none.", - "cws.capability.imageInputUnavailable": "Unavailable until every selected target supports image input.", + "cws.capability.imageInputUnavailable": "Pick every target from the catalog first — unknown models can't be covered by the Vision Sidecar.", "cws.capability.imageInputHint": "On by default when every target supports images. Turn off to accept text only.", + "cws.capability.imageInputSidecarHint": "On by default. {models} will be declared text-only on save and use the Vision Sidecar for images.", + "cws.capability.imageInputSidecarDisabled": "The Vision Sidecar is turned off — images sent to {models} are replaced with an omission note and the request continues without them. Enable it in dashboard settings to restore image descriptions.", + "cws.capability.imageInputBlockedHint": "{models} cannot be covered by the Vision Sidecar: its input modalities are unknown or have no text.", "cws.capability.imageInput": "Image / multimodal", "cws.capability.adaptiveEffort": "Adaptive reasoning ladder", "cws.capability.adaptiveEffortHint": "Off: a target with no reasoning control hides the effort picker for the whole combo. On: those targets stay usable and the picker keeps the levels the remaining targets share.", diff --git a/gui/src/i18n/fr.ts b/gui/src/i18n/fr.ts index 1f387a8bafb..0be98536b37 100644 --- a/gui/src/i18n/fr.ts +++ b/gui/src/i18n/fr.ts @@ -3199,8 +3199,11 @@ export const fr: Record = { "cws.emptyTitle": "Créer votre première combinaison", "cws.empty.createDesc": "Nommez un modèle virtuel et enchaînez au moins deux services principaux.", "cws.backToAll": "Retour à toutes les combinaisons", - "cws.capability.imageInputUnavailable": "Indisponible tant que toutes les cibles sélectionnées ne prennent pas en charge les images.", + "cws.capability.imageInputUnavailable": "Sélectionnez d'abord chaque cible dans le catalogue — les modèles inconnus ne peuvent pas être couverts par le Vision Sidecar.", "cws.capability.imageInputHint": "Activé par défaut lorsque toutes les cibles prennent en charge les images. Désactivez cette option pour n’accepter que du texte.", + "cws.capability.imageInputSidecarHint": "Activé par défaut. Lors de l’enregistrement, les cibles {models} seront déclarées en mode texte uniquement et utiliseront le Vision Sidecar pour les images.", + "cws.capability.imageInputSidecarDisabled": "Le Vision Sidecar est désactivé — les images envoyées à {models} sont remplacées par une note d’omission et la requête continue sans elles. Activez-le dans les paramètres du tableau de bord pour rétablir les descriptions.", + "cws.capability.imageInputBlockedHint": "{models} ne peut pas être couvert par le Vision Sidecar : ses modalités d’entrée sont inconnues ou sans texte.", "cws.capability.imageInput": "Images / multimodal", "cws.capability.adaptiveEffort": "Échelle de raisonnement adaptative", "cws.capability.adaptiveEffortHint": "Désactivé : une cible sans réglage de raisonnement masque le sélecteur pour toute la combinaison. Activé : ces cibles restent utilisables et le sélecteur conserve les niveaux communs aux autres cibles.", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index ceca93cc617..75e914eb3b5 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -3306,8 +3306,11 @@ export const ja: Record = { "cws.field.defaultEffort": "デフォルトの推論", "cws.field.defaultEffortNone": "なし(ターゲットのデフォルト)", "cws.field.defaultEffortHint": "クライアントが推論負荷を省略した場合のみ使用されます。選択肢は選択ターゲットが広告する負荷の交差です。", - "cws.capability.imageInputUnavailable": "選択した全ターゲットが画像入力に対応すると有効になります。", + "cws.capability.imageInputUnavailable": "まずすべてのターゲットをカタログから選択してください。不明なモデルはVision Sidecarでカバーできません。", "cws.capability.imageInputHint": "全ターゲットが画像対応なら既定でオン。オフにするとテキストのみ。", + "cws.capability.imageInputSidecarHint": "既定でオン。{models} は保存時にテキスト専用として宣言され、画像にはVision Sidecarを使用します。", + "cws.capability.imageInputSidecarDisabled": "Vision Sidecarがオフのため、{models} へ送信される画像は省略を示す注記に置き換えられ、リクエストはそのまま続行します。画像の説明を復元するにはダッシュボード設定で有効化してください。", + "cws.capability.imageInputBlockedHint": "{models} はVision Sidecarでカバーできません：入力モダリティが不明か、テキストを含みません。", "cws.capability.imageInput": "画像 / マルチモーダル", "cws.capability.adaptiveEffort": "適応的な推論レベル", "cws.capability.adaptiveEffortHint": "オフ: 推論レベルを持たない対象があると、コンボ全体のセレクターが消えます。オン: その対象はそのまま使え、セレクターには残りの対象で共通するレベルが表示されます。", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 9f4026aa7b2..60b094d854d 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -3254,8 +3254,11 @@ export const ko: Record = { "cws.field.defaultEffort": "기본 추론 수준", "cws.field.defaultEffortNone": "없음 (대상 기본값)", "cws.field.defaultEffortHint": "클라이언트가 추론 수준을 생략한 경우에만 사용합니다. 옵션은 선택한 대상이 광고하는 수준의 교집합입니다.", - "cws.capability.imageInputUnavailable": "선택한 모든 대상이 이미지 입력을 지원해야 사용할 수 있습니다.", + "cws.capability.imageInputUnavailable": "먼저 모든 대상을 카탈로그에서 선택하세요. 알 수 없는 모델은 Vision Sidecar로 처리할 수 없습니다.", "cws.capability.imageInputHint": "모든 대상이 이미지를 지원하면 기본으로 켜집니다. 끄면 텍스트만 허용합니다.", + "cws.capability.imageInputSidecarHint": "기본으로 켜짐. {models}은(는) 저장 시 텍스트 전용으로 선언되어 이미지에 Vision Sidecar를 사용합니다.", + "cws.capability.imageInputSidecarDisabled": "Vision Sidecar가 꺼져 있어 {models}에 전송되는 이미지는 생략 안내 텍스트로 대체되고 요청은 계속 진행됩니다. 이미지 설명을 복원하려면 대시보드 설정에서 활성화하세요.", + "cws.capability.imageInputBlockedHint": "{models}은(는) Vision Sidecar로 처리할 수 없습니다: 입력 모달리티를 알 수 없거나 텍스트가 없습니다.", "cws.capability.imageInput": "이미지 / 멀티모달", "cws.capability.adaptiveEffort": "적응형 추론 단계", "cws.capability.adaptiveEffortHint": "끔: 추론 단계를 조절할 수 없는 대상이 하나라도 있으면 콤보 전체의 선택기가 사라집니다. 켬: 그런 대상도 그대로 쓰면서, 선택기에는 나머지 대상이 공통으로 지원하는 단계가 남습니다.", diff --git a/gui/src/i18n/pt.ts b/gui/src/i18n/pt.ts index 8f9bb868fa1..3abefc3f4fe 100644 --- a/gui/src/i18n/pt.ts +++ b/gui/src/i18n/pt.ts @@ -3277,8 +3277,11 @@ export const pt: Record = { "cws.field.defaultEffort": "Raciocínio padrão", "cws.field.defaultEffortNone": "Nenhum (padrão do destino)", "cws.field.defaultEffortHint": "Usado somente quando o cliente omite o esforço de raciocínio. As opções são a interseção dos esforços informados pelos destinos selecionados; destinos sem metadados de esforço no catálogo não oferecem nenhuma.", - "cws.capability.imageInputUnavailable": "Indisponível até que todos os destinos selecionados aceitem entrada de imagem.", + "cws.capability.imageInputUnavailable": "Escolha primeiro todos os destinos no catálogo — modelos desconhecidos não podem ser cobertos pelo Vision Sidecar.", "cws.capability.imageInputHint": "Ativado por padrão quando todos os destinos aceitam imagens. Desative para aceitar somente texto.", + "cws.capability.imageInputSidecarHint": "Ativado por padrão. Ao salvar, {models} serão declarados como somente texto e usarão o Vision Sidecar para imagens.", + "cws.capability.imageInputSidecarDisabled": "O Vision Sidecar está desativado — as imagens enviadas a {models} são substituídas por uma nota de omissão e a requisição continua sem elas. Ative-o nas configurações do painel para restaurar as descrições.", + "cws.capability.imageInputBlockedHint": "{models} não pode ser coberto pelo Vision Sidecar: suas modalidades de entrada são desconhecidas ou não têm texto.", "cws.capability.imageInput": "Imagem / multimodal", "cws.capability.adaptiveEffort": "Escala de raciocínio adaptativa", "cws.capability.adaptiveEffortHint": "Desativado: um destino sem controle de raciocínio oculta o seletor de esforço para o combo inteiro. Ativado: esses destinos continuam utilizáveis e o seletor mantém os níveis compartilhados pelos demais destinos.", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 3d16af32b33..aaa46af5839 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -3396,8 +3396,11 @@ export const ru: Record = { "cws.field.defaultEffort": "Рассуждения по умолчанию", "cws.field.defaultEffortNone": "Нет (по умолчанию для цели)", "cws.field.defaultEffortHint": "Используется, только если клиент не указал уровень рассуждений. Варианты — пересечение заявленных уровней выбранных целей.", - "cws.capability.imageInputUnavailable": "Доступно, когда все выбранные цели поддерживают ввод изображений.", + "cws.capability.imageInputUnavailable": "Сначала выберите все цели из каталога — неизвестные модели нельзя обработать через Vision Sidecar.", "cws.capability.imageInputHint": "Включено по умолчанию, если все цели поддерживают изображения. Выключите, чтобы принимать только текст.", + "cws.capability.imageInputSidecarHint": "Включено по умолчанию. При сохранении для {models} будет указана только текстовая модальность; изображения будут обрабатываться через Vision Sidecar.", + "cws.capability.imageInputSidecarDisabled": "Vision Sidecar выключен — изображения, отправляемые в {models}, заменяются текстовой пометкой, и запрос продолжается без них. Включите его в настройках панели управления, чтобы восстановить описания.", + "cws.capability.imageInputBlockedHint": "{models} не может быть покрыт Vision Sidecar: его входные модальности неизвестны или не содержат текст.", "cws.capability.imageInput": "Изображения / мультимодальность", "cws.capability.adaptiveEffort": "Адаптивная шкала рассуждений", "cws.capability.adaptiveEffortHint": "Выкл.: цель без настройки рассуждений скрывает выбор уровня для всей комбинации. Вкл.: такие цели остаются доступными, а в выборе сохраняются уровни, общие для остальных целей.", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index 3bf06e2157f..8a3e370044e 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -3257,8 +3257,11 @@ export const tr: Record = { "cws.field.defaultEffort": "Varsayılan akıl yürütme", "cws.field.defaultEffortNone": "Yok (hedef varsayılanı)", "cws.field.defaultEffortHint": "Yalnızca istemci akıl yürütme çabasını belirtmediğinde (atladığında) kullanılır. Seçenekler, seçilen hedeflerin duyurulan çabalarının kesişimidir; katalog çaba meta verisi olmayan hedefler hiçbir seçenek sunmaz.", - "cws.capability.imageInputUnavailable": "Seçilen tüm hedefler görsel girişini destekleyene kadar kullanılamaz.", + "cws.capability.imageInputUnavailable": "Önce tüm hedefleri katalogdan seçin — bilinmeyen modeller Vision Sidecar ile kapsanamaz.", "cws.capability.imageInputHint": "Tüm hedefler görselleri desteklediğinde varsayılan olarak açıktır. Yalnızca metin kabul etmek için kapatın.", + "cws.capability.imageInputSidecarHint": "Varsayılan olarak açık. {models} kaydedildiğinde yalnızca metin olarak bildirilir ve görseller için Vision Sidecar kullanır.", + "cws.capability.imageInputSidecarDisabled": "Vision Sidecar kapalı — {models} modeline gönderilen görseller bir atlama notuyla değiştirilir ve istek görseller olmadan devam eder. Görsel açıklamalarını geri getirmek için gösterge paneli ayarlarından etkinleştirin.", + "cws.capability.imageInputBlockedHint": "{models} Vision Sidecar tarafından kapsanamaz: giriş modları bilinmiyor veya metin içermiyor.", "cws.capability.imageInput": "Görsel / çok modlu", "cws.capability.adaptiveEffort": "Uyarlanabilir akıl yürütme düzeyi", "cws.capability.adaptiveEffortHint": "Kapalı: akıl yürütme denetimi olmayan bir hedef, tüm kombinasyonun seçicisini gizler. Açık: bu hedefler kullanılabilir kalır ve seçici, kalan hedeflerin ortak düzeylerini gösterir.", diff --git a/gui/src/i18n/vi.ts b/gui/src/i18n/vi.ts index 92b5561346f..493c2433c39 100644 --- a/gui/src/i18n/vi.ts +++ b/gui/src/i18n/vi.ts @@ -3245,8 +3245,11 @@ export const vi: Record = { "cws.field.defaultEffort": "Suy luận (reasoning) mặc định", "cws.field.defaultEffortNone": "Không (mặc định của mục tiêu)", "cws.field.defaultEffortHint": "Chỉ sử dụng khi client không chỉ định mức độ suy luận (reasoning effort). Các tùy chọn là giao điểm của các mức độ được công bố từ các mục tiêu đã chọn; những mục tiêu không có metadata về mức độ suy luận trong danh mục sẽ không cung cấp tùy chọn này.", - "cws.capability.imageInputUnavailable": "Không khả dụng cho đến khi mọi mục tiêu được chọn đều hỗ trợ đầu vào hình ảnh (image input).", + "cws.capability.imageInputUnavailable": "Hãy chọn tất cả mục tiêu từ danh mục trước — Vision Sidecar không thể hỗ trợ các mô hình không xác định.", "cws.capability.imageInputHint": "Được bật theo mặc định khi mọi mục tiêu đều hỗ trợ hình ảnh. Tắt để chỉ chấp nhận văn bản.", + "cws.capability.imageInputSidecarHint": "Bật theo mặc định. {models} sẽ được khai báo chỉ-text khi lưu và dùng Vision Sidecar cho hình ảnh.", + "cws.capability.imageInputSidecarDisabled": "Vision Sidecar đang tắt — hình ảnh gửi tới {models} sẽ được thay bằng ghi chú bỏ qua và yêu cầu vẫn tiếp tục. Bật nó trong cài đặt bảng điều khiển để khôi phục mô tả hình ảnh.", + "cws.capability.imageInputBlockedHint": "{models} không thể được Vision Sidecar hỗ trợ: modalities đầu vào không rõ hoặc không có văn bản.", "cws.capability.imageInput": "Hình ảnh / đa phương thức (multimodal)", "cws.capability.adaptiveEffort": "Thang suy luận thích ứng (Adaptive reasoning ladder)", "cws.capability.adaptiveEffortHint": "Tắt: một mục tiêu không có kiểm soát suy luận sẽ ẩn đi bộ chọn mức độ (effort picker) cho toàn bộ combo. Bật: những mục tiêu đó vẫn có thể sử dụng và bộ chọn sẽ giữ lại các cấp độ mà các mục tiêu còn lại dùng chung.", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index fff221d842b..b253ea48b0c 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -2438,8 +2438,11 @@ export const zhTW: Record = { "cws.field.defaultEffort": "預設推理級別", "cws.field.defaultEffortNone": "無（使用目標預設）", "cws.field.defaultEffortHint": "僅在客戶端未指定推理級別時使用。客戶端值優先，每個目標會按自身能力進行處理。", - "cws.capability.imageInputUnavailable": "所有已選目標都支援圖片輸入後才可使用。", + "cws.capability.imageInputUnavailable": "請先從目錄選擇所有目標——未知模型無法由 Vision Sidecar 處理。", "cws.capability.imageInputHint": "所有目標都支援圖片時預設開啟；關閉後僅接受文字。", + "cws.capability.imageInputSidecarHint": "預設開啟。{models} 將在儲存時宣告為純文字，並使用 Vision Sidecar 處理圖片。", + "cws.capability.imageInputSidecarDisabled": "Vision Sidecar 已關閉——傳送給 {models} 的圖片將被替換為省略說明，請求會繼續進行。在儀表板設定中啟用它可恢復圖片描述。", + "cws.capability.imageInputBlockedHint": "{models} 無法由 Vision Sidecar 覆蓋：其輸入模態未知或不含文字。", "cws.capability.imageInput": "圖片 / 多模態", "cws.capability.adaptiveEffort": "自適應推理層級", "cws.capability.adaptiveEffortHint": "關閉：只要有一個目標不支援推理層級，整個組合的選擇器都會消失。開啟：這些目標仍可使用，選擇器保留其餘目標共有的層級。", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 81bc809e20e..76768f3fb62 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -3227,8 +3227,11 @@ export const zh: Record = { "cws.field.defaultEffort": "默认推理级别", "cws.field.defaultEffortNone": "无（使用目标默认）", "cws.field.defaultEffortHint": "仅在客户端未指定推理级别时使用。选项为所选目标已公布努力级别的交集。", - "cws.capability.imageInputUnavailable": "所有已选目标均支持图片输入后才可用。", + "cws.capability.imageInputUnavailable": "请先从目录中选择所有目标——未知模型无法由 Vision Sidecar 处理。", "cws.capability.imageInputHint": "所有目标均支持图片时默认开启；关闭后仅接受文本。", + "cws.capability.imageInputSidecarHint": "默认开启。{models} 将在保存时声明为纯文本，并使用 Vision Sidecar 处理图片。", + "cws.capability.imageInputSidecarDisabled": "Vision Sidecar 已关闭——发送给 {models} 的图片将被替换为省略说明，请求会继续进行。在仪表板设置中启用它可恢复图片描述。", + "cws.capability.imageInputBlockedHint": "{models} 无法由 Vision Sidecar 覆盖：其输入模态未知或不含文本。", "cws.capability.imageInput": "图片 / 多模态", "cws.capability.adaptiveEffort": "自适应推理档位", "cws.capability.adaptiveEffortHint": "关闭：只要有一个目标不支持推理档位，整个组合的选择器都会消失。开启：这些目标仍可使用，选择器保留其余目标共有的档位。", diff --git a/gui/src/pages/Combos.tsx b/gui/src/pages/Combos.tsx index c7cd9194727..5b37b8b4c02 100644 --- a/gui/src/pages/Combos.tsx +++ b/gui/src/pages/Combos.tsx @@ -1,5 +1,6 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import ComboWorkspace from "../components/ComboWorkspace"; +import { comboVisionSidecarTargets } from "../combo-capabilities"; import { type ComboItem, comboModelId, @@ -29,7 +30,15 @@ type ProviderOption = { defaultModel?: string; models?: string[]; }; -type ModelOption = { provider: string; id: string; namespaced?: string; reasoningEfforts?: string[]; inputModalities?: string[] }; +type ModelOption = { + provider: string; + id: string; + namespaced?: string; + reasoningEfforts?: string[]; + inputModalities?: string[]; + inputModalitiesDeclared?: string[]; + visionSidecarConsumer?: boolean; +}; type ProviderDto = { adapter: string; baseUrl: string; @@ -172,6 +181,13 @@ export default function Combos({ const models: ModelOption[] = []; const capabilityModels: ModelOption[] = []; const catalogued = new Set(); + const parseModalities = (raw: unknown): string[] | undefined => + Array.isArray(raw) + ? raw.flatMap((modality) => { + const trimmed = typeof modality === "string" ? modality.trim() : ""; + return trimmed ? [trimmed] : []; + }) + : undefined; for (const row of modelRows) { if (!row || typeof row !== "object") continue; const model = row as { @@ -181,6 +197,8 @@ export default function Combos({ disabled?: unknown; reasoningEfforts?: unknown; inputModalities?: unknown; + inputModalitiesDeclared?: unknown; + visionSidecarConsumer?: unknown; }; if (typeof model.provider !== "string" || typeof model.id !== "string") continue; const provider = model.provider.trim(); @@ -193,18 +211,16 @@ export default function Combos({ const reasoningEfforts = Array.isArray(model.reasoningEfforts) ? model.reasoningEfforts.filter((effort): effort is string => typeof effort === "string") : undefined; - const inputModalities = Array.isArray(model.inputModalities) - ? model.inputModalities.flatMap((modality) => { - const trimmed = typeof modality === "string" ? modality.trim() : ""; - return trimmed ? [trimmed] : []; - }) - : undefined; + const inputModalities = parseModalities(model.inputModalities); + const inputModalitiesDeclared = parseModalities(model.inputModalitiesDeclared); const option: ModelOption = { provider, id, namespaced: typeof model.namespaced === "string" ? model.namespaced : undefined, ...(reasoningEfforts ? { reasoningEfforts } : {}), ...(inputModalities && inputModalities.length > 0 ? { inputModalities } : {}), + ...(inputModalitiesDeclared && inputModalitiesDeclared.length > 0 ? { inputModalitiesDeclared } : {}), + visionSidecarConsumer: model.visionSidecarConsumer === true, }; capabilityModels.push(option); if (model.disabled !== true) models.push(option); @@ -247,6 +263,28 @@ export default function Combos({ ); const { state } = resource; + /* + * Vision Sidecar enabled state for the enrollment warning. Deliberately + * separate from the workspace payload: a failure here must never block or + * fail the combo workspace, so errors are swallowed and `undefined` + * (unknown) renders no warning. + */ + const [visionEnabled, setVisionEnabled] = useState(undefined); + useEffect(() => { + if (!active) return; + let cancelled = false; + fetch(`${apiBase}/api/sidecar-settings`) + .then((res) => (res.ok ? res.json() : null)) + .then((data: unknown) => { + if (cancelled || !data || typeof data !== "object") return; + const vision = (data as { vision?: { enabled?: unknown } }).vision; + if (!vision || typeof vision !== "object") return; + setVisionEnabled(vision.enabled !== false); + }) + .catch(() => { /* unknown stays warning-free */ }); + return () => { cancelled = true; }; + }, [active, apiBase]); + const [quotaNow, setQuotaClock] = useState(() => Date.now()); const loadProviderQuotas = useCallback(async (signal?: AbortSignal): Promise => { const response = await fetch(`${apiBase}/api/provider-quotas`, { signal }); @@ -307,7 +345,14 @@ export default function Combos({ const res = await fetch(`${apiBase}/api/combos`, { method: "PUT", headers: { "content-type": "application/json" }, - body: JSON.stringify(toPutBody(item, renameFrom ? { renameFrom } : {})), + body: JSON.stringify(toPutBody(item, { + ...(renameFrom ? { renameFrom } : {}), + // Enabling images declares text-only members for the Vision Sidecar so the + // operator never has to hand-edit provider config for a multimodal combo. + ...(item.imageInput !== "disabled" + ? { visionSidecarTargets: comboVisionSidecarTargets(item.targets, capabilityModels) } + : {}), + })), }); const data = res.ok ? await res.json() as unknown @@ -403,6 +448,7 @@ export default function Combos({ models={models} capabilityModels={capabilityModels} cataloguedComboIds={cataloguedComboIds} + visionEnabled={visionEnabled} loading={false} onRefresh={() => { resource.refresh(); quotaResource.refresh(); }} onSave={saveCombo} diff --git a/gui/src/styles-combos-workspace.css b/gui/src/styles-combos-workspace.css index 1423733101d..f7ab49d7cdf 100644 --- a/gui/src/styles-combos-workspace.css +++ b/gui/src/styles-combos-workspace.css @@ -474,6 +474,11 @@ font-size: 12px; } +/* Requests still continue without the images, so this is a warning (amber), not a failure. */ +.cwi-capability-hint--warn { + color: var(--amber); +} + .cwi-target-list { display: flex; diff --git a/gui/tests/combo-capability-hidden-targets.test.tsx b/gui/tests/combo-capability-hidden-targets.test.tsx index e4d7c75bec2..56bf68002c4 100644 --- a/gui/tests/combo-capability-hidden-targets.test.tsx +++ b/gui/tests/combo-capability-hidden-targets.test.tsx @@ -32,6 +32,9 @@ const hiddenImage: CatalogRow = { const textModel: CatalogRow = { provider: "text", id: "plain", inputModalities: ["text"], reasoningEfforts: ["high", "xhigh"], }; +const audioModel: CatalogRow = { + provider: "audio", id: "listen", inputModalities: ["audio"], reasoningEfforts: ["high"], +}; const target = (row: CatalogRow): Target => ({ provider: row.provider, model: row.id }); const combo = (targets: Target[], extra: Record = {}) => ({ id: "hidden-test", model: "combo/hidden-test", strategy: "failover", stickyLimit: 1, @@ -136,15 +139,29 @@ test("a disabled image-capable target enables the Image switch and stays visible expect(effortValues()).toEqual(["", "low", "high"]); }); +// Text-capable members are covered by the Vision Sidecar, so they keep images on. for (const [name, targets] of [ ["text-only", [target(textModel)]], - ["unknown", [{ provider: "vision", model: "missing" }]], ["mixed image/text", [target(hiddenImage), target(textModel)]], ] as const) { - test(`${name} targets keep the Image switch off`, async () => { + test(`${name} targets keep the Image switch on through the Vision Sidecar`, async () => { mockCatalog([hiddenImage, textModel], [...targets]); await mount(); await openDetail(); + expectImages(true); + }); +} + +// Members the sidecar cannot cover still force the switch off. +for (const [name, targets] of [ + ["unknown", [{ provider: "vision", model: "missing" }]], + ["audio-only", [target(audioModel)]], + ["mixed text/audio-only", [target(textModel), target(audioModel)]], +] as const) { + test(`${name} targets keep the Image switch off`, async () => { + mockCatalog([hiddenImage, textModel, audioModel], [...targets]); + await mount(); + await openDetail(); expectImages(false); }); } @@ -157,7 +174,9 @@ for (const provider of ["vision", "text"]) { ], [{ provider, model: "shared" }]); await mount(); await openDetail(); - expectImages(provider === "vision"); + // The text row is sidecar-eligible, so both resolve to an enabled Image switch; + // the provider still decides which reasoning ladder applies. + expectImages(true); expect(effortValues()).toEqual(provider === "vision" ? ["", "low", "high"] : ["", "high", "xhigh"]); }); } @@ -221,6 +240,7 @@ test("cache remount and live revalidation preserve hidden-target capabilities", const cached = readSessionListCacheEntry<{ capabilityModels: CatalogRow[]; models: CatalogRow[] }>(CACHE_KEY)!; expect(cached.data.capabilityModels).toEqual([{ provider: "vision", id: "hidden", inputModalities: ["text", "image"], reasoningEfforts: ["low", "high"], + visionSidecarConsumer: false, }]); expect(cached.data.models).toEqual([]); await act(async () => { root!.unmount(); }); diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index ffd84bbd044..470fba1b3a6 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -702,6 +702,7 @@ "combo-child-headers.test.ts": "routing", "combo-last-resort.test.ts": "codex-integration", "combo-management-api.test.ts": "routing", + "combo-vision-enrollment.test.ts": "routing", "combo-stream-preflight.test.ts": "routing", "combo-workspace-data.test.ts": "gui", "combo-workspace-jev-decision.test.ts": "gui", diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index ff2a89fe06a..6f4979e0073 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -52,6 +52,15 @@ import { setDebugSettings, type DebugFlag, } from "../../lib/debug-settings"; +import { mergeModelCapabilities, modelCapabilitiesConfigError } from "../../config/provider-validation"; +import { modelRecordValue } from "../../reasoning-effort"; +import { commitProviderPatch } from "./provider-patch-transaction"; +import { ConfigWritePublishedError } from "../../config/persist-unlocked"; +import { + customRowInputModalities, + isVisionSidecarConsumer, + modelAcceptsImageInput, +} from "../../vision/eligibility"; import type { OcxClaudeCodeConfig, OcxComboConfig, OcxConfig, OcxCustomModel, OcxProviderConfig, OcxComboCooldownWaitPolicy } from "../../types"; import { drainAndShutdown } from "../lifecycle"; import { reconcileLiveStateStores } from "../../lib/state-store-registrations"; @@ -116,6 +125,43 @@ function sparseComboConfig, +): { targets?: VisionSidecarTarget[]; error?: string } { + if (raw === undefined) return {}; + if (!Array.isArray(raw)) return { error: "visionSidecarTargets must be an array" }; + const seen = new Set(); + const targets: VisionSidecarTarget[] = []; + for (const entry of raw) { + if (!isPlainRecord(entry)) return { error: "visionSidecarTargets entries must be objects" }; + const provider = typeof entry.provider === "string" ? entry.provider.trim() : ""; + const model = typeof entry.model === "string" ? entry.model.trim() : ""; + if (!provider || !model) { + return { error: "visionSidecarTargets entries must have nonblank provider and model" }; + } + const key = `${provider}/${model}`; + if (!comboTargetKeys.has(key)) { + return { error: `visionSidecarTargets entry "${key}" is not a target of this combo` }; + } + if (seen.has(key)) continue; + seen.add(key); + targets.push({ provider, model }); + } + return { targets }; +} + export async function handleComboRoutes(ctx: ManagementContext): Promise { const { req, url, config, deps, convergeCodexCatalog, syncClaudeAgentDefsBestEffort } = ctx; @@ -289,6 +335,54 @@ export async function handleComboRoutes(ctx: ManagementContext): Promise `${target.provider}/${target.model}`)), + ); + if (sidecarParsed.error) return jsonResponse({ error: sidecarParsed.error }, 400); + const sidecarPatches = new Map>(); + if (sidecarParsed.targets?.length) { + if (normalized.imageInput === "disabled") { + return jsonResponse({ error: "visionSidecarTargets requires imageInput not disabled" }, 400); + } + for (const { provider, model } of sidecarParsed.targets) { + const providerRow = config.providers?.[provider]; + if (!providerRow) { + return jsonResponse({ error: `visionSidecarTargets entry "${provider}/${model}" has no configured provider` }, 400); + } + // A text-only declaration would HIDE a real capability; never overwrite one. + if (modelAcceptsImageInput(config, { provider, id: model }) === true) { + return jsonResponse({ error: `visionSidecarTargets entry "${provider}/${model}" already accepts image input` }, 400); + } + // Already a consumer through any runtime source (exact declaration, custom row, + // noVisionModels, legacy record, registry enrichment): the catalog already + // advertises image for it. Writing the declaration again would be a no-op write. + if (isVisionSidecarConsumer(config, provider, model)) continue; + // Read the declaration the way the runtime does: exact capability axis, then the + // operator's custom row, then the provider's per-model modality hint. A row without + // text (audio-only) cannot be widened by the sidecar, and writing ["text"] on the + // capability axis would silently override that declaration. + const capabilityRow = Object.hasOwn(providerRow.modelCapabilities ?? {}, model) + ? providerRow.modelCapabilities?.[model] : undefined; + const declared = capabilityRow?.inputModalities + ?? customRowInputModalities(config, provider, model) + ?? modelRecordValue(providerRow.modelInputModalities, model); + if (declared !== undefined && !declared.includes("text")) { + return jsonResponse({ error: `visionSidecarTargets entry "${provider}/${model}" is declared without text input; the Vision Sidecar cannot cover it` }, 400); + } + const patch = sidecarPatches.get(provider) ?? {}; + patch[model] = { inputModalities: ["text"] }; + sidecarPatches.set(provider, patch); + } + for (const [providerName, patch] of sidecarPatches) { + // Validate the map that will be persisted, not just the patch: a provider already at + // the modelCapabilities cap would otherwise save an oversized map that load-time + // sanitization truncates, silently dropping the new text-only declaration. + const merged = mergeModelCapabilities(config.providers?.[providerName]?.modelCapabilities, patch); + const error = modelCapabilitiesConfigError(patch) ?? (merged ? modelCapabilitiesConfigError(merged) : null); + if (error) return jsonResponse({ error: `visionSidecarTargets for provider "${providerName}": ${error}` }, 400); + } + } // Persist only non-default identity/capability fields so config stays sparse. // Capability defaults (`imageInput`, `reasoningEffortMode`) go through the same // helper the GET/PUT responses use, so the wire shape and the stored shape cannot drift. @@ -350,58 +444,80 @@ export async function handleComboRoutes(ctx: ManagementContext): Promise 0) { - const migrateReference = (model: string): string => migratedModels.get(model) ?? model; - const migrateAgentReference = (model: string): string => { - const migrated = migrateReference(model); - if (migrated !== model) shouldSyncClaudeAgentDefs = true; - return migrated; - }; - if (config.subagentModels) { - config.subagentModels = [...new Set(config.subagentModels.map(migrateAgentReference))]; - } - if (config.injectionModel && migratedModels.has(config.injectionModel)) { - config.injectionModel = migrateReference(config.injectionModel); - } - if (config.shadowCallIntercept?.model && migratedModels.has(config.shadowCallIntercept.model)) { - config.shadowCallIntercept = { - ...config.shadowCallIntercept, - model: migrateReference(config.shadowCallIntercept.model), - }; - } - if (config.claudeCode) { - const claudeCode = { ...config.claudeCode }; - for (const field of ["model", "smallFastModel"] as const) { - if (claudeCode[field]) claudeCode[field] = migrateAgentReference(claudeCode[field]); - } - if (claudeCode.tierModels) { - claudeCode.tierModels = Object.fromEntries( - Object.entries(claudeCode.tierModels).map(([tier, model]) => [tier, migrateAgentReference(model)]), - ); + // One atomic transaction: a failed save must restore the combo, the capability + // declarations, and every migrated reference together, never partially. + try { + commitProviderPatch(config, () => { + config.combos = nextCombos; + if (migratedModels.size > 0) { + const migrateReference = (model: string): string => migratedModels.get(model) ?? model; + const migrateAgentReference = (model: string): string => { + const migrated = migrateReference(model); + if (migrated !== model) shouldSyncClaudeAgentDefs = true; + return migrated; + }; + if (config.subagentModels) { + config.subagentModels = [...new Set(config.subagentModels.map(migrateAgentReference))]; + } + if (config.injectionModel && migratedModels.has(config.injectionModel)) { + config.injectionModel = migrateReference(config.injectionModel); + } + if (config.shadowCallIntercept?.model && migratedModels.has(config.shadowCallIntercept.model)) { + config.shadowCallIntercept = { + ...config.shadowCallIntercept, + model: migrateReference(config.shadowCallIntercept.model), + }; + } + if (config.claudeCode) { + const claudeCode = { ...config.claudeCode }; + for (const field of ["model", "smallFastModel"] as const) { + if (claudeCode[field]) claudeCode[field] = migrateAgentReference(claudeCode[field]); + } + if (claudeCode.tierModels) { + claudeCode.tierModels = Object.fromEntries( + Object.entries(claudeCode.tierModels).map(([tier, model]) => [tier, migrateAgentReference(model)]), + ); + } + if (claudeCode.modelMap) { + claudeCode.modelMap = Object.fromEntries( + Object.entries(claudeCode.modelMap).map(([source, model]) => [source, migrateAgentReference(model)]), + ); + } + if (claudeCode.intercept?.modelMap) { + claudeCode.intercept = { + ...claudeCode.intercept, + modelMap: Object.fromEntries( + Object.entries(claudeCode.intercept.modelMap).map(([pickerId, route]) => [pickerId, migrateAgentReference(route)]), + ), + }; + } + config.claudeCode = claudeCode; + } } - if (claudeCode.modelMap) { - claudeCode.modelMap = Object.fromEntries( - Object.entries(claudeCode.modelMap).map(([source, model]) => [source, migrateAgentReference(model)]), - ); + if (oldDisabledSelectors.size > 0 && config.disabledModels) { + config.disabledModels = [...new Set(config.disabledModels.map(model => ( + oldDisabledSelectors.has(model) ? newDisabledModel : model + )))]; } - if (claudeCode.intercept?.modelMap) { - claudeCode.intercept = { - ...claudeCode.intercept, - modelMap: Object.fromEntries( - Object.entries(claudeCode.intercept.modelMap).map(([pickerId, route]) => [pickerId, migrateAgentReference(route)]), - ), - }; + for (const [providerName, patch] of sidecarPatches) { + const provider = config.providers?.[providerName]; + if (!provider) continue; + const capabilities = mergeModelCapabilities(provider.modelCapabilities, patch); + // Re-check under the config lock: a concurrent edit may have grown the map since + // the pre-check, and an oversized map must never be published. + if (capabilities && modelCapabilitiesConfigError(capabilities)) { + throw new Error("modelCapabilities would exceed its limit"); + } + if (capabilities === undefined) delete provider.modelCapabilities; + else provider.modelCapabilities = capabilities; } - config.claudeCode = claudeCode; + }, deps.saveConfigPreservingClaudeCode ?? saveConfigPreservingClaudeCode); + } catch (error) { + if (!(error instanceof ConfigWritePublishedError)) { + return jsonResponse({ error: "combo could not be saved" }, 500); } + throw error; } - if (oldDisabledSelectors.size > 0 && config.disabledModels) { - config.disabledModels = [...new Set(config.disabledModels.map(model => ( - oldDisabledSelectors.has(model) ? newDisabledModel : model - )))]; - } - saveConfigPreservingClaudeCode(config); reconcileLiveStateStores(); clearComboSelectionState(id); clearComboTargetCooldowns(id); diff --git a/src/server/management/model-rows.ts b/src/server/management/model-rows.ts index b764009e364..479c0c880cc 100644 --- a/src/server/management/model-rows.ts +++ b/src/server/management/model-rows.ts @@ -52,6 +52,7 @@ import { reconcileSuccessfulModelDiscoveries } from "../../providers/new-model-p import { initialModelSelectionPending, pendingModelSelectionProviders } from "../../providers/initial-model-selection"; import { catalogFastRowEligible, fastRowId } from "../fast-row"; import { knownEffortRowIds } from "../effort-row"; +import { visionSidecarConsumerClassifier } from "../../vision/eligibility"; /** * One row of the `/api/models` list. Routed rows spread a `CatalogModel`, so the shape is @@ -78,6 +79,15 @@ export type ManagementModelRow = Partial & { * one thing and saves another, so its own writes never appear to take. */ inputModalitiesDeclared?: string[]; + /** + * The runtime's verdict on whether the Vision Sidecar describes this model's images, from + * the same predicate the request path consults (legacy `noVisionModels`, registry + * enrichment, explicit declarations, custom rows). `inputModalitiesDeclared` cannot answer + * this: a model covered ONLY by legacy `noVisionModels` has no declaration while its + * advertised modalities are widened to image, so a client inferring consumer status from + * declared-vs-advertised modalities misclassifies the row after reload. + */ + visionSidecarConsumer?: boolean; /** Exact stored context override, distinct from the effective catalog window. */ contextWindowDeclared?: number; /** @@ -496,6 +506,7 @@ export async function listManagementModelRows( // a hidden real `x--fast` must never become a synthetic selector for another model. const knownIds = config.fastRows === false ? new Set() : knownEffortRowIds(config); for (const row of rows) knownIds.add(row.namespaced); + const isVisionSidecarConsumer = visionSidecarConsumerClassifier(config); return rows.map(row => { const pending = initialModelSelectionPending(config.providers[row.provider]); const modelCosts = Object.hasOwn(config.providers, row.provider) @@ -519,6 +530,9 @@ export async function listManagementModelRows( }); return { ...row, + // One shared definition with the request path: the GUI must not re-derive consumer + // status from declared-vs-advertised modalities (see the field doc above). + visionSidecarConsumer: isVisionSidecarConsumer(row.provider, row.id), ...(!row.native && modelCosts !== undefined && Object.hasOwn(modelCosts, row.id) ? { manualPricing: true } : {}), ...(pending ? { disabled: true, initialSelectionPending: true } : {}), diff --git a/src/vision/eligibility.ts b/src/vision/eligibility.ts index d772c8f9769..cdeefcc5b33 100644 --- a/src/vision/eligibility.ts +++ b/src/vision/eligibility.ts @@ -226,6 +226,19 @@ export function isVisionSidecarConsumer(config: VisionCapabilityConfig, provider return isVisionSidecarConsumerWithCache(config, providerName, modelId, new Map()); } +/** + * `isVisionSidecarConsumer` for many rows of one config. Rows of the same provider share one + * registry enrichment instead of rebuilding it per row, which matters for `/api/models` + * listings with thousands of rows. Bind it to a single config snapshot; it does not observe + * provider changes made after the first lookup. + */ +export function visionSidecarConsumerClassifier( + config: VisionCapabilityConfig, +): (providerName: string, modelId: string) => boolean { + const cache: EnrichedProviderCache = new Map(); + return (providerName, modelId) => isVisionSidecarConsumerWithCache(config, providerName, modelId, cache); +} + /** * Can this model accept an image on the wire? Sources are consulted in descending * trustworthiness; the first that speaks wins. `undefined` means nothing knows, diff --git a/structure/catalog.md b/structure/catalog.md index 65a9ef46769..599fe327461 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -573,7 +573,7 @@ Live sideband admission and its bounded upstream handshake follow the [runtime c The [explicit model-capability contract](config.md#explicit-per-model-capability-declarations) preserves operator declarations through provider storage and catalog capture; it does not infer upstream capability or change this surface's routing behavior. -Exact [model input declarations](config.md#explicit-per-model-capability-declarations) now feed text-only eligibility and catalog hints; existing image-description/omission handling consumes them before the main upstream send. +Exact [model input declarations](config.md#explicit-per-model-capability-declarations) now feed text-only eligibility and catalog hints; existing image-description/omission handling consumes them before the main upstream send. Combo saves can create those declarations through the `PUT /api/combos` `visionSidecarTargets` request field, which commits the combo and its declarations in one transaction, skips members the sidecar already covers, and converges the catalog in the same save. ## Renamed destination reasoning metadata diff --git a/structure/config.md b/structure/config.md index 2ddd9cc3719..73568bafc5f 100644 --- a/structure/config.md +++ b/structure/config.md @@ -563,7 +563,7 @@ Display-name validation retains prototype-shaped model IDs as data; reviewer-tar `modelCapabilities` on `src/types/provider.ts` stores exact model-ID entries with optional inputModalities, contextTier and video.processing axes. `src/config/provider-validation.ts` strictly validates writes and merges PATCH axes without sharing live objects; null map/model/axis/processing tombstones delete, while empty PATCH objects do nothing. Complete POST/PUT replacements reject tombstones. File reads retain valid axes; malformed explicit modalities restrict to text with a diagnostic. The two catalog writers receive explicit config and gather fingerprints include the map. This storage contract alone does not activate a context tier, advertise a larger window or enable video processing. Separately, `modelContextTiers` actively selects `default` or `long_context` for GitHub Copilot models: config and management writes validate exact IDs, PATCH merges entries and null clears the map, CLI edit writes the map, and OAuth login preserves it. -The text-only consumer reads exact inputModalities declarations before legacy hints. CLI add/edit `--text-only` targets one model and preserves sibling declarations; `src/vision/eligibility.ts` routes declared text-only models into existing image-description or explicit-omission handling. Positive routed image declarations override stale candidate metadata, while native catalog authority retains its existing legacy policy. +The text-only consumer reads exact inputModalities declarations before legacy hints. CLI add/edit `--text-only` targets one model and preserves sibling declarations; `src/vision/eligibility.ts` routes declared text-only models into existing image-description or explicit-omission handling. Positive routed image declarations override stale candidate metadata, while native catalog authority retains its existing legacy policy. `PUT /api/combos` accepts a request-only `visionSidecarTargets` array of exact combo-member `{ provider, model }` pairs (validated against the submitted combo's normalized targets, deduplicated, rejected while `imageInput` is `"disabled"`). An entry whose provider is unconfigured, whose model already accepts image input through any source, or whose exact declaration, custom row, or provider `modelInputModalities` hint lacks `text` is rejected with 400 instead of being overwritten; an entry the sidecar already covers through any runtime source (exact declaration, custom row, `noVisionModels`, legacy record, registry enrichment) is skipped as a no-op. Accepted entries merge `inputModalities: ["text"]` through the same validation/merge helpers, and the combo write, reference migrations, disabled-selector rewrite and capability patches commit in one provider-patch transaction — a failed save restores all of them. The field never persists under `config.combos`, and removing a member from the combo leaves its provider declaration in place until cleared explicitly. An explicit custom row is the operator's own definition of one routed model, so its `customModels[].inputModalities` outranks the provider-level vision hints diff --git a/structure/dashboard-and-usage.md b/structure/dashboard-and-usage.md index b49f40a4c78..8254b5a1de5 100644 --- a/structure/dashboard-and-usage.md +++ b/structure/dashboard-and-usage.md @@ -504,7 +504,7 @@ Dashboard overview polling observes authorization failures independently of stal The [explicit model-capability contract](config.md#explicit-per-model-capability-declarations) preserves operator declarations through provider storage and catalog capture; it does not infer upstream capability or change this surface's routing behavior. -Exact [model input declarations](config.md#explicit-per-model-capability-declarations) now feed text-only eligibility and catalog hints; existing image-description/omission handling consumes them before the main upstream send. +Exact [model input declarations](config.md#explicit-per-model-capability-declarations) now feed text-only eligibility and catalog hints; existing image-description/omission handling consumes them before the main upstream send. The Combos page sends non-image members as the request-only `visionSidecarTargets` field on `PUT /api/combos` whenever the Image switch is enabled, so text-only enrollment needs no manual provider edit. The GUI classifies members from the authoritative `visionSidecarConsumer` flag served on `/api/models` rows, so a member covered only by legacy `noVisionModels` keeps its sidecar classification after reload. Members whose modalities are unknown or have no text input keep the switch unavailable and are named in the hint; a member later removed from the combo keeps its provider declaration until cleared in the provider editor. The raw provider editor round-trips `autoReviewModel` and `autoReviewModelOverrides` through editor-owned DTO fields. POST/PATCH/PUT share validation; PUT copies schema-normalized values into the persisted and live candidate before adoption. Canonical `openai` rejects these fields, including clear forms. Field-masked writes (PATCH, editor PUT, reload) pin every registry-seed key and ignore operator overlays the seed never defines, most commonly `selectedModels`; POST keeps the exact-key comparison. Canonical `openai` still rejects `allowPrivateNetwork`, which must not short-circuit destination DNS checks on the ChatGPT forward row. Existing authentication, origin checks and stale-baseline protection still govern the writes. See [reviewer projection](catalog.md#provider-scoped-approval-reviewer). Stored Direct substitution follows the [credential identity contract](providers/openai-accounts.md#sidecars-management-and-ui): both synchronous and asynchronous materializers discard the caller account header before applying the stored credential; ordinary native Direct passthrough is unchanged. diff --git a/structure/gui-and-management-api.md b/structure/gui-and-management-api.md index 782cf24163b..6d597deb963 100644 --- a/structure/gui-and-management-api.md +++ b/structure/gui-and-management-api.md @@ -572,6 +572,8 @@ Pairing-grant source limiting applies only to invalid guesses from an allowed br ## Durable provider PATCH +`PUT /api/combos` shares this transaction for combo changes, rename references, disabled selectors and request-only Vision Sidecar enrollment; see `tests/routing/combo-vision-enrollment.test.ts`. + `src/server/management/provider-routes.ts` commits every provider PATCH variant through `src/server/management/provider-patch-transaction.ts`, including standalone default-provider and account-mode changes. The synchronous mutation lock encloses snapshot, mutation, save, @@ -590,9 +592,7 @@ Reconciliation, model-cache invalidation, quota/thread-cache changes, priming an convergence remain after successful persistence in their existing order. Pacing-only updates keep their existing no-catalog-refresh behavior. The regression suite is `tests/server/management-provider-atomicity.test.ts`. - > Decision record: [Durable provider PATCH](decisions/ADR-0104-durable-provider-patch.md) - > Decision record: [Publication-aware rollback](decisions/ADR-0120-provider-patch-publication-boundary.md) Automatic account exhaustion and recovery use the [spendable Codex credit evidence contract](providers/openai-tiers.md#spendable-codex-credits), including independent freshness, upstream refusal, and reset-ticket separation. The PINNED badge confirms before sending a null selection, which clears the pin and returns routing to automatic while bound threads keep their captured account. The dashboard also sends `expectedPinnedAccountId`; the persisted mutation rejects a changed pin with `account_pin_changed` before writing. Refused or uncertain releases reload `/active` and close the old confirmation without claiming that the prior selection survived. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 5e817d539f2..44d37a1e5b3 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -519,6 +519,7 @@ "combo-permanent-failure-cooldown.test.ts": "codex-integration", "combo-reset-window-model-scope.test.ts": "codex-integration", "combo-child-headers.test.ts": "routing", "combo-last-resort.test.ts": "codex-integration", "combo-management-api.test.ts": "routing", + "combo-vision-enrollment.test.ts": "routing", "combo-stream-preflight.test.ts": "routing", "combo-workspace-data.test.ts": "gui", "combo-workspace-jev-decision.test.ts": "gui", "combos.test.ts": "codex-integration", "command-code-error-finish.test.ts": "providers", "command-code-efforts.test.ts": "providers", diff --git a/tests/gui/combo-workspace-data.test.ts b/tests/gui/combo-workspace-data.test.ts index a0acfc2def9..9d1adfd2b89 100644 --- a/tests/gui/combo-workspace-data.test.ts +++ b/tests/gui/combo-workspace-data.test.ts @@ -19,7 +19,7 @@ import { updateComboAliasDraft, validateComboDraft, } from "../../gui/src/combo-workspace-data"; -import { comboImagesSupported } from "../../gui/src/combo-capabilities"; +import { comboImageBlockedTargets, comboImagesSupported, comboVisionSidecarTargets } from "../../gui/src/combo-capabilities"; const configuredProviders = { a: {}, @@ -849,7 +849,7 @@ describe("comboImagesSupported", () => { )).toBe(false); }); - test("returns true only when every complete target advertises image", () => { + test("returns true when every known target advertises image", () => { const models = [ { provider: "a", id: "m1", inputModalities: ["text", "image"] }, { provider: "b", id: "m2", inputModalities: ["text", "image"] }, @@ -860,20 +860,108 @@ describe("comboImagesSupported", () => { )).toBe(true); }); - test("returns false when any target is missing from the catalog or lacks image", () => { + test("allows text-only members for sidecar coverage but fails closed on unknown, silent, and text-free rows", () => { const models = [ { provider: "a", id: "m1", inputModalities: ["text", "image"] }, { provider: "b", id: "m2", inputModalities: ["text"] }, ]; + // Known text-only rows can be declared text-only on save; only catalog-absent rows block. expect(comboImagesSupported( [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }], models, + )).toBe(true); + // No known modalities: the runtime cannot prove text input, so enrollment is refused. + expect(comboImagesSupported( + [{ provider: "a", model: "m1" }, { provider: "c", model: "m3" }], + [...models, { provider: "c", id: "m3" }], + )).toBe(false); + // Audio-only rows have no text input; the sidecar cannot cover them. + expect(comboImagesSupported( + [{ provider: "a", model: "m1" }, { provider: "c", model: "m3" }], + [...models, { provider: "c", id: "m3", inputModalities: ["audio"] }], )).toBe(false); expect(comboImagesSupported( [{ provider: "a", model: "m1" }, { provider: "b", model: "ghost" }], models, )).toBe(false); }); + + test("classifies from the declared modalities so enrollment survives reload", () => { + // After enrollment the catalog WIDENS the row to image, but the declaration + // still says text-only: reload must keep it a sidecar member, not native vision. + const reloaded = [ + { provider: "a", id: "m1", inputModalities: ["text", "image"], inputModalitiesDeclared: ["text"] }, + ]; + expect(comboImagesSupported([{ provider: "a", model: "m1" }], reloaded)).toBe(true); + expect(comboVisionSidecarTargets([{ provider: "a", model: "m1" }], reloaded)) + .toEqual([{ provider: "a", model: "m1" }]); + }); + + test("visionSidecarConsumer beats image-advertised modalities when no declaration exists", () => { + // Legacy `noVisionModels` rows carry no declaration while the catalog widens + // them to image; the runtime flag alone keeps them sidecar members. + const reloaded = [ + { provider: "a", id: "m1", inputModalities: ["text", "image"], visionSidecarConsumer: true }, + ]; + expect(comboImagesSupported([{ provider: "a", model: "m1" }], reloaded)).toBe(true); + expect(comboVisionSidecarTargets([{ provider: "a", model: "m1" }], reloaded)) + .toEqual([{ provider: "a", model: "m1" }]); + }); +}); + +describe("comboVisionSidecarTargets", () => { + test("returns only known text-only members, deduplicated", () => { + const models = [ + { provider: "a", id: "m1", inputModalities: ["text", "image"] }, + { provider: "b", id: "m2", inputModalities: ["text"] }, + { provider: "c", id: "m3" }, + ]; + expect(comboVisionSidecarTargets( + [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + { provider: "b", model: "m2" }, + ], + models, + )).toEqual([ + { provider: "b", model: "m2" }, + ]); + // A blocked member forces the image switch off, so nothing may be enrolled on save. + expect(comboVisionSidecarTargets( + [ + { provider: "b", model: "m2" }, + { provider: "c", model: "m3" }, + ], + models, + )).toEqual([]); + }); + + test("ignores incomplete or catalog-absent targets", () => { + expect(comboVisionSidecarTargets( + [{ provider: "", model: "" }, { provider: "b", model: "ghost" }], + [{ provider: "b", id: "m2", inputModalities: ["text"] }], + )).toEqual([]); + }); + + test("comboImageBlockedTargets names the members that refuse coverage", () => { + const models = [ + { provider: "a", id: "m1", inputModalities: ["text", "image"] }, + { provider: "b", id: "m2" }, + { provider: "c", id: "m3", inputModalities: ["audio"] }, + ]; + expect(comboImageBlockedTargets( + [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + { provider: "c", model: "m3" }, + { provider: "b", model: "ghost" }, + ], + models, + )).toEqual([ + { provider: "b", model: "m2" }, + { provider: "c", model: "m3" }, + ]); + }); }); describe("combo imageInput draft persistence", () => { @@ -903,4 +991,15 @@ describe("combo imageInput draft persistence", () => { const disabled = { ...auto, imageInput: "disabled" as const }; expect(toPutBody(disabled).combo.imageInput).toBe("disabled"); }); + + test("toPutBody carries visionSidecarTargets as a top-level request-only field", () => { + const auto = emptyDraft("x"); + auto.targets = [{ provider: "a", model: "m1" }]; + const sidecar = [{ provider: "a", model: "m1" }]; + const body = toPutBody(auto, { visionSidecarTargets: sidecar }); + expect(body.visionSidecarTargets).toEqual(sidecar); + expect(body.combo).not.toHaveProperty("visionSidecarTargets"); + // Empty lists stay off the wire. + expect(toPutBody(auto, { visionSidecarTargets: [] })).not.toHaveProperty("visionSidecarTargets"); + }); }); diff --git a/tests/providers/minimax-combo-image-input.test.ts b/tests/providers/minimax-combo-image-input.test.ts index 9949fbd37c9..4ae06470c62 100644 --- a/tests/providers/minimax-combo-image-input.test.ts +++ b/tests/providers/minimax-combo-image-input.test.ts @@ -77,6 +77,10 @@ describe("MiniMax Coding Plan Combo image input", () => { expect(comboImagesSupported( [{ provider: region, model: M3 }, { provider: "other", model: "blind" }], [...rows, { provider: "other", id: "blind", inputModalities: ["text"] }], + )).toBe(true); // a text-only member is covered by the Vision Sidecar + expect(comboImagesSupported( + [{ provider: region, model: M3 }, { provider: "other", model: "listener" }], + [...rows, { provider: "other", id: "listener", inputModalities: ["audio"] }], )).toBe(false); } } diff --git a/tests/routing/combo-vision-enrollment.test.ts b/tests/routing/combo-vision-enrollment.test.ts new file mode 100644 index 00000000000..6454d3ffefb --- /dev/null +++ b/tests/routing/combo-vision-enrollment.test.ts @@ -0,0 +1,467 @@ +import { ManagementRequest as Request } from "../helpers/management-auth"; +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { comboImagesSupported, comboVisionSidecarTargets } from "../../gui/src/combo-capabilities"; +import type { ModelOption } from "../../gui/src/components/combo-workspace-types"; +import { ConfigWritePublishedError } from "../../src/config/persist-unlocked"; +import { getConfigPath, saveConfig } from "../../src/config"; +import { handleManagementAPI } from "../../src/server/management-api"; +import type { OcxConfig } from "../../src/types"; +import { isModelVisionSidecarConsumer } from "../../src/vision/eligibility"; +import { MODEL_DISCOVERY_MAX_MODELS } from "../../src/providers/model-discovery-limits"; +import { catalogConvergenceFactory } from "../helpers/catalog-convergence"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const VALID_COMBO = { targets: [{ provider: "a", model: "m1" }] }; +function baseConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"], liveModels: false }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"], liveModels: false }, + c: { adapter: "openai-chat", baseUrl: "https://c.example/v1", apiKey: "kc", models: ["m3"], liveModels: false }, + }, + combos: { + free: { + strategy: "failover", + targets: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }, + ...overrides, + }; +} + +async function withTempHome(run: (dir: string) => Promise): Promise { + const previous = { ...process.env }; + const dir = mkdtempSync(join(tmpdir(), "ocx-combo-sidecar-")); + process.env.OPENCODEX_HOME = dir; + process.env.CODEX_HOME = join(dir, "codex"); + process.env.CLAUDE_CONFIG_DIR = join(dir, "claude"); + try { await run(dir); } + finally { + for (const key of ["OPENCODEX_HOME", "CODEX_HOME", "CLAUDE_CONFIG_DIR"] as const) { + if (previous[key] === undefined) delete process.env[key]; + else process.env[key] = previous[key]; + } + removeTreeWithRetry(dir); + } +} + +async function comboApi( + config: OcxConfig, + method: string, + path: string, + body?: unknown, + refreshCodexCatalog: () => Promise = async () => {}, +): Promise { + const req = new Request(`http://localhost${path}`, { + method, + headers: body === undefined ? undefined : { "content-type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + return handleManagementAPI(req, new URL(req.url), config, { + createManagementConvergeCodex: catalogConvergenceFactory(refreshCodexCatalog), + }); +} + +async function responseJson(response: Response | null): Promise> { + expect(response).not.toBeNull(); + return response!.json() as Promise>; +} + +describe("combo Vision Sidecar enrollment", () => { + test("PUT visionSidecarTargets declares exact members text-only and never persists the request field", async () => { + await withTempHome(async () => { + const base = baseConfig(); + const config = baseConfig({ + providers: { + ...base.providers, + b: { + ...base.providers.b!, + modelCapabilities: { m2: { contextTier: "long_context" } }, + }, + }, + combos: undefined, + }); + saveConfig(config); + const response = await comboApi(config, "PUT", "/api/combos", { + id: "mixed", + visionSidecarTargets: [ + { provider: "b", model: "m2" }, + { provider: "b", model: "m2" }, // exact duplicates are deduplicated + ], + combo: { + targets: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }); + expect(response?.status).toBe(200); + // Exact text-only declaration added; the sibling capability axis survives. + expect(config.providers?.b?.modelCapabilities).toEqual({ + m2: { contextTier: "long_context", inputModalities: ["text"] }, + }); + // Untouched member provider gains no declaration. + expect(config.providers?.a?.modelCapabilities).toBeUndefined(); + // Request-only field never leaks into the persisted combo. + expect(config.combos?.mixed).not.toHaveProperty("visionSidecarTargets"); + }); + }); + test("PUT rejects invalid visionSidecarTargets without mutating config", async () => { + await withTempHome(async () => { + const config = baseConfig({ combos: undefined }); + saveConfig(config); + const before = readFileSync(getConfigPath(), "utf8"); + const bodies: unknown[] = [ + { id: "x", visionSidecarTargets: "nope", combo: VALID_COMBO }, + { id: "x", visionSidecarTargets: [{ provider: "a" }], combo: VALID_COMBO }, + // Not a target of the submitted combo. + { id: "x", visionSidecarTargets: [{ provider: "c", model: "m3" }], combo: VALID_COMBO }, + // Enrollment only makes sense while images are accepted. + { + id: "x", + visionSidecarTargets: [{ provider: "a", model: "m1" }], + combo: { targets: [{ provider: "a", model: "m1" }], imageInput: "disabled" }, + }, + ]; + for (const body of bodies) { + const response = await comboApi(config, "PUT", "/api/combos", body); + expect(response?.status).toBe(400); + } + expect(readFileSync(getConfigPath(), "utf8")).toBe(before); + expect(config.providers?.a?.modelCapabilities).toBeUndefined(); + }); + }); + test("PUT rejects enrollment that would push a provider's modelCapabilities past its limit", async () => { + await withTempHome(async () => { + const base = baseConfig(); + // A provider already at the cap: the patch alone is valid, the merged map is not. + const full = Object.fromEntries( + Array.from({ length: MODEL_DISCOVERY_MAX_MODELS }, (_, i) => [`filler-${i}`, { contextTier: "long_context" as const }]), + ); + const config = baseConfig({ + providers: { ...base.providers, b: { ...base.providers.b!, modelCapabilities: full } }, + combos: undefined, + }); + saveConfig(config); + const before = readFileSync(getConfigPath(), "utf8"); + const response = await comboApi(config, "PUT", "/api/combos", { + id: "mixed", + visionSidecarTargets: [{ provider: "b", model: "m2" }], + combo: { targets: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] }, + }); + expect(response?.status).toBe(400); + expect(String((await responseJson(response)).error)).toContain("too many models"); + // Nothing is published: neither the combo nor a truncation-bound declaration. + expect(readFileSync(getConfigPath(), "utf8")).toBe(before); + expect(Object.keys(config.providers?.b?.modelCapabilities ?? {})).toHaveLength(MODEL_DISCOVERY_MAX_MODELS); + expect(config.combos?.mixed).toBeUndefined(); + }); + }); + test("PUT rejects already-image-capable and audio-only sidecar targets instead of overwriting", async () => { + await withTempHome(async () => { + const cases: Array<{ name: string; config: OcxConfig }> = [ + { + name: "capability axis already declares image", + config: baseConfig({ + providers: { + ...baseConfig().providers, + b: { ...baseConfig().providers.b!, modelCapabilities: { m2: { inputModalities: ["text", "image"] } } }, + }, + combos: undefined, + }), + }, + { + name: "capability axis declares audio only", + config: baseConfig({ + providers: { + ...baseConfig().providers, + b: { ...baseConfig().providers.b!, modelCapabilities: { m2: { inputModalities: ["audio"] } } }, + }, + combos: undefined, + }), + }, + { + name: "unrelated capability axis must not hide custom audio declaration", + config: baseConfig({ + providers: { + ...baseConfig().providers, + b: { ...baseConfig().providers.b!, modelCapabilities: { m2: { contextTier: "long_context" } } }, + }, + customModels: [{ id: "custom-b-m2", provider: "b", modelId: "m2", inputModalities: ["audio"] }], + combos: undefined, + }), + }, + { + name: "custom row declares audio only", + config: baseConfig({ + customModels: [{ id: "custom-b-m2", provider: "b", modelId: "m2", inputModalities: ["audio"] }], + combos: undefined, + }), + }, + { + name: "provider modality hint declares audio only", + config: baseConfig({ + providers: { + ...baseConfig().providers, + b: { ...baseConfig().providers.b!, modelInputModalities: { m2: ["audio"] } }, + }, + combos: undefined, + }), + }, + ]; + for (const { config } of cases) { + saveConfig(config); + const before = readFileSync(getConfigPath(), "utf8"); + const response = await comboApi(config, "PUT", "/api/combos", { + id: "mixed", + visionSidecarTargets: [{ provider: "b", model: "m2" }], + combo: { + targets: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }); + expect(response?.status).toBe(400); + expect(config.combos).toBeUndefined(); + expect(readFileSync(getConfigPath(), "utf8")).toBe(before); + } + }); + }); + test("PUT treats an already-covered member as a no-op write", async () => { + await withTempHome(async () => { + const exactTextOnly = baseConfig({ + providers: { + ...baseConfig().providers, + b: { ...baseConfig().providers.b!, modelCapabilities: { m2: { inputModalities: ["text"] } } }, + }, + combos: undefined, + }); + saveConfig(exactTextOnly); + const viaCapability = await comboApi(exactTextOnly, "PUT", "/api/combos", { + id: "mixed", + visionSidecarTargets: [{ provider: "b", model: "m2" }], + combo: { targets: [{ provider: "b", model: "m2" }] }, + }); + expect(viaCapability?.status).toBe(200); + // The operator's declaration is preserved verbatim, never rewritten. + expect(exactTextOnly.providers?.b?.modelCapabilities).toEqual({ m2: { inputModalities: ["text"] } }); + expect(isModelVisionSidecarConsumer(exactTextOnly.providers!.b!, "m2")).toBe(true); + + const viaNoVision = baseConfig({ + providers: { + ...baseConfig().providers, + b: { ...baseConfig().providers.b!, noVisionModels: ["m2"] }, + }, + combos: undefined, + }); + saveConfig(viaNoVision); + const response = await comboApi(viaNoVision, "PUT", "/api/combos", { + id: "mixed", + visionSidecarTargets: [{ provider: "b", model: "m2" }], + combo: { targets: [{ provider: "b", model: "m2" }] }, + }); + expect(response?.status).toBe(200); + expect(viaNoVision.providers?.b?.modelCapabilities).toBeUndefined(); + expect(isModelVisionSidecarConsumer(viaNoVision.providers!.b!, "m2")).toBe(true); + }); + }); + test("PUT enrolls the member so the runtime treats it as a sidecar consumer", async () => { + await withTempHome(async () => { + const config = baseConfig({ combos: undefined }); + saveConfig(config); + const response = await comboApi(config, "PUT", "/api/combos", { + id: "mixed", + visionSidecarTargets: [{ provider: "b", model: "m2" }], + combo: { targets: [{ provider: "b", model: "m2" }] }, + }); + expect(response?.status).toBe(200); + // The exact declaration the catalog's consumer check reads: text without image. + expect(isModelVisionSidecarConsumer(config.providers!.b!, "m2")).toBe(true); + const persisted = JSON.parse(readFileSync(getConfigPath(), "utf8")) as OcxConfig; + expect(persisted.providers?.b?.modelCapabilities?.m2?.inputModalities).toEqual(["text"]); + }); + }); + test("PUT rolls back combo, declarations, and migrated references when the save fails", async () => { + await withTempHome(async () => { + const config = baseConfig({ + disabledModels: ["combo/old"], + injectionModel: "combo/old", + claudeCode: { model: "combo/old", intercept: { modelMap: { upstream: "combo/old" } } }, + combos: { + old: { strategy: "failover", targets: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] }, + }, + subagentModels: ["combo/old"], + }); + saveConfig(config); + const before = readFileSync(getConfigPath(), "utf8"); + const req = new Request("http://localhost/api/combos", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: "new", + renameFrom: "old", + visionSidecarTargets: [{ provider: "b", model: "m2" }], + combo: { targets: [{ provider: "a", model: "m1" }, { provider: "b", model: "m2" }] }, + }), + }); + const response = await handleManagementAPI(req, new URL(req.url), config, { + createManagementConvergeCodex: catalogConvergenceFactory(), + saveConfigPreservingClaudeCode: () => { throw new Error("disk full"); }, + }); + expect(response?.status).toBe(500); + expect(await responseJson(response)).toEqual({ error: "combo could not be saved" }); + // Nothing moved: the combo keeps its old key, references stay unmigrated, + // and the sidecar declaration was never applied. + expect(Object.keys(config.combos!)).toEqual(["old"]); + expect(config.subagentModels).toEqual(["combo/old"]); + expect(config.disabledModels).toEqual(["combo/old"]); + expect(config.injectionModel).toBe("combo/old"); + expect(config.claudeCode).toMatchObject({ model: "combo/old", intercept: { modelMap: { upstream: "combo/old" } } }); + expect(config.providers?.b?.modelCapabilities).toBeUndefined(); + expect(readFileSync(getConfigPath(), "utf8")).toBe(before); + }); + }); + test("PUT rename commits enrollment and every public reference together", async () => { + await withTempHome(async () => { + const config = baseConfig({ + combos: { old: VALID_COMBO }, + disabledModels: ["combo/old"], + subagentModels: ["combo/old"], + injectionModel: "combo/old", + claudeCode: { model: "combo/old", intercept: { modelMap: { upstream: "combo/old" } } }, + }); + saveConfig(config); + const response = await comboApi(config, "PUT", "/api/combos", { + id: "new", renameFrom: "old", + visionSidecarTargets: [{ provider: "a", model: "m1" }], + combo: VALID_COMBO, + }); + expect(response?.status).toBe(200); + expect(config.combos?.old).toBeUndefined(); + expect(config.combos?.new).toBeDefined(); + expect(config.providers.a?.modelCapabilities?.m1?.inputModalities).toEqual(["text"]); + expect(config.disabledModels).toEqual(["combo/new"]); + expect(config.subagentModels).toEqual(["combo/new"]); + expect(config.injectionModel).toBe("combo/new"); + expect(config.claudeCode).toMatchObject({ model: "combo/new", intercept: { modelMap: { upstream: "combo/new" } } }); + const persisted = JSON.parse(readFileSync(getConfigPath(), "utf8")); + expect(persisted.combos.new).toBeDefined(); + expect(persisted.providers.a.modelCapabilities.m1.inputModalities).toEqual(["text"]); + expect(persisted.disabledModels).toEqual(["combo/new"]); + }); + }); + + test("published save failures retain live enrollment and surface the error", async () => { + await withTempHome(async () => { + const config = baseConfig({ combos: undefined }); + saveConfig(config); + const req = new Request("http://localhost/api/combos", { + method: "PUT", headers: { "content-type": "application/json" }, + body: JSON.stringify({ id: "mixed", combo: VALID_COMBO, + visionSidecarTargets: [{ provider: "a", model: "m1" }] }), + }); + await expect(handleManagementAPI(req, new URL(req.url), config, { + createManagementConvergeCodex: catalogConvergenceFactory(), + saveConfigPreservingClaudeCode: (value) => { + saveConfig(value); + throw new ConfigWritePublishedError(new Error("bookkeeping failed")); + }, + })).rejects.toBeInstanceOf(ConfigWritePublishedError); + const persisted = JSON.parse(readFileSync(getConfigPath(), "utf8")); + expect(config.combos?.mixed).toEqual(persisted.combos.mixed); + expect(config.providers.a?.modelCapabilities).toEqual(persisted.providers.a.modelCapabilities); + }); + }); + + test("GET models flags a legacy noVisionModels member as the sidecar consumer the runtime sees", async () => { + await withTempHome(async () => { + const config = baseConfig(); + for (const provider of Object.values(config.providers)) provider.liveModels = false; + config.providers.a!.noVisionModels = ["m1"]; + + const response = await comboApi(config, "GET", "/api/models"); + expect(response?.status).toBe(200); + const rows = await response!.json() as Array<{ + provider: string; + id: string; + inputModalities?: string[]; + inputModalitiesDeclared?: string[]; + visionSidecarConsumer?: boolean; + }>; + const row = rows.find(r => r.provider === "a" && r.id === "m1"); + // Legacy `noVisionModels` carries no declaration while the catalog widens the + // row to image: only the flag keeps the GUI from calling this native vision. + expect(row).toMatchObject({ visionSidecarConsumer: true }); + expect(row!.inputModalitiesDeclared).toBeUndefined(); + expect(row!.inputModalities).toContain("image"); + + // The Combos page parses exactly these whitelisted fields off the row. + const models: ModelOption[] = rows + .filter(r => r.provider !== "combo") + .map(r => ({ + provider: r.provider, + id: r.id, + ...(r.inputModalities?.length ? { inputModalities: r.inputModalities } : {}), + ...(r.inputModalitiesDeclared?.length ? { inputModalitiesDeclared: r.inputModalitiesDeclared } : {}), + ...(r.visionSidecarConsumer === true ? { visionSidecarConsumer: true } : {}), + })); + expect(comboImagesSupported([{ provider: "a", model: "m1" }], models)).toBe(true); + expect(comboVisionSidecarTargets([{ provider: "a", model: "m1" }], models)) + .toEqual([{ provider: "a", model: "m1" }]); + }); + }); + test("GET models flags registry-seeded noVisionModels rows as sidecar consumers", async () => { + await withTempHome(async () => { + const config = baseConfig({ + providers: { + ...baseConfig().providers, + umans: { + adapter: "anthropic", + baseUrl: "https://api.code.umans.ai", + models: ["umans-glm-5.2"], + liveModels: false, + }, + }, + }); + for (const provider of Object.values(config.providers)) provider.liveModels = false; + + const response = await comboApi(config, "GET", "/api/models"); + expect(response?.status).toBe(200); + const rows = await response!.json() as Array<{ + provider: string; + id: string; + inputModalities?: string[]; + inputModalitiesDeclared?: string[]; + visionSidecarConsumer?: boolean; + }>; + const row = rows.find(r => r.provider === "umans" && r.id === "umans-glm-5.2"); + // Registry enrichment classifies the row at read time and also seeds its + // declaration, but the flag stays the authoritative verdict for the GUI. + expect(row).toMatchObject({ visionSidecarConsumer: true }); + expect(row!.inputModalitiesDeclared).toEqual(["text"]); + expect(row!.inputModalities).toContain("image"); + + const models: ModelOption[] = rows + .filter(r => r.provider !== "combo") + .map(r => ({ + provider: r.provider, + id: r.id, + ...(r.inputModalities?.length ? { inputModalities: r.inputModalities } : {}), + ...(r.inputModalitiesDeclared?.length ? { inputModalitiesDeclared: r.inputModalitiesDeclared } : {}), + ...(r.visionSidecarConsumer === true ? { visionSidecarConsumer: true } : {}), + })); + expect(comboImagesSupported([{ provider: "umans", model: "umans-glm-5.2" }], models)).toBe(true); + expect(comboVisionSidecarTargets([{ provider: "umans", model: "umans-glm-5.2" }], models)) + .toEqual([{ provider: "umans", model: "umans-glm-5.2" }]); + }); + }); +});