diff --git a/README.md b/README.md index 31e24e2..22a54b6 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ opencode | 🔐 **Auth-aware** | Honours `LITELLM_API_KEY` / `LITELLM_MASTER_KEY` env vars, `provider.litellm.options.apiKey`, or the key you stored via OpenCode's `/connect`. | | 🌐 **Gateway-friendly** | Supports `customHeaders` for proxies behind Cloudflare Access or other API gateways requiring extra HTTP headers. | | 🧩 **Splittable catalog** | `includeModels` / `excludeModels` (glob patterns) let one LiteLLM proxy be divided into several OpenCode providers — e.g. by naming prefix — without hand-maintaining a model list. | +| 🎚️ **Capability overrides** | `modelCapabilities` forces or retracts per-model capability flags (`supports_vision`, `supports_function_calling`, …) when `/v1/model/info` is unavailable or disagrees with your deployment. | | ⏱️ **Non-blocking startup** | Health checks fail fast (3 s); discovery fetches are capped at **15 s** (configurable via `LITELLM_REQUEST_TIMEOUT_MS`) for slow remote proxies. Repeat config-hook invocations are a no-op. | | 📝 **TUI-safe logging** | All plugin logs go through OpenCode's log API (into OpenCode's own log files), never to stdout — the TUI stays intact. | | 🤝 **Non-destructive merge** | Only adds models you don't already have configured. Hand-curated entries are preserved verbatim. | @@ -280,6 +281,31 @@ If your LiteLLM catalog mixes naming conventions from different teams or environ - Patterns support only `*` (any run of characters); everything else is matched literally, so dots in ids like `gpt-4.1` need no escaping. - Filtering happens before the on-disk cache is written, so each provider's cached view respects its own filters. +### Correcting capability flags (`modelCapabilities`) + +Model classification (tool-call badge, attachments, reasoning, input modalities) leans on the capability flags LiteLLM reports via `/v1/model/info`. If your proxy doesn't expose that endpoint, or reports a flag that doesn't match your deployment, override flags per model id: + +```jsonc +{ + "provider": { + "litellm": { + "options": { + "baseURL": "http://localhost:4000/v1", + "modelCapabilities": { + "te-gpt-5.4-mini": { "supports_function_calling": true }, + "openai/gpt-4o": { "supports_vision": false } + } + } + } + } +} +``` + +- Overrides apply on top of whatever the proxy reports, with an explicit `false` winning — a flag the proxy never reported can be forced on just the same. +- Keys are exact model ids as they appear in `/v1/models` (not globs). +- Overridden flags flow into the picker exactly like natively reported ones, and the adjusted view is what gets persisted to the model cache. +- Changing `modelCapabilities` (or `includeModels`/`excludeModels`) starts a fresh discovery on the next start — the cache is scoped by that config — so the picker reflects the new flags immediately. + ## 🔧 How it works ```mermaid @@ -444,9 +470,11 @@ src/ │ ├── litellm-api.ts # health check, discovery (/v1/models + /v1/model/info), auto-detect │ ├── format-model-name.ts # name formatting, categorization │ ├── model-cache.ts # stale-while-revalidate on-disk model cache +│ ├── model-filter.ts # includeModels/excludeModels glob filtering +│ ├── model-capabilities.ts # per-model capability flag overrides │ └── opencode-auth.ts # fallback to OpenCode's /connect-stored credentials └── plugin/ - └── index.ts # LiteLLMPlugin entry (config hook, enrichment, filtering) + └── index.ts # LiteLLMPlugin entry (config hook, enrichment, filtering, capability overrides) test/ # vitest suite for the pure logic ``` diff --git a/src/plugin/index.ts b/src/plugin/index.ts index 641213f..3be27e3 100644 --- a/src/plugin/index.ts +++ b/src/plugin/index.ts @@ -13,9 +13,16 @@ import { } from '../utils/format-model-name' import type { LiteLLMModel, LiteLLMModelInfo } from '../types' import { getOpenCodeStoredApiKey } from '../utils/opencode-auth' -import { readModelCache, writeModelCache, readModelCacheSavedAt } from '../utils/model-cache' +import { + buildCacheKey, + readModelCache, + writeModelCache, + readModelCacheSavedAt, +} from '../utils/model-cache' import { passesModelFilter } from '../utils/model-filter' import type { ModelFilters } from '../utils/model-filter' +import { applyCapabilityOverrides, parseModelCapabilities } from '../utils/model-capabilities' +import type { ModelCapabilities } from '../utils/model-capabilities' const CHAT_PROVIDER_ID = 'litellm' // Covers the 3 s health check plus the parallel models/model-info fetch @@ -73,6 +80,7 @@ interface RefreshContext { apiKey?: string customHeaders?: Record filters: ModelFilters + capabilities: ModelCapabilities providerId: string } const refreshContexts = new Map() @@ -149,29 +157,40 @@ function readModelFilters(options: Record): ModelFilters { } /** - * Overlay metadata from `/v1/model/info` onto a `/v1/models` entry. - * Fields already present on the lean entry win; the info block only - * fills gaps (notably `mode`, which `/v1/models` omits for - * database-defined models). + * Overlay metadata onto a `/v1/models` entry in three tiers: the entry's + * own fields win, `/v1/model/info` fills gaps (notably `mode`, which + * `/v1/models` omits for database-defined models), and finally + * user-configured `modelCapabilities` overrides apply — explicit + * `false` included — because the user knows their deployment better + * than either endpoint (issue #25). */ -function enrichModel(model: LiteLLMModel, info: LiteLLMModelInfo): LiteLLMModel { - return { - ...model, - mode: model.mode ?? info.mode, - max_tokens: model.max_tokens ?? info.max_tokens, - max_input_tokens: model.max_input_tokens ?? info.max_input_tokens, - max_output_tokens: model.max_output_tokens ?? info.max_output_tokens, - supports_function_calling: model.supports_function_calling ?? info.supports_function_calling, - supports_vision: model.supports_vision ?? info.supports_vision, - supports_reasoning: model.supports_reasoning ?? info.supports_reasoning, - supports_pdf_input: model.supports_pdf_input ?? info.supports_pdf_input, - supports_audio_input: model.supports_audio_input ?? info.supports_audio_input, - input_cost_per_token: model.input_cost_per_token ?? info.input_cost_per_token, - output_cost_per_token: model.output_cost_per_token ?? info.output_cost_per_token, - cache_read_input_token_cost: model.cache_read_input_token_cost ?? info.cache_read_input_token_cost, - cache_creation_input_token_cost: - model.cache_creation_input_token_cost ?? info.cache_creation_input_token_cost, - } +function enrichModel( + model: LiteLLMModel, + info: LiteLLMModelInfo | undefined, + overrides?: Record, +): LiteLLMModel { + return applyCapabilityOverrides( + { + ...model, + mode: model.mode ?? info?.mode, + max_tokens: model.max_tokens ?? info?.max_tokens, + max_input_tokens: model.max_input_tokens ?? info?.max_input_tokens, + max_output_tokens: model.max_output_tokens ?? info?.max_output_tokens, + supports_function_calling: + model.supports_function_calling ?? info?.supports_function_calling, + supports_vision: model.supports_vision ?? info?.supports_vision, + supports_reasoning: model.supports_reasoning ?? info?.supports_reasoning, + supports_pdf_input: model.supports_pdf_input ?? info?.supports_pdf_input, + supports_audio_input: model.supports_audio_input ?? info?.supports_audio_input, + input_cost_per_token: model.input_cost_per_token ?? info?.input_cost_per_token, + output_cost_per_token: model.output_cost_per_token ?? info?.output_cost_per_token, + cache_read_input_token_cost: + model.cache_read_input_token_cost ?? info?.cache_read_input_token_cost, + cache_creation_input_token_cost: + model.cache_creation_input_token_cost ?? info?.cache_creation_input_token_cost, + }, + overrides, + ) } /** @@ -258,10 +277,10 @@ function toConfigModel( * * Pure with respect to plugin config: it performs the network calls, * classifies + formats each model, and returns a `{ id -> entry }` map. - * The provider's `includeModels`/`excludeModels` filters are applied - * here (not at merge time) so every path that persists or serves a - * cache — cold discovery and background refresh — writes the same - * filtered view. + * The provider's `includeModels`/`excludeModels` filters and + * `modelCapabilities` overrides are applied here (not at merge time) so + * every path that persists or serves a cache — cold discovery and + * background refresh — writes the same adjusted view. * * Returns `null` when the proxy is unreachable/unauthorized or exposes * no models, so callers can distinguish "no data" from "empty result". @@ -272,6 +291,7 @@ async function discoverModels( customHeaders: Record | undefined, providerId: string, filters: ModelFilters = {}, + capabilities: ModelCapabilities = {}, ): Promise | null> { if (!(await checkLiteLLMHealth(baseURL, apiKey, customHeaders))) { log( @@ -343,7 +363,7 @@ async function discoverModels( } const info = infoByName?.get(model.id) if (infoByName && !info) unmatched.push(model.id) - const entry = toConfigModel(info ? enrichModel(model, info) : model, info) + const entry = toConfigModel(enrichModel(model, info, capabilities[model.id]), info) if (!entry) { skipped++ continue @@ -424,7 +444,14 @@ async function backgroundRefresh(cacheKey: string): Promise { refreshInFlight.add(cacheKey) try { const built = await withTimeout( - discoverModels(ctx.baseURL, ctx.apiKey, ctx.customHeaders, ctx.providerId, ctx.filters), + discoverModels( + ctx.baseURL, + ctx.apiKey, + ctx.customHeaders, + ctx.providerId, + ctx.filters, + ctx.capabilities, + ), DISCOVERY_TIMEOUT_MS, ) if (built && Object.keys(built).length > 0) { @@ -525,6 +552,7 @@ export const LiteLLMPlugin: Plugin = async (input: PluginInput) => { const apiKey = configuredKey ?? envKey ?? storedKey const customHeaders = readCustomHeaders(options) const filters = readModelFilters(options) + const capabilities = parseModelCapabilities(options.modelCapabilities) // Resolve base URL let baseURL: string | null = null @@ -580,11 +608,21 @@ export const LiteLLMPlugin: Plugin = async (input: PluginInput) => { const models = actualProvider.models as Record - const cacheKey = `${providerId}@${baseURL}` + // Identity includes the filter/capability config: those are + // baked into cached entries, so changing them must start a + // fresh discovery instead of serving the old adjusted view. + const cacheKey = buildCacheKey(providerId, baseURL, filters, capabilities) // Remember how to reach this proxy so the `event` hook can // revalidate its cache in the background on new sessions. - refreshContexts.set(cacheKey, { baseURL, apiKey, customHeaders, filters, providerId }) + refreshContexts.set(cacheKey, { + baseURL, + apiKey, + customHeaders, + filters, + capabilities, + providerId, + }) // Repeat config-hook invocations within a run are a no-op once // we've injected this provider's models. @@ -616,7 +654,7 @@ export const LiteLLMPlugin: Plugin = async (input: PluginInput) => { // persist for subsequent startups. Capped by a timeout so a slow // proxy never blocks boot. const built = await withTimeout( - discoverModels(baseURL, apiKey, customHeaders, providerId, filters), + discoverModels(baseURL, apiKey, customHeaders, providerId, filters, capabilities), DISCOVERY_TIMEOUT_MS, ) if (built && Object.keys(built).length > 0) { diff --git a/src/utils/index.ts b/src/utils/index.ts index 7343ed7..a62fab8 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,3 +1,4 @@ export * from './litellm-api' export * from './format-model-name' +export * from './model-capabilities' export * from './model-filter' diff --git a/src/utils/litellm-api.ts b/src/utils/litellm-api.ts index 26eab9a..4125220 100644 --- a/src/utils/litellm-api.ts +++ b/src/utils/litellm-api.ts @@ -1,4 +1,5 @@ import type { LiteLLMModel, LiteLLMModelInfo, LiteLLMModelInfoResponse, LiteLLMModelsResponse } from '../types' +import { CAPABILITY_FLAGS } from './model-capabilities' export const DEFAULT_LITELLM_URL = 'http://localhost:4000' const MODELS_ENDPOINT = '/v1/models' @@ -128,14 +129,7 @@ export async function discoverLiteLLMModelInfo( // Some deployments set capability flags on the params block rather // than inside model_info. Fill those gaps so enrichment sees them. const info: LiteLLMModelInfo = { ...entry.model_info } - const capabilityFlags = [ - 'supports_vision', - 'supports_function_calling', - 'supports_reasoning', - 'supports_pdf_input', - 'supports_audio_input', - ] as const - for (const flag of capabilityFlags) { + for (const flag of CAPABILITY_FLAGS) { const paramsValue = entry.litellm_params?.[flag] if (info[flag] == null && typeof paramsValue === 'boolean') { info[flag] = paramsValue diff --git a/src/utils/model-cache.ts b/src/utils/model-cache.ts index 04853dc..e8c829a 100644 --- a/src/utils/model-cache.ts +++ b/src/utils/model-cache.ts @@ -47,6 +47,57 @@ function cacheFile(cacheKey: string): string { return join(cacheDir(), `models-${hash}.json`) } +/** + * Canonical JSON serialization (object keys sorted) so semantically + * identical filter/capability configs fingerprint identically no matter + * the key order in the user's `opencode.json`. + */ +function canonicalize(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalize).join(',')}]` + if (value && typeof value === 'object') { + const entries = Object.entries(value as Record) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalize(v)}`) + return `{${entries.join(',')}}` + } + return JSON.stringify(value) +} + +/** + * Cache identity for a provider's model view. `providerId@baseURL` + * alone isn't enough: `includeModels`/`excludeModels` and + * `modelCapabilities` are baked into cached entries by discovery, so a + * config change must not reuse the old cache — it would keep serving + * the previous adjusted view until a second restart. A fingerprint is + * appended whenever adjustments exist (pattern order ignored); default + * configs keep the plain key so existing caches stay warm across + * plugin upgrades. + */ +export function buildCacheKey( + providerId: string, + baseURL: string, + filters: { includeModels?: string[]; excludeModels?: string[] }, + capabilities: Record>, +): string { + const base = `${providerId}@${baseURL}` + const hasAdjustments = + (filters.includeModels?.length ?? 0) > 0 || + (filters.excludeModels?.length ?? 0) > 0 || + Object.keys(capabilities).length > 0 + if (!hasAdjustments) return base + const fingerprint = createHash('sha256') + .update( + canonicalize({ + includeModels: filters.includeModels ? [...filters.includeModels].sort() : undefined, + excludeModels: filters.excludeModels ? [...filters.excludeModels].sort() : undefined, + capabilities, + }), + ) + .digest('hex') + .slice(0, 12) + return `${base}@${fingerprint}` +} + /** * Read cached model entries for a cache key. Returns `null` on any * problem (missing file, parse error, version mismatch, or an entry diff --git a/src/utils/model-capabilities.ts b/src/utils/model-capabilities.ts new file mode 100644 index 0000000..95fe615 --- /dev/null +++ b/src/utils/model-capabilities.ts @@ -0,0 +1,50 @@ +export const CAPABILITY_FLAGS = [ + 'supports_vision', + 'supports_function_calling', + 'supports_reasoning', + 'supports_pdf_input', + 'supports_audio_input', +] as const + +export type CapabilityFlag = (typeof CAPABILITY_FLAGS)[number] + +/** + * Per-model boolean capability overrides keyed by exact model id + * (`provider.litellm.options.modelCapabilities`). Keys of the inner + * record are not restricted to `CapabilityFlag`: any boolean capability + * key LiteLLM reports is accepted, so future flags work without a + * plugin change. + */ +export type ModelCapabilities = Record> + +/** + * Defensively parse the `modelCapabilities` provider option. Non-object + * input, non-object per-model values, and non-boolean flag values are + * dropped; everything else (including explicit `false`) is kept. + */ +export function parseModelCapabilities(raw: unknown): ModelCapabilities { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {} + const out: ModelCapabilities = {} + for (const [modelId, flags] of Object.entries(raw as Record)) { + if (!flags || typeof flags !== 'object' || Array.isArray(flags)) continue + const parsed: Record = {} + for (const [flag, value] of Object.entries(flags as Record)) { + if (typeof value === 'boolean') parsed[flag] = value + } + if (Object.keys(parsed).length > 0) out[modelId] = parsed + } + return out +} + +/** + * Overlay user-configured capability flags onto a (possibly enriched) + * model entry. Explicit `false` wins over whatever the proxy reported, + * and unknown-to-the-proxy flags are added outright. + */ +export function applyCapabilityOverrides( + model: T, + overrides?: Readonly>, +): T { + if (!overrides || Object.keys(overrides).length === 0) return model + return Object.assign({}, model, overrides) +} diff --git a/test/model-cache.test.ts b/test/model-cache.test.ts index 1c958f5..d77b962 100644 --- a/test/model-cache.test.ts +++ b/test/model-cache.test.ts @@ -3,7 +3,7 @@ import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'n import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, beforeAll, describe, expect, it } from 'vitest' -import { readModelCache, readModelCacheSavedAt, writeModelCache } from '../src/utils/model-cache' +import { readModelCache, readModelCacheSavedAt, writeModelCache, buildCacheKey } from '../src/utils/model-cache' const KEY = 'litellm@http://localhost:4000' @@ -83,3 +83,90 @@ describe('model cache', () => { expect(readdirSync(dir).filter((f) => f.endsWith('.tmp'))).toEqual([]) }) }) + +describe('buildCacheKey', () => { + it('keeps the plain providerId@baseURL key when no adjustments are configured', () => { + expect(buildCacheKey('litellm', 'http://localhost:4000', {}, {})).toBe(KEY) + expect( + buildCacheKey( + 'litellm', + 'http://localhost:4000', + { includeModels: undefined, excludeModels: undefined }, + {}, + ), + ).toBe(KEY) + }) + + it('changes the key when capability overrides are added or changed', () => { + // Regression for the CodeRabbit review finding: a model previously + // cached with `supports_vision: false` must not be served again + // after the user flips the override to true. + const base = 'litellm@http://localhost:4000' + const withoutOverrides = buildCacheKey('litellm', 'http://localhost:4000', {}, {}) + const visionOff = buildCacheKey( + 'litellm', + 'http://localhost:4000', + {}, + { 'openai/gpt-4o': { supports_vision: false } }, + ) + const visionOn = buildCacheKey( + 'litellm', + 'http://localhost:4000', + {}, + { 'openai/gpt-4o': { supports_vision: true } }, + ) + expect(visionOff).not.toBe(withoutOverrides) + expect(visionOff).not.toBe(visionOn) + expect(visionOn.startsWith(base)).toBe(true) + }) + + it('changes the key when includeModels/excludeModels are added', () => { + const plain = buildCacheKey('litellm', 'http://localhost:4000', {}, {}) + const withFilters = buildCacheKey( + 'litellm', + 'http://localhost:4000', + { includeModels: ['prod/*'] }, + {}, + ) + const withMore = buildCacheKey( + 'litellm', + 'http://localhost:4000', + { includeModels: ['prod/*', 'team/*'] }, + {}, + ) + expect(withFilters).not.toBe(plain) + expect(withFilters).not.toBe(withMore) + }) + + it('is stable regardless of config key order', () => { + expect( + buildCacheKey( + 'litellm', + 'http://localhost:4000', + {}, + { + 'openai/gpt-4o': { supports_vision: true, supports_reasoning: false }, + 'claude-sonnet': { supports_pdf_input: true }, + }, + ), + ).toBe( + buildCacheKey( + 'litellm', + 'http://localhost:4000', + {}, + { + 'claude-sonnet': { supports_pdf_input: true }, + 'openai/gpt-4o': { supports_reasoning: false, supports_vision: true }, + }, + ), + ) + }) + + it('treats reordered glob patterns as the same config', () => { + expect( + buildCacheKey('litellm', 'http://localhost:4000', { includeModels: ['prod/*', 'team/*'] }, {}), + ).toBe( + buildCacheKey('litellm', 'http://localhost:4000', { includeModels: ['team/*', 'prod/*'] }, {}), + ) + }) +}) diff --git a/test/model-capabilities.test.ts b/test/model-capabilities.test.ts new file mode 100644 index 0000000..53b3995 --- /dev/null +++ b/test/model-capabilities.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' +import { + CAPABILITY_FLAGS, + applyCapabilityOverrides, + parseModelCapabilities, +} from '../src/utils/model-capabilities' + +describe('parseModelCapabilities', () => { + it('returns an empty map for non-object input', () => { + expect(parseModelCapabilities(undefined)).toEqual({}) + expect(parseModelCapabilities(null)).toEqual({}) + expect(parseModelCapabilities('supports_vision')).toEqual({}) + expect(parseModelCapabilities(['supports_vision'])).toEqual({}) + }) + + it('keeps boolean flags and drops non-boolean values', () => { + expect( + parseModelCapabilities({ + 'gpt-4o': { supports_vision: true, mode: 'chat', supports_reasoning: 'yes' }, + }), + ).toEqual({ 'gpt-4o': { supports_vision: true } }) + }) + + it('preserves explicit false overrides', () => { + expect(parseModelCapabilities({ 'gpt-4o': { supports_vision: false } })).toEqual({ + 'gpt-4o': { supports_vision: false }, + }) + }) + + it('accepts capability keys beyond the known flags', () => { + expect( + parseModelCapabilities({ 'gpt-4o': { supports_prompt_caching: true } }), + ).toEqual({ 'gpt-4o': { supports_prompt_caching: true } }) + }) + + it('skips ids whose flags contain no boolean values', () => { + expect(parseModelCapabilities({ m1: { mode: 'chat' }, m2: {} })).toEqual({}) + }) + + it('skips non-object per-model values', () => { + expect( + parseModelCapabilities({ m1: true, m2: 'vision', m3: ['supports_vision'] }), + ).toEqual({}) + }) +}) + +describe('applyCapabilityOverrides', () => { + it('spreads overrides over the base model so explicit false wins', () => { + expect( + applyCapabilityOverrides({ id: 'gpt-4o', supports_vision: true }, { supports_vision: false }), + ).toEqual({ id: 'gpt-4o', supports_vision: false }) + }) + + it('adds capability flags the proxy never reported', () => { + expect(applyCapabilityOverrides({ id: 'm' }, { supports_audio_input: true })).toEqual({ + id: 'm', + supports_audio_input: true, + }) + }) + + it('returns the base object untouched when there are no overrides', () => { + const base = { id: 'gpt-4o', supports_vision: true } + expect(applyCapabilityOverrides(base)).toBe(base) + expect(applyCapabilityOverrides(base, {})).toBe(base) + }) + + it('does not mutate the base object', () => { + const base = { id: 'gpt-4o', supports_vision: true as boolean | undefined } + const merged = applyCapabilityOverrides(base, { supports_vision: false }) + expect(base.supports_vision).toBe(true) + expect(merged).not.toBe(base) + }) +}) + +describe('CAPABILITY_FLAGS', () => { + it('covers the capability flags enriched from /v1/model/info', () => { + expect([...CAPABILITY_FLAGS]).toEqual([ + 'supports_vision', + 'supports_function_calling', + 'supports_reasoning', + 'supports_pdf_input', + 'supports_audio_input', + ]) + }) +})