From 959a58b726bc91ef040891dfc9a91bbc1c2e37da Mon Sep 17 00:00:00 2001 From: Xi Lyu Date: Thu, 29 Oct 2026 21:27:50 +0000 Subject: [PATCH 1/4] Check the VS Code extension's requests against a real gateway npm run live sends the requests the extension builds to a real AI Gateway: a streamed turn and a tool call on each route, an image inside a tool result, a prompt-cache read, the refused-output-limit retry, and that a Model Provider Service's models are served only with its routing header and not on the mlflow route. A new CI job runs it with the workspace secrets, and skips where those aren't available. Co-authored-by: Isaac --- .github/workflows/ci.yml | 34 ++ vscode-extension/.live-out/requests.live.js | 580 ++++++++++++++++++++ vscode-extension/README.md | 17 + vscode-extension/package.json | 3 +- vscode-extension/test/live/requests.live.ts | 197 +++++++ 5 files changed, 830 insertions(+), 1 deletion(-) create mode 100644 vscode-extension/.live-out/requests.live.js create mode 100644 vscode-extension/test/live/requests.live.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a3a0f2b4..a497f8332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,40 @@ jobs: exit 1 } + vscode-extension-live: + name: VS Code extension · live gateway + runs-on: ubuntu-latest + needs: vscode-extension + # Needs the workspace secrets, which pull requests from forks don't get, and the merge queue + # reruns nothing that already ran on the PR. + if: ${{ github.event_name != 'merge_group' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + timeout-minutes: 10 + defaults: + run: + working-directory: vscode-extension + env: + UG_LIVE_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} + # See the e2e shards: a hosted runner has no cached login, so CI supplies a pre-fetched bearer. + UG_LIVE_TOKEN: ${{ secrets.DATABRICKS_BEARER }} + UG_LIVE_CLAUDE_MODEL: system.ai.claude-haiku-4-5 + UG_LIVE_GPT_MODEL: system.ai.gpt-5-4-nano + # The services and targets the integration CUJs use (scripts/run_integration.py). + UG_LIVE_MPS: main.ucode.ci_e2e_anthropic_nonrelay_mps + UG_LIVE_MPS_MODEL: claude-haiku-4-5-20251001 + UG_LIVE_OPENAI_MPS: main.ucode.ci_openai_mps + UG_LIVE_OPENAI_MPS_MODEL: gpt-5-nano + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: vscode-extension/package-lock.json + - run: npm ci --no-audit --no-fund + - run: npm run live + e2e-shards: name: ${{ matrix.name }} · ${{ matrix.os == 'windows-latest' && 'Windows' || 'Linux' }} runs-on: ${{ matrix.os }} diff --git a/vscode-extension/.live-out/requests.live.js b/vscode-extension/.live-out/requests.live.js new file mode 100644 index 000000000..c1f109de5 --- /dev/null +++ b/vscode-extension/.live-out/requests.live.js @@ -0,0 +1,580 @@ +"use strict"; +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + // If the importer is in node compatibility mode or this is not an ESM + // file that has been converted to a CommonJS file using a Babel- + // compatible transform (i.e. "__esModule" has not been set), then set + // "default" to the CommonJS "module.exports" for node compatibility. + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); + +// test/live/requests.live.ts +var import_strict2 = __toESM(require("node:assert/strict")); +var import_node_test = require("node:test"); + +// src/auth.ts +var import_node_child_process = require("node:child_process"); +var EXPIRY_MARGIN_MS = 5 * 6e4; +var DEFAULT_LIFETIME_MS = 60 * 6e4; +var run = (command) => new Promise((resolve, reject) => { + (0, import_node_child_process.exec)(command, { timeout: 6e4, windowsHide: true }, (error, stdout, stderr) => { + if (error) { + reject(new Error(`Couldn't get a Databricks token: \`${command}\` failed: ${String(stderr).trim().slice(0, 500) || error.message}`)); + } else { + resolve(String(stdout).trim()); + } + }); +}); +var expiresAt = (token) => { + try { + const { exp } = JSON.parse(Buffer.from(token.split(".")[1] ?? "", "base64url").toString()); + if (typeof exp === "number") { + return exp * 1e3; + } + } catch { + } + return Date.now() + DEFAULT_LIFETIME_MS; +}; +var Tokens = class { + constructor(command, onFetch = () => { + }) { + this.command = command; + this.onFetch = onFetch; + } + command; + onFetch; + cached; + pending; + get() { + if (this.cached && this.cached.expiresAt - EXPIRY_MARGIN_MS > Date.now()) { + return Promise.resolve(this.cached.value); + } + return this.fetch(this.command); + } + /** A token the gateway rejected. The CLI can hand back its cached copy while `exp` is still ahead, so force a new one. */ + async renew(rejected) { + await this.pending?.catch(() => { + }); + if (this.cached && this.cached.value !== rejected) { + return this.cached.value; + } + this.cached = void 0; + return this.fetch(`${this.command} --force-refresh`); + } + fetch(command) { + this.pending ??= run(command).then((value) => { + if (!value) { + throw new Error(`Couldn't get a Databricks token: \`${command}\` printed nothing.`); + } + this.cached = { value, expiresAt: expiresAt(value) }; + this.onFetch(this.cached.expiresAt); + return value; + }).finally(() => { + this.pending = void 0; + }); + return this.pending; + } +}; + +// test/vscode.ts +var LanguageModelTextPart = class { + constructor(value) { + this.value = value; + } + value; +}; +var LanguageModelToolCallPart = class { + constructor(callId, name, input) { + this.callId = callId; + this.name = name; + this.input = input; + } + callId; + name; + input; +}; +var LanguageModelToolResultPart = class { + constructor(callId, content) { + this.callId = callId; + this.content = content; + } + callId; + content; +}; +var LanguageModelDataPart = class { + constructor(data, mimeType) { + this.data = data; + this.mimeType = mimeType; + } + data; + mimeType; +}; +var LanguageModelPromptTsxPart = class { + constructor(value) { + this.value = value; + } + value; +}; +var LanguageModelChatMessageRole = { User: 1, Assistant: 2 }; + +// src/formats.ts +var SYSTEM_ROLE = 3; +var textOf = (part) => { + if (part instanceof LanguageModelTextPart) { + return part.value; + } + if (part instanceof LanguageModelDataPart && !part.mimeType.startsWith("image/")) { + return new TextDecoder().decode(part.data); + } + if (part instanceof LanguageModelPromptTsxPart) { + return JSON.stringify(part.value); + } + return ""; +}; +var imageData = (part) => part instanceof LanguageModelDataPart && part.mimeType.startsWith("image/") ? { mimeType: part.mimeType, data: Buffer.from(part.data).toString("base64") } : void 0; +var imageUrl = (part) => { + const image = imageData(part); + return image && `data:${image.mimeType};base64,${image.data}`; +}; +var parametersOf = (tool) => tool.inputSchema ?? { type: "object", properties: {} }; +var toolCall = (callId, name, args) => { + let input; + try { + input = args.trim() ? JSON.parse(args) : {}; + } catch { + input = void 0; + } + if (typeof input !== "object" || input === null || Array.isArray(input)) { + throw new Error(`The model called ${name} with arguments that aren't a JSON object: ${args}`); + } + return new LanguageModelToolCallPart(callId, name, input); +}; +var stoppedEarly = (reason) => reason === "max_output_tokens" || reason === "length" || reason === "max_tokens" ? new Error("The model stopped at its output token limit before finishing the response.") : reason === "content_filter" || reason === "refusal" ? new Error("The model's response was stopped by a content filter.") : new Error(`The model stopped before finishing the response (${reason ?? "no reason given"}).`); +var cutOff = () => new Error("The response ended before the model finished. Try again."); +var toolOptions = (turn, toTool) => turn.tools.length ? { tools: turn.tools.map(toTool), ...turn.toolRequired ? { tool_choice: "required" } : {} } : {}; +var imagesOf = (parts) => parts.map(imageUrl).filter((url) => url !== void 0); +var sortParts = (content) => ({ + results: content.filter((part) => part instanceof LanguageModelToolResultPart), + calls: content.filter((part) => part instanceof LanguageModelToolCallPart), + text: content.map(textOf).join(""), + images: imagesOf(content) +}); +function chatCompletionsRequest(turn) { + const messages = turn.messages.flatMap((message2) => { + const { results, calls, text: text2, images } = sortParts(message2.content); + if (Number(message2.role) === SYSTEM_ROLE) { + return [{ role: "system", content: text2 }]; + } + if (message2.role === LanguageModelChatMessageRole.Assistant) { + const toolCalls = calls.map((call2) => ({ + id: call2.callId, + type: "function", + function: { name: call2.name, arguments: JSON.stringify(call2.input) } + })); + return text2 || toolCalls.length ? [{ role: "assistant", content: text2 || null, ...toolCalls.length ? { tool_calls: toolCalls } : {} }] : []; + } + const content = [ + ...text2 ? [{ type: "text", text: text2 }] : [], + ...[...results.flatMap((result2) => imagesOf(result2.content)), ...images].map((url) => ({ type: "image_url", image_url: { url } })) + ]; + return [ + ...results.map((result2) => ({ role: "tool", tool_call_id: result2.callId, content: result2.content.map(textOf).join("") })), + ...content.length ? [{ role: "user", content }] : [] + ]; + }); + return { + model: turn.model, + messages, + stream: true, + ...toolOptions(turn, (tool) => ({ type: "function", function: { name: tool.name, description: tool.description, parameters: parametersOf(tool) } })) + }; +} +var deltaText = (content) => typeof content === "string" ? content : (content ?? []).filter((block) => block.type === "text").map((block) => block.text ?? "").join(""); +var CHAT_FINISHED = /* @__PURE__ */ new Set(["stop", "tool_calls", "function_call"]); +async function* chatCompletionsReplies(events) { + const calls = []; + let finish; + for await (const event of events) { + const { error } = event; + if (error) { + throw new Error(typeof error === "string" ? error : error.message ?? JSON.stringify(error)); + } + for (const choice of event.choices ?? []) { + const text2 = deltaText(choice.delta?.content); + if (text2) { + yield new LanguageModelTextPart(text2); + } + for (const fragment of choice.delta?.tool_calls ?? []) { + const call2 = calls[fragment.index ?? 0] ??= { id: "", name: "", args: "" }; + call2.id = fragment.id || call2.id; + call2.name = fragment.function?.name || call2.name; + call2.args += fragment.function?.arguments ?? ""; + } + if (choice.finish_reason) { + finish = choice.finish_reason; + if (!CHAT_FINISHED.has(finish)) { + throw stoppedEarly(finish); + } + } + } + } + if (!finish) { + throw cutOff(); + } + for (const call2 of calls.filter(Boolean)) { + yield toolCall(call2.id, call2.name, call2.args); + } +} +var functionOutput = (content) => { + const text2 = content.map(textOf).join(""); + const images = imagesOf(content); + return images.length ? [...text2 ? [{ type: "input_text", text: text2 }] : [], ...images.map((url) => ({ type: "input_image", image_url: url, detail: "auto" }))] : text2; +}; +function responsesRequest(turn) { + const input = turn.messages.flatMap((message2) => { + const { results, calls, text: text2, images } = sortParts(message2.content); + if (Number(message2.role) === SYSTEM_ROLE) { + return [{ role: "system", content: text2 }]; + } + if (message2.role === LanguageModelChatMessageRole.Assistant) { + return [ + ...text2 ? [{ role: "assistant", content: text2 }] : [], + ...calls.map((call2) => ({ type: "function_call", call_id: call2.callId, name: call2.name, arguments: JSON.stringify(call2.input) })) + ]; + } + const content = [ + ...text2 ? [{ type: "input_text", text: text2 }] : [], + ...images.map((url) => ({ type: "input_image", image_url: url, detail: "auto" })) + ]; + return [ + ...results.map((result2) => ({ type: "function_call_output", call_id: result2.callId, output: functionOutput(result2.content) })), + ...content.length ? [{ role: "user", content }] : [] + ]; + }); + return { + model: turn.model, + input, + stream: true, + store: false, + ...toolOptions(turn, (tool) => ({ type: "function", name: tool.name, description: tool.description, parameters: parametersOf(tool), strict: false })) + }; +} +async function* responsesReplies(events) { + const calls = []; + let completed = false; + for await (const raw of events) { + const event = raw; + if (event.type === "response.output_text.delta" && event.delta) { + yield new LanguageModelTextPart(event.delta); + } else if (event.type === "response.output_item.done" && event.item?.type === "function_call") { + calls.push(toolCall(event.item.call_id ?? "", event.item.name ?? "", event.item.arguments ?? "")); + } else if (event.type === "response.completed") { + completed = true; + } else if (event.type === "response.incomplete") { + throw stoppedEarly(event.response?.incomplete_details?.reason); + } else if (event.type === "response.failed" || event.type === "error") { + throw new Error(event.response?.error?.message ?? event.message ?? "The model response failed."); + } + } + if (!completed) { + throw cutOff(); + } + yield* calls; +} +var textBlocks = (text2) => text2 ? [{ type: "text", text: text2 }] : []; +var imageBlocks = (parts) => parts.flatMap((part) => { + const image = imageData(part); + return image ? [{ type: "image", source: { type: "base64", media_type: image.mimeType, data: image.data } }] : []; +}); +var CACHE_POINT = { cache_control: { type: "ephemeral" } }; +var cacheUpTo = (blocks) => { + if (blocks.length) { + blocks[blocks.length - 1] = { ...blocks[blocks.length - 1], ...CACHE_POINT }; + } +}; +function anthropicMessagesRequest(turn, maxTokens) { + const system = []; + const messages = []; + for (const message2 of turn.messages) { + const { results, calls, text: text2 } = sortParts(message2.content); + if (Number(message2.role) === SYSTEM_ROLE) { + system.push(...textBlocks(text2)); + continue; + } + const role = message2.role === LanguageModelChatMessageRole.Assistant ? "assistant" : "user"; + const content = role === "assistant" ? [...textBlocks(text2), ...calls.map((call2) => ({ type: "tool_use", id: call2.callId, name: call2.name, input: call2.input }))] : [ + // Tool results have to come first in the user message that follows the calls. + ...results.map((result2) => { + const blocks = [...textBlocks(result2.content.map(textOf).join("")), ...imageBlocks(result2.content)]; + return { type: "tool_result", tool_use_id: result2.callId, ...blocks.length ? { content: blocks } : {} }; + }), + ...textBlocks(text2), + ...imageBlocks(message2.content) + ]; + const last = messages[messages.length - 1]; + if (!content.length) { + continue; + } else if (last?.role === role) { + last.content.push(...content); + } else { + messages.push({ role, content }); + } + } + cacheUpTo(system); + cacheUpTo(messages[messages.length - 1]?.content ?? []); + return { + model: turn.model, + max_tokens: maxTokens, + ...system.length ? { system } : {}, + messages, + stream: true, + ...turn.tools.length ? { + tools: turn.tools.map((tool) => ({ name: tool.name, description: tool.description, input_schema: parametersOf(tool) })), + ...turn.toolRequired ? { tool_choice: { type: "any" } } : {} + } : {} + }; +} +var ANTHROPIC_FINISHED = /* @__PURE__ */ new Set(["end_turn", "tool_use", "stop_sequence"]); +async function* anthropicMessagesReplies(events) { + const calls = /* @__PURE__ */ new Map(); + let stopped = false; + for await (const raw of events) { + const event = raw; + if (event.type === "content_block_start" && event.content_block?.type === "tool_use") { + calls.set(event.index ?? 0, { id: event.content_block.id ?? "", name: event.content_block.name ?? "", args: "" }); + } else if (event.type === "content_block_delta" && event.delta?.type === "text_delta" && event.delta.text) { + yield new LanguageModelTextPart(event.delta.text); + } else if (event.type === "content_block_delta" && event.delta?.type === "input_json_delta") { + const call2 = calls.get(event.index ?? 0); + if (call2) { + call2.args += event.delta.partial_json ?? ""; + } + } else if (event.type === "message_delta" && event.delta?.stop_reason && !ANTHROPIC_FINISHED.has(event.delta.stop_reason)) { + throw stoppedEarly(event.delta.stop_reason); + } else if (event.type === "message_stop") { + stopped = true; + } else if (event.type === "error") { + throw new Error(event.error?.message ?? "The model response failed."); + } + } + if (!stopped) { + throw cutOff(); + } + for (const call2 of calls.values()) { + yield toolCall(call2.id, call2.name, call2.args); + } +} + +// src/gateway.ts +async function* streamEvents(workspace, path, body, tokens2, headers, signal) { + const post = (token2, payload) => fetch(`${workspace}${path}`, { + method: "POST", + headers: { ...headers, Authorization: `Bearer ${token2}`, "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal + }); + let token = await tokens2.get(); + let response = await post(token, body); + if (response.status === 401) { + await response.body?.cancel(); + token = await tokens2.renew(token); + response = await post(token, body); + } + if (response.status === 401) { + await response.body?.cancel(); + throw new Error("Databricks rejected the sign-in. Run `ug configure --agents vscode` to sign in again, then retry."); + } + if (!response.ok || !response.body) { + const detail = (await response.text()).slice(0, 300); + const allowed = Number(/max_tokens: \d+ > (\d+)/.exec(detail)?.[1]); + if (!allowed) { + throw new Error(`AI Gateway returned ${response.status}: ${detail}`); + } + response = await post(token, { ...body, max_tokens: allowed }); + if (!response.ok || !response.body) { + throw new Error(`AI Gateway returned ${response.status}: ${(await response.text()).slice(0, 300)}`); + } + } + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of response.body) { + buffer += decoder.decode(chunk, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + for (const line of lines) { + const data = dataOf(line); + if (data === "[DONE]") { + return; + } + if (data) { + yield JSON.parse(data); + } + } + } + const last = dataOf(buffer + decoder.decode()); + if (last && last !== "[DONE]") { + yield JSON.parse(last); + } +} +var dataOf = (line) => line.startsWith("data:") ? line.slice(5).trim() : ""; + +// src/extension.ts +var PROVIDER_HEADER = "Databricks-Model-Provider-Service"; + +// test/support.ts +var import_strict = __toESM(require("node:assert/strict")); +var SYSTEM = 3; +var USER = 1; +var ASSISTANT = 2; +var message = (role, ...content) => ({ role, content, name: void 0 }); +var text = (value) => new LanguageModelTextPart(value); +var call = (id, name, input) => new LanguageModelToolCallPart(id, name, input); +var result = (id, ...content) => new LanguageModelToolResultPart(id, content); +var collect = async (items) => { + const out = []; + for await (const item of items) { + out.push(item); + } + return out; +}; +var rejectsWith = (promise, text2) => import_strict.default.rejects(promise, (error) => { + import_strict.default.ok(error.message.includes(text2), `expected "${error.message}" to contain "${text2}"`); + return true; +}); + +// test/live/requests.live.ts +var env = (name) => process.env[name] ?? ""; +var WORKSPACE = env("UG_LIVE_WORKSPACE").replace(/\/+$/, ""); +var AUTH_COMMAND = env("UG_LIVE_AUTH_COMMAND") || env("UG_LIVE_TOKEN") && `printf %s "$UG_LIVE_TOKEN"`; +if (!WORKSPACE || !AUTH_COMMAND) { + throw new Error("Set UG_LIVE_WORKSPACE and UG_LIVE_AUTH_COMMAND (or UG_LIVE_TOKEN) to run the live tests."); +} +var MLFLOW = "/ai-gateway/mlflow/v1"; +var OPENAI = "/ai-gateway/openai/v1"; +var MESSAGES = "/ai-gateway/anthropic/v1/messages"; +var ANTHROPIC_VERSION = { "anthropic-version": "2023-06-01" }; +var tokens = new Tokens(AUTH_COMMAND); +var send = (path, body, headers = {}) => streamEvents(WORKSPACE, path, body, tokens, { ...headers }, new AbortController().signal); +var turnOf = (over) => ({ + messages: [], + tools: [], + toolRequired: false, + ...over +}); +var textOf2 = (replies) => replies.filter((reply) => reply instanceof LanguageModelTextPart).map((part) => part.value).join(""); +var callsOf = (replies) => replies.filter((reply) => reply instanceof LanguageModelToolCallPart); +var RED_PNG = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAFklEQVR4nGO4IyJCEmIY1TCqYfhqAAACcQQQFwFJQgAAAABJRU5ErkJggg=="; +var redImage = () => new LanguageModelDataPart(Buffer.from(RED_PNG, "base64"), "image/png"); +var viewImageTool = { name: "view_image", description: "Show the image at a path", inputSchema: { type: "object", properties: { path: { type: "string" } } } }; +var marker = () => `UG${Math.floor(Math.random() * 1e6)}`; +var echoTurn = (model, mark) => turnOf({ + model, + messages: [message(SYSTEM, text("Answer in as few words as possible.")), message(USER, text(`Reply with exactly: ${mark}`))] +}); +var imageTurn = (model) => turnOf({ + model, + tools: [viewImageTool], + messages: [ + message(USER, text("Look at a.png with the tool, then name its main colour in one lowercase word.")), + message(ASSISTANT, call("c1", "view_image", { path: "a.png" })), + message(USER, result("c1", text("a.png:"), redImage())) + ] +}); +var anthropic = (turn, maxTokens, headers = {}) => collect(anthropicMessagesReplies(send(MESSAGES, anthropicMessagesRequest(turn, maxTokens), { ...ANTHROPIC_VERSION, ...headers }))); +var responses = (turn, base = MLFLOW, headers = {}) => collect(responsesReplies(send(`${base}/responses`, responsesRequest(turn), headers))); +var chat = (turn, base = MLFLOW, headers = {}) => collect(chatCompletionsReplies(send(`${base}/chat/completions`, chatCompletionsRequest(turn), headers))); +var CLAUDE = env("UG_LIVE_CLAUDE_MODEL"); +var GPT = env("UG_LIVE_GPT_MODEL"); +var CHAT_GPT = env("UG_LIVE_CHAT_MODEL"); +var MPS = env("UG_LIVE_MPS"); +var MPS_MODEL = env("UG_LIVE_MPS_MODEL"); +var OPENAI_MPS = env("UG_LIVE_OPENAI_MPS"); +var OPENAI_MPS_MODEL = env("UG_LIVE_OPENAI_MPS_MODEL"); +(0, import_node_test.describe)("the workspace's own models", () => { + if (CLAUDE) { + (0, import_node_test.it)("streams a Claude answer on Anthropic Messages, and reads the prompt cache on a repeat", async () => { + const mark = marker(); + const system = text(`Answer in as few words as possible. ${"Ignore this sentence. ".repeat(400)}`); + const turn = turnOf({ model: CLAUDE, messages: [message(SYSTEM, system), message(USER, text(`Reply with exactly: ${mark}`))] }); + import_strict2.default.match(textOf2(await anthropic(turn, 64)), new RegExp(mark)); + const events = await collect(send(MESSAGES, anthropicMessagesRequest(turn, 64), ANTHROPIC_VERSION)); + const start = events.find((event) => event.type === "message_start"); + import_strict2.default.ok((start?.message?.usage?.cache_read_input_tokens ?? 0) > 0, JSON.stringify(start?.message?.usage)); + }); + (0, import_node_test.it)("gets a Claude tool call, and an image inside its result reaches the model", async () => { + const forced = turnOf({ + model: CLAUDE, + tools: [viewImageTool], + toolRequired: true, + messages: [message(USER, text("Look at a.png."))] + }); + const [only, ...rest] = callsOf(await anthropic(forced, 256)); + import_strict2.default.equal(rest.length, 0); + import_strict2.default.equal(only?.name, "view_image"); + import_strict2.default.equal(typeof only?.input, "object"); + import_strict2.default.match(textOf2(await anthropic(imageTurn(CLAUDE), 256)).toLowerCase(), /red/); + }); + (0, import_node_test.it)("retries with the output limit the gateway allows instead of failing the turn", async () => { + import_strict2.default.ok(textOf2(await anthropic(echoTurn(CLAUDE, marker()), 999999)).length > 0); + }); + } + if (GPT) { + (0, import_node_test.it)("streams a GPT answer and a tool call on Responses", async () => { + const mark = marker(); + import_strict2.default.match(textOf2(await responses(echoTurn(GPT, mark))), new RegExp(mark)); + const forced = turnOf({ model: GPT, tools: [viewImageTool], toolRequired: true, messages: [message(USER, text("Look at a.png."))] }); + const [only] = callsOf(await responses(forced)); + import_strict2.default.equal(only?.name, "view_image"); + }); + (0, import_node_test.it)("sends a tool result's image in the request Responses accepts", async () => { + import_strict2.default.match(textOf2(await responses(imageTurn(GPT))).toLowerCase(), /red/); + }); + } + if (CHAT_GPT) { + (0, import_node_test.it)("streams an answer on Chat Completions", async () => { + const mark = marker(); + import_strict2.default.match(textOf2(await chat(echoTurn(CHAT_GPT, mark))), new RegExp(mark)); + }); + } +}); +(0, import_node_test.describe)("a Model Provider Service", () => { + if (MPS && MPS_MODEL) { + const header = { [PROVIDER_HEADER]: MPS }; + (0, import_node_test.it)("serves its Claude models on Anthropic Messages, with tool calls", async () => { + const mark = marker(); + import_strict2.default.match(textOf2(await anthropic(echoTurn(MPS_MODEL, mark), 256, header)), new RegExp(mark)); + const forced = turnOf({ model: MPS_MODEL, tools: [viewImageTool], toolRequired: true, messages: [message(USER, text("Look at a.png."))] }); + const [only] = callsOf(await anthropic(forced, 256, header)); + import_strict2.default.equal(only?.name, "view_image"); + }); + (0, import_node_test.it)("needs the routing header, and isn't served on the mlflow route", async () => { + const turn = echoTurn(MPS_MODEL, marker()); + await rejectsWith(anthropic(turn, 64), "AI Gateway returned"); + await rejectsWith(chat(turn, MLFLOW, header), "AI Gateway returned"); + }); + } + if (OPENAI_MPS && OPENAI_MPS_MODEL) { + const header = { [PROVIDER_HEADER]: OPENAI_MPS }; + (0, import_node_test.it)("serves its OpenAI models on the gateway's OpenAI route", async () => { + const mark = marker(); + import_strict2.default.match(textOf2(await responses(echoTurn(OPENAI_MPS_MODEL, mark), OPENAI, header)), new RegExp(mark)); + }); + (0, import_node_test.it)("isn't served on the mlflow route", async () => { + await rejectsWith(responses(echoTurn(OPENAI_MPS_MODEL, marker()), MLFLOW, header), "AI Gateway returned"); + }); + } +}); diff --git a/vscode-extension/README.md b/vscode-extension/README.md index 8abee96ef..daead9241 100644 --- a/vscode-extension/README.md +++ b/vscode-extension/README.md @@ -21,3 +21,20 @@ Token fetches and failed requests are logged to the **Unity Gateway** output cha `npm ci`, then `npm run typecheck`, `npm test` and `npm run build`. The build writes the bundle and `package.json` to `../src/ucode/vscode/extension/`, which ug packs from; commit them (`just vscode-build`). + +`npm run live` sends the requests this extension builds to a real AI Gateway and checks the replies, +covering what a mocked test can't: the gateway accepting each route's request shape, tool calls, +images inside tool results, prompt caching, and that a Model Provider Service's models need its +routing header. It takes the workspace and models from the environment: + +``` +UG_LIVE_WORKSPACE=https:// \ +UG_LIVE_AUTH_COMMAND='ug auth-token --host https://' \ +UG_LIVE_CLAUDE_MODEL=system.ai.claude-haiku-4-5 \ +UG_LIVE_GPT_MODEL=system.ai.gpt-5-nano \ +npm run live +``` + +`UG_LIVE_TOKEN` works instead of `UG_LIVE_AUTH_COMMAND`. Setting `UG_LIVE_MPS` and `UG_LIVE_MPS_MODEL` +(and `UG_LIVE_OPENAI_MPS` / `UG_LIVE_OPENAI_MPS_MODEL`) adds the Model Provider Service cases; a case +whose variables are unset isn't registered. The models cost a few cents per run. diff --git a/vscode-extension/package.json b/vscode-extension/package.json index dc5d88801..85e177dc9 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -33,7 +33,8 @@ "scripts": { "build": "esbuild src/extension.ts --bundle --minify --legal-comments=none --outfile=../src/ucode/vscode/extension/out/extension.js --external:vscode --format=cjs --platform=node --target=node22 && node -e \"const m = require('./package.json'); delete m.scripts; delete m.devDependencies; require('fs').writeFileSync('../src/ucode/vscode/extension/package.json', JSON.stringify(m, null, 2) + '\\n')\"", "typecheck": "tsc", - "test": "esbuild test/*.test.ts --bundle --platform=node --format=cjs --target=node22 --outdir=.test-out --alias:vscode=./test/vscode.ts --log-level=warning && node --test .test-out/*.test.js" + "test": "esbuild test/*.test.ts --bundle --platform=node --format=cjs --target=node22 --outdir=.test-out --alias:vscode=./test/vscode.ts --log-level=warning && node --test .test-out/*.test.js", + "live": "esbuild test/live/requests.live.ts --bundle --platform=node --format=cjs --target=node22 --outdir=.live-out --alias:vscode=./test/vscode.ts --log-level=warning && node --test .live-out/requests.live.js" }, "devDependencies": { "@types/node": "^22.20.3", diff --git a/vscode-extension/test/live/requests.live.ts b/vscode-extension/test/live/requests.live.ts new file mode 100644 index 000000000..bf5ec9cab --- /dev/null +++ b/vscode-extension/test/live/requests.live.ts @@ -0,0 +1,197 @@ +// Sends the requests the extension builds to a real AI Gateway and checks the replies it gets back. +// Run with `npm run live`; the offline suite globs `test/*.test.ts` and never collects this. +// +// Needs UG_LIVE_WORKSPACE and either UG_LIVE_AUTH_COMMAND or UG_LIVE_TOKEN. The models and Model +// Provider Services come from the UG_LIVE_* variables below; a case whose variables are unset isn't +// registered, so the CI job decides what gets covered. +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { Tokens } from "../../src/auth"; +import { + anthropicMessagesReplies, + anthropicMessagesRequest, + chatCompletionsReplies, + chatCompletionsRequest, + responsesReplies, + responsesRequest, + type Reply, + type Turn, +} from "../../src/formats"; +import { PROVIDER_HEADER } from "../../src/extension"; +import { streamEvents } from "../../src/gateway"; +import { ASSISTANT, call, collect, message, rejectsWith, result, SYSTEM, text, USER } from "../support"; +import { LanguageModelDataPart, LanguageModelTextPart, LanguageModelToolCallPart } from "../vscode"; + +const env = (name: string): string => process.env[name] ?? ""; + +const WORKSPACE = env("UG_LIVE_WORKSPACE").replace(/\/+$/, ""); +const AUTH_COMMAND = env("UG_LIVE_AUTH_COMMAND") || (env("UG_LIVE_TOKEN") && `printf %s "$UG_LIVE_TOKEN"`); +if (!WORKSPACE || !AUTH_COMMAND) { + throw new Error("Set UG_LIVE_WORKSPACE and UG_LIVE_AUTH_COMMAND (or UG_LIVE_TOKEN) to run the live tests."); +} + +const MLFLOW = "/ai-gateway/mlflow/v1"; +const OPENAI = "/ai-gateway/openai/v1"; +const MESSAGES = "/ai-gateway/anthropic/v1/messages"; +const ANTHROPIC_VERSION = { "anthropic-version": "2023-06-01" }; + +const tokens = new Tokens(AUTH_COMMAND); + +const send = (path: string, body: object, headers: Record = {}) => + streamEvents(WORKSPACE, path, body, tokens, { ...headers }, new AbortController().signal); + +const turnOf = (over: Partial & { model: string }): Turn => ({ + messages: [], + tools: [], + toolRequired: false, + ...over, +}); + +const textOf = (replies: Reply[]): string => + replies + .filter((reply): reply is LanguageModelTextPart => reply instanceof LanguageModelTextPart) + .map((part) => part.value) + .join(""); + +const callsOf = (replies: Reply[]): LanguageModelToolCallPart[] => + replies.filter((reply): reply is LanguageModelToolCallPart => reply instanceof LanguageModelToolCallPart); + +// A 16x16 red PNG, so a model asked about the image has something real to look at. +const RED_PNG = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAFklEQVR4nGO4IyJCEmIY1TCqYfhqAAACcQQQFwFJQgAAAABJRU5ErkJggg=="; +const redImage = () => new LanguageModelDataPart(Buffer.from(RED_PNG, "base64"), "image/png"); + +const viewImageTool = { name: "view_image", description: "Show the image at a path", inputSchema: { type: "object", properties: { path: { type: "string" } } } }; + +/** The model must echo this back, so a reply can't pass on stray text alone. */ +const marker = () => `UG${Math.floor(Math.random() * 1e6)}`; + +const echoTurn = (model: string, mark: string): Turn => + turnOf({ + model, + messages: [message(SYSTEM, text("Answer in as few words as possible.")), message(USER, text(`Reply with exactly: ${mark}`))], + }); + +const imageTurn = (model: string): Turn => + turnOf({ + model, + tools: [viewImageTool], + messages: [ + message(USER, text("Look at a.png with the tool, then name its main colour in one lowercase word.")), + message(ASSISTANT, call("c1", "view_image", { path: "a.png" })), + message(USER, result("c1", text("a.png:"), redImage())), + ], + }); + +const anthropic = (turn: Turn, maxTokens: number, headers: Record = {}) => + collect(anthropicMessagesReplies(send(MESSAGES, anthropicMessagesRequest(turn, maxTokens), { ...ANTHROPIC_VERSION, ...headers }))); + +const responses = (turn: Turn, base = MLFLOW, headers: Record = {}) => + collect(responsesReplies(send(`${base}/responses`, responsesRequest(turn), headers))); + +const chat = (turn: Turn, base = MLFLOW, headers: Record = {}) => + collect(chatCompletionsReplies(send(`${base}/chat/completions`, chatCompletionsRequest(turn), headers))); + +const CLAUDE = env("UG_LIVE_CLAUDE_MODEL"); +const GPT = env("UG_LIVE_GPT_MODEL"); +// Unset in CI: every OpenAI-type model there is GPT-5 or newer, which ug routes to Responses. +const CHAT_GPT = env("UG_LIVE_CHAT_MODEL"); +const MPS = env("UG_LIVE_MPS"); +const MPS_MODEL = env("UG_LIVE_MPS_MODEL"); +const OPENAI_MPS = env("UG_LIVE_OPENAI_MPS"); +const OPENAI_MPS_MODEL = env("UG_LIVE_OPENAI_MPS_MODEL"); + +describe("the workspace's own models", () => { + if (CLAUDE) { + it("streams a Claude answer on Anthropic Messages, and reads the prompt cache on a repeat", async () => { + const mark = marker(); + // Long enough that the gateway caches it; the cache point is on the system prompt. + const system = text(`Answer in as few words as possible. ${"Ignore this sentence. ".repeat(400)}`); + const turn = turnOf({ model: CLAUDE, messages: [message(SYSTEM, system), message(USER, text(`Reply with exactly: ${mark}`))] }); + + assert.match(textOf(await anthropic(turn, 64)), new RegExp(mark)); + // The second turn is identical, so its prompt should be served from the cache. + const events = await collect(send(MESSAGES, anthropicMessagesRequest(turn, 64), ANTHROPIC_VERSION)); + const start = events.find((event) => (event as { type?: string }).type === "message_start") as + | { message?: { usage?: Record } } + | undefined; + assert.ok((start?.message?.usage?.cache_read_input_tokens ?? 0) > 0, JSON.stringify(start?.message?.usage)); + }); + + it("gets a Claude tool call, and an image inside its result reaches the model", async () => { + const forced = turnOf({ + model: CLAUDE, + tools: [viewImageTool], + toolRequired: true, + messages: [message(USER, text("Look at a.png."))], + }); + const [only, ...rest] = callsOf(await anthropic(forced, 256)); + assert.equal(rest.length, 0); + assert.equal(only?.name, "view_image"); + assert.equal(typeof only?.input, "object"); + + assert.match(textOf(await anthropic(imageTurn(CLAUDE), 256)).toLowerCase(), /red/); + }); + + it("retries with the output limit the gateway allows instead of failing the turn", async () => { + // No Claude model serves this many output tokens, so the first request is refused. + assert.ok(textOf(await anthropic(echoTurn(CLAUDE, marker()), 999_999)).length > 0); + }); + } + + if (GPT) { + it("streams a GPT answer and a tool call on Responses", async () => { + const mark = marker(); + assert.match(textOf(await responses(echoTurn(GPT, mark))), new RegExp(mark)); + + const forced = turnOf({ model: GPT, tools: [viewImageTool], toolRequired: true, messages: [message(USER, text("Look at a.png."))] }); + const [only] = callsOf(await responses(forced)); + assert.equal(only?.name, "view_image"); + }); + + it("sends a tool result's image in the request Responses accepts", async () => { + assert.match(textOf(await responses(imageTurn(GPT))).toLowerCase(), /red/); + }); + } + + if (CHAT_GPT) { + it("streams an answer on Chat Completions", async () => { + const mark = marker(); + assert.match(textOf(await chat(echoTurn(CHAT_GPT, mark))), new RegExp(mark)); + }); + } +}); + +describe("a Model Provider Service", () => { + if (MPS && MPS_MODEL) { + const header = { [PROVIDER_HEADER]: MPS }; + + it("serves its Claude models on Anthropic Messages, with tool calls", async () => { + const mark = marker(); + assert.match(textOf(await anthropic(echoTurn(MPS_MODEL, mark), 256, header)), new RegExp(mark)); + + const forced = turnOf({ model: MPS_MODEL, tools: [viewImageTool], toolRequired: true, messages: [message(USER, text("Look at a.png."))] }); + const [only] = callsOf(await anthropic(forced, 256, header)); + assert.equal(only?.name, "view_image"); + }); + + it("needs the routing header, and isn't served on the mlflow route", async () => { + const turn = echoTurn(MPS_MODEL, marker()); + // Without this the gateway could be answering from system.ai, and a pass would prove nothing. + await rejectsWith(anthropic(turn, 64), "AI Gateway returned"); + await rejectsWith(chat(turn, MLFLOW, header), "AI Gateway returned"); + }); + } + + if (OPENAI_MPS && OPENAI_MPS_MODEL) { + const header = { [PROVIDER_HEADER]: OPENAI_MPS }; + + it("serves its OpenAI models on the gateway's OpenAI route", async () => { + const mark = marker(); + assert.match(textOf(await responses(echoTurn(OPENAI_MPS_MODEL, mark), OPENAI, header)), new RegExp(mark)); + }); + + it("isn't served on the mlflow route", async () => { + await rejectsWith(responses(echoTurn(OPENAI_MPS_MODEL, marker()), MLFLOW, header), "AI Gateway returned"); + }); + } +}); From b69fa1adf1beec52dadb9f8bc0463d134588a4ec Mon Sep 17 00:00:00 2001 From: Xi Lyu Date: Thu, 29 Oct 2026 21:28:19 +0000 Subject: [PATCH 2/4] Keep the live suite's bundle out of git Co-authored-by: Isaac --- vscode-extension/.gitignore | 1 + vscode-extension/.live-out/requests.live.js | 580 -------------------- 2 files changed, 1 insertion(+), 580 deletions(-) delete mode 100644 vscode-extension/.live-out/requests.live.js diff --git a/vscode-extension/.gitignore b/vscode-extension/.gitignore index 13f124c2b..707d70413 100644 --- a/vscode-extension/.gitignore +++ b/vscode-extension/.gitignore @@ -1,3 +1,4 @@ node_modules/ *.vsix .test-out/ +.live-out/ diff --git a/vscode-extension/.live-out/requests.live.js b/vscode-extension/.live-out/requests.live.js deleted file mode 100644 index c1f109de5..000000000 --- a/vscode-extension/.live-out/requests.live.js +++ /dev/null @@ -1,580 +0,0 @@ -"use strict"; -var __create = Object.create; -var __defProp = Object.defineProperty; -var __getOwnPropDesc = Object.getOwnPropertyDescriptor; -var __getOwnPropNames = Object.getOwnPropertyNames; -var __getProtoOf = Object.getPrototypeOf; -var __hasOwnProp = Object.prototype.hasOwnProperty; -var __copyProps = (to, from, except, desc) => { - if (from && typeof from === "object" || typeof from === "function") { - for (let key of __getOwnPropNames(from)) - if (!__hasOwnProp.call(to, key) && key !== except) - __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); - } - return to; -}; -var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( - // If the importer is in node compatibility mode or this is not an ESM - // file that has been converted to a CommonJS file using a Babel- - // compatible transform (i.e. "__esModule" has not been set), then set - // "default" to the CommonJS "module.exports" for node compatibility. - isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, - mod -)); - -// test/live/requests.live.ts -var import_strict2 = __toESM(require("node:assert/strict")); -var import_node_test = require("node:test"); - -// src/auth.ts -var import_node_child_process = require("node:child_process"); -var EXPIRY_MARGIN_MS = 5 * 6e4; -var DEFAULT_LIFETIME_MS = 60 * 6e4; -var run = (command) => new Promise((resolve, reject) => { - (0, import_node_child_process.exec)(command, { timeout: 6e4, windowsHide: true }, (error, stdout, stderr) => { - if (error) { - reject(new Error(`Couldn't get a Databricks token: \`${command}\` failed: ${String(stderr).trim().slice(0, 500) || error.message}`)); - } else { - resolve(String(stdout).trim()); - } - }); -}); -var expiresAt = (token) => { - try { - const { exp } = JSON.parse(Buffer.from(token.split(".")[1] ?? "", "base64url").toString()); - if (typeof exp === "number") { - return exp * 1e3; - } - } catch { - } - return Date.now() + DEFAULT_LIFETIME_MS; -}; -var Tokens = class { - constructor(command, onFetch = () => { - }) { - this.command = command; - this.onFetch = onFetch; - } - command; - onFetch; - cached; - pending; - get() { - if (this.cached && this.cached.expiresAt - EXPIRY_MARGIN_MS > Date.now()) { - return Promise.resolve(this.cached.value); - } - return this.fetch(this.command); - } - /** A token the gateway rejected. The CLI can hand back its cached copy while `exp` is still ahead, so force a new one. */ - async renew(rejected) { - await this.pending?.catch(() => { - }); - if (this.cached && this.cached.value !== rejected) { - return this.cached.value; - } - this.cached = void 0; - return this.fetch(`${this.command} --force-refresh`); - } - fetch(command) { - this.pending ??= run(command).then((value) => { - if (!value) { - throw new Error(`Couldn't get a Databricks token: \`${command}\` printed nothing.`); - } - this.cached = { value, expiresAt: expiresAt(value) }; - this.onFetch(this.cached.expiresAt); - return value; - }).finally(() => { - this.pending = void 0; - }); - return this.pending; - } -}; - -// test/vscode.ts -var LanguageModelTextPart = class { - constructor(value) { - this.value = value; - } - value; -}; -var LanguageModelToolCallPart = class { - constructor(callId, name, input) { - this.callId = callId; - this.name = name; - this.input = input; - } - callId; - name; - input; -}; -var LanguageModelToolResultPart = class { - constructor(callId, content) { - this.callId = callId; - this.content = content; - } - callId; - content; -}; -var LanguageModelDataPart = class { - constructor(data, mimeType) { - this.data = data; - this.mimeType = mimeType; - } - data; - mimeType; -}; -var LanguageModelPromptTsxPart = class { - constructor(value) { - this.value = value; - } - value; -}; -var LanguageModelChatMessageRole = { User: 1, Assistant: 2 }; - -// src/formats.ts -var SYSTEM_ROLE = 3; -var textOf = (part) => { - if (part instanceof LanguageModelTextPart) { - return part.value; - } - if (part instanceof LanguageModelDataPart && !part.mimeType.startsWith("image/")) { - return new TextDecoder().decode(part.data); - } - if (part instanceof LanguageModelPromptTsxPart) { - return JSON.stringify(part.value); - } - return ""; -}; -var imageData = (part) => part instanceof LanguageModelDataPart && part.mimeType.startsWith("image/") ? { mimeType: part.mimeType, data: Buffer.from(part.data).toString("base64") } : void 0; -var imageUrl = (part) => { - const image = imageData(part); - return image && `data:${image.mimeType};base64,${image.data}`; -}; -var parametersOf = (tool) => tool.inputSchema ?? { type: "object", properties: {} }; -var toolCall = (callId, name, args) => { - let input; - try { - input = args.trim() ? JSON.parse(args) : {}; - } catch { - input = void 0; - } - if (typeof input !== "object" || input === null || Array.isArray(input)) { - throw new Error(`The model called ${name} with arguments that aren't a JSON object: ${args}`); - } - return new LanguageModelToolCallPart(callId, name, input); -}; -var stoppedEarly = (reason) => reason === "max_output_tokens" || reason === "length" || reason === "max_tokens" ? new Error("The model stopped at its output token limit before finishing the response.") : reason === "content_filter" || reason === "refusal" ? new Error("The model's response was stopped by a content filter.") : new Error(`The model stopped before finishing the response (${reason ?? "no reason given"}).`); -var cutOff = () => new Error("The response ended before the model finished. Try again."); -var toolOptions = (turn, toTool) => turn.tools.length ? { tools: turn.tools.map(toTool), ...turn.toolRequired ? { tool_choice: "required" } : {} } : {}; -var imagesOf = (parts) => parts.map(imageUrl).filter((url) => url !== void 0); -var sortParts = (content) => ({ - results: content.filter((part) => part instanceof LanguageModelToolResultPart), - calls: content.filter((part) => part instanceof LanguageModelToolCallPart), - text: content.map(textOf).join(""), - images: imagesOf(content) -}); -function chatCompletionsRequest(turn) { - const messages = turn.messages.flatMap((message2) => { - const { results, calls, text: text2, images } = sortParts(message2.content); - if (Number(message2.role) === SYSTEM_ROLE) { - return [{ role: "system", content: text2 }]; - } - if (message2.role === LanguageModelChatMessageRole.Assistant) { - const toolCalls = calls.map((call2) => ({ - id: call2.callId, - type: "function", - function: { name: call2.name, arguments: JSON.stringify(call2.input) } - })); - return text2 || toolCalls.length ? [{ role: "assistant", content: text2 || null, ...toolCalls.length ? { tool_calls: toolCalls } : {} }] : []; - } - const content = [ - ...text2 ? [{ type: "text", text: text2 }] : [], - ...[...results.flatMap((result2) => imagesOf(result2.content)), ...images].map((url) => ({ type: "image_url", image_url: { url } })) - ]; - return [ - ...results.map((result2) => ({ role: "tool", tool_call_id: result2.callId, content: result2.content.map(textOf).join("") })), - ...content.length ? [{ role: "user", content }] : [] - ]; - }); - return { - model: turn.model, - messages, - stream: true, - ...toolOptions(turn, (tool) => ({ type: "function", function: { name: tool.name, description: tool.description, parameters: parametersOf(tool) } })) - }; -} -var deltaText = (content) => typeof content === "string" ? content : (content ?? []).filter((block) => block.type === "text").map((block) => block.text ?? "").join(""); -var CHAT_FINISHED = /* @__PURE__ */ new Set(["stop", "tool_calls", "function_call"]); -async function* chatCompletionsReplies(events) { - const calls = []; - let finish; - for await (const event of events) { - const { error } = event; - if (error) { - throw new Error(typeof error === "string" ? error : error.message ?? JSON.stringify(error)); - } - for (const choice of event.choices ?? []) { - const text2 = deltaText(choice.delta?.content); - if (text2) { - yield new LanguageModelTextPart(text2); - } - for (const fragment of choice.delta?.tool_calls ?? []) { - const call2 = calls[fragment.index ?? 0] ??= { id: "", name: "", args: "" }; - call2.id = fragment.id || call2.id; - call2.name = fragment.function?.name || call2.name; - call2.args += fragment.function?.arguments ?? ""; - } - if (choice.finish_reason) { - finish = choice.finish_reason; - if (!CHAT_FINISHED.has(finish)) { - throw stoppedEarly(finish); - } - } - } - } - if (!finish) { - throw cutOff(); - } - for (const call2 of calls.filter(Boolean)) { - yield toolCall(call2.id, call2.name, call2.args); - } -} -var functionOutput = (content) => { - const text2 = content.map(textOf).join(""); - const images = imagesOf(content); - return images.length ? [...text2 ? [{ type: "input_text", text: text2 }] : [], ...images.map((url) => ({ type: "input_image", image_url: url, detail: "auto" }))] : text2; -}; -function responsesRequest(turn) { - const input = turn.messages.flatMap((message2) => { - const { results, calls, text: text2, images } = sortParts(message2.content); - if (Number(message2.role) === SYSTEM_ROLE) { - return [{ role: "system", content: text2 }]; - } - if (message2.role === LanguageModelChatMessageRole.Assistant) { - return [ - ...text2 ? [{ role: "assistant", content: text2 }] : [], - ...calls.map((call2) => ({ type: "function_call", call_id: call2.callId, name: call2.name, arguments: JSON.stringify(call2.input) })) - ]; - } - const content = [ - ...text2 ? [{ type: "input_text", text: text2 }] : [], - ...images.map((url) => ({ type: "input_image", image_url: url, detail: "auto" })) - ]; - return [ - ...results.map((result2) => ({ type: "function_call_output", call_id: result2.callId, output: functionOutput(result2.content) })), - ...content.length ? [{ role: "user", content }] : [] - ]; - }); - return { - model: turn.model, - input, - stream: true, - store: false, - ...toolOptions(turn, (tool) => ({ type: "function", name: tool.name, description: tool.description, parameters: parametersOf(tool), strict: false })) - }; -} -async function* responsesReplies(events) { - const calls = []; - let completed = false; - for await (const raw of events) { - const event = raw; - if (event.type === "response.output_text.delta" && event.delta) { - yield new LanguageModelTextPart(event.delta); - } else if (event.type === "response.output_item.done" && event.item?.type === "function_call") { - calls.push(toolCall(event.item.call_id ?? "", event.item.name ?? "", event.item.arguments ?? "")); - } else if (event.type === "response.completed") { - completed = true; - } else if (event.type === "response.incomplete") { - throw stoppedEarly(event.response?.incomplete_details?.reason); - } else if (event.type === "response.failed" || event.type === "error") { - throw new Error(event.response?.error?.message ?? event.message ?? "The model response failed."); - } - } - if (!completed) { - throw cutOff(); - } - yield* calls; -} -var textBlocks = (text2) => text2 ? [{ type: "text", text: text2 }] : []; -var imageBlocks = (parts) => parts.flatMap((part) => { - const image = imageData(part); - return image ? [{ type: "image", source: { type: "base64", media_type: image.mimeType, data: image.data } }] : []; -}); -var CACHE_POINT = { cache_control: { type: "ephemeral" } }; -var cacheUpTo = (blocks) => { - if (blocks.length) { - blocks[blocks.length - 1] = { ...blocks[blocks.length - 1], ...CACHE_POINT }; - } -}; -function anthropicMessagesRequest(turn, maxTokens) { - const system = []; - const messages = []; - for (const message2 of turn.messages) { - const { results, calls, text: text2 } = sortParts(message2.content); - if (Number(message2.role) === SYSTEM_ROLE) { - system.push(...textBlocks(text2)); - continue; - } - const role = message2.role === LanguageModelChatMessageRole.Assistant ? "assistant" : "user"; - const content = role === "assistant" ? [...textBlocks(text2), ...calls.map((call2) => ({ type: "tool_use", id: call2.callId, name: call2.name, input: call2.input }))] : [ - // Tool results have to come first in the user message that follows the calls. - ...results.map((result2) => { - const blocks = [...textBlocks(result2.content.map(textOf).join("")), ...imageBlocks(result2.content)]; - return { type: "tool_result", tool_use_id: result2.callId, ...blocks.length ? { content: blocks } : {} }; - }), - ...textBlocks(text2), - ...imageBlocks(message2.content) - ]; - const last = messages[messages.length - 1]; - if (!content.length) { - continue; - } else if (last?.role === role) { - last.content.push(...content); - } else { - messages.push({ role, content }); - } - } - cacheUpTo(system); - cacheUpTo(messages[messages.length - 1]?.content ?? []); - return { - model: turn.model, - max_tokens: maxTokens, - ...system.length ? { system } : {}, - messages, - stream: true, - ...turn.tools.length ? { - tools: turn.tools.map((tool) => ({ name: tool.name, description: tool.description, input_schema: parametersOf(tool) })), - ...turn.toolRequired ? { tool_choice: { type: "any" } } : {} - } : {} - }; -} -var ANTHROPIC_FINISHED = /* @__PURE__ */ new Set(["end_turn", "tool_use", "stop_sequence"]); -async function* anthropicMessagesReplies(events) { - const calls = /* @__PURE__ */ new Map(); - let stopped = false; - for await (const raw of events) { - const event = raw; - if (event.type === "content_block_start" && event.content_block?.type === "tool_use") { - calls.set(event.index ?? 0, { id: event.content_block.id ?? "", name: event.content_block.name ?? "", args: "" }); - } else if (event.type === "content_block_delta" && event.delta?.type === "text_delta" && event.delta.text) { - yield new LanguageModelTextPart(event.delta.text); - } else if (event.type === "content_block_delta" && event.delta?.type === "input_json_delta") { - const call2 = calls.get(event.index ?? 0); - if (call2) { - call2.args += event.delta.partial_json ?? ""; - } - } else if (event.type === "message_delta" && event.delta?.stop_reason && !ANTHROPIC_FINISHED.has(event.delta.stop_reason)) { - throw stoppedEarly(event.delta.stop_reason); - } else if (event.type === "message_stop") { - stopped = true; - } else if (event.type === "error") { - throw new Error(event.error?.message ?? "The model response failed."); - } - } - if (!stopped) { - throw cutOff(); - } - for (const call2 of calls.values()) { - yield toolCall(call2.id, call2.name, call2.args); - } -} - -// src/gateway.ts -async function* streamEvents(workspace, path, body, tokens2, headers, signal) { - const post = (token2, payload) => fetch(`${workspace}${path}`, { - method: "POST", - headers: { ...headers, Authorization: `Bearer ${token2}`, "Content-Type": "application/json" }, - body: JSON.stringify(payload), - signal - }); - let token = await tokens2.get(); - let response = await post(token, body); - if (response.status === 401) { - await response.body?.cancel(); - token = await tokens2.renew(token); - response = await post(token, body); - } - if (response.status === 401) { - await response.body?.cancel(); - throw new Error("Databricks rejected the sign-in. Run `ug configure --agents vscode` to sign in again, then retry."); - } - if (!response.ok || !response.body) { - const detail = (await response.text()).slice(0, 300); - const allowed = Number(/max_tokens: \d+ > (\d+)/.exec(detail)?.[1]); - if (!allowed) { - throw new Error(`AI Gateway returned ${response.status}: ${detail}`); - } - response = await post(token, { ...body, max_tokens: allowed }); - if (!response.ok || !response.body) { - throw new Error(`AI Gateway returned ${response.status}: ${(await response.text()).slice(0, 300)}`); - } - } - const decoder = new TextDecoder(); - let buffer = ""; - for await (const chunk of response.body) { - buffer += decoder.decode(chunk, { stream: true }); - const lines = buffer.split("\n"); - buffer = lines.pop() ?? ""; - for (const line of lines) { - const data = dataOf(line); - if (data === "[DONE]") { - return; - } - if (data) { - yield JSON.parse(data); - } - } - } - const last = dataOf(buffer + decoder.decode()); - if (last && last !== "[DONE]") { - yield JSON.parse(last); - } -} -var dataOf = (line) => line.startsWith("data:") ? line.slice(5).trim() : ""; - -// src/extension.ts -var PROVIDER_HEADER = "Databricks-Model-Provider-Service"; - -// test/support.ts -var import_strict = __toESM(require("node:assert/strict")); -var SYSTEM = 3; -var USER = 1; -var ASSISTANT = 2; -var message = (role, ...content) => ({ role, content, name: void 0 }); -var text = (value) => new LanguageModelTextPart(value); -var call = (id, name, input) => new LanguageModelToolCallPart(id, name, input); -var result = (id, ...content) => new LanguageModelToolResultPart(id, content); -var collect = async (items) => { - const out = []; - for await (const item of items) { - out.push(item); - } - return out; -}; -var rejectsWith = (promise, text2) => import_strict.default.rejects(promise, (error) => { - import_strict.default.ok(error.message.includes(text2), `expected "${error.message}" to contain "${text2}"`); - return true; -}); - -// test/live/requests.live.ts -var env = (name) => process.env[name] ?? ""; -var WORKSPACE = env("UG_LIVE_WORKSPACE").replace(/\/+$/, ""); -var AUTH_COMMAND = env("UG_LIVE_AUTH_COMMAND") || env("UG_LIVE_TOKEN") && `printf %s "$UG_LIVE_TOKEN"`; -if (!WORKSPACE || !AUTH_COMMAND) { - throw new Error("Set UG_LIVE_WORKSPACE and UG_LIVE_AUTH_COMMAND (or UG_LIVE_TOKEN) to run the live tests."); -} -var MLFLOW = "/ai-gateway/mlflow/v1"; -var OPENAI = "/ai-gateway/openai/v1"; -var MESSAGES = "/ai-gateway/anthropic/v1/messages"; -var ANTHROPIC_VERSION = { "anthropic-version": "2023-06-01" }; -var tokens = new Tokens(AUTH_COMMAND); -var send = (path, body, headers = {}) => streamEvents(WORKSPACE, path, body, tokens, { ...headers }, new AbortController().signal); -var turnOf = (over) => ({ - messages: [], - tools: [], - toolRequired: false, - ...over -}); -var textOf2 = (replies) => replies.filter((reply) => reply instanceof LanguageModelTextPart).map((part) => part.value).join(""); -var callsOf = (replies) => replies.filter((reply) => reply instanceof LanguageModelToolCallPart); -var RED_PNG = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAFklEQVR4nGO4IyJCEmIY1TCqYfhqAAACcQQQFwFJQgAAAABJRU5ErkJggg=="; -var redImage = () => new LanguageModelDataPart(Buffer.from(RED_PNG, "base64"), "image/png"); -var viewImageTool = { name: "view_image", description: "Show the image at a path", inputSchema: { type: "object", properties: { path: { type: "string" } } } }; -var marker = () => `UG${Math.floor(Math.random() * 1e6)}`; -var echoTurn = (model, mark) => turnOf({ - model, - messages: [message(SYSTEM, text("Answer in as few words as possible.")), message(USER, text(`Reply with exactly: ${mark}`))] -}); -var imageTurn = (model) => turnOf({ - model, - tools: [viewImageTool], - messages: [ - message(USER, text("Look at a.png with the tool, then name its main colour in one lowercase word.")), - message(ASSISTANT, call("c1", "view_image", { path: "a.png" })), - message(USER, result("c1", text("a.png:"), redImage())) - ] -}); -var anthropic = (turn, maxTokens, headers = {}) => collect(anthropicMessagesReplies(send(MESSAGES, anthropicMessagesRequest(turn, maxTokens), { ...ANTHROPIC_VERSION, ...headers }))); -var responses = (turn, base = MLFLOW, headers = {}) => collect(responsesReplies(send(`${base}/responses`, responsesRequest(turn), headers))); -var chat = (turn, base = MLFLOW, headers = {}) => collect(chatCompletionsReplies(send(`${base}/chat/completions`, chatCompletionsRequest(turn), headers))); -var CLAUDE = env("UG_LIVE_CLAUDE_MODEL"); -var GPT = env("UG_LIVE_GPT_MODEL"); -var CHAT_GPT = env("UG_LIVE_CHAT_MODEL"); -var MPS = env("UG_LIVE_MPS"); -var MPS_MODEL = env("UG_LIVE_MPS_MODEL"); -var OPENAI_MPS = env("UG_LIVE_OPENAI_MPS"); -var OPENAI_MPS_MODEL = env("UG_LIVE_OPENAI_MPS_MODEL"); -(0, import_node_test.describe)("the workspace's own models", () => { - if (CLAUDE) { - (0, import_node_test.it)("streams a Claude answer on Anthropic Messages, and reads the prompt cache on a repeat", async () => { - const mark = marker(); - const system = text(`Answer in as few words as possible. ${"Ignore this sentence. ".repeat(400)}`); - const turn = turnOf({ model: CLAUDE, messages: [message(SYSTEM, system), message(USER, text(`Reply with exactly: ${mark}`))] }); - import_strict2.default.match(textOf2(await anthropic(turn, 64)), new RegExp(mark)); - const events = await collect(send(MESSAGES, anthropicMessagesRequest(turn, 64), ANTHROPIC_VERSION)); - const start = events.find((event) => event.type === "message_start"); - import_strict2.default.ok((start?.message?.usage?.cache_read_input_tokens ?? 0) > 0, JSON.stringify(start?.message?.usage)); - }); - (0, import_node_test.it)("gets a Claude tool call, and an image inside its result reaches the model", async () => { - const forced = turnOf({ - model: CLAUDE, - tools: [viewImageTool], - toolRequired: true, - messages: [message(USER, text("Look at a.png."))] - }); - const [only, ...rest] = callsOf(await anthropic(forced, 256)); - import_strict2.default.equal(rest.length, 0); - import_strict2.default.equal(only?.name, "view_image"); - import_strict2.default.equal(typeof only?.input, "object"); - import_strict2.default.match(textOf2(await anthropic(imageTurn(CLAUDE), 256)).toLowerCase(), /red/); - }); - (0, import_node_test.it)("retries with the output limit the gateway allows instead of failing the turn", async () => { - import_strict2.default.ok(textOf2(await anthropic(echoTurn(CLAUDE, marker()), 999999)).length > 0); - }); - } - if (GPT) { - (0, import_node_test.it)("streams a GPT answer and a tool call on Responses", async () => { - const mark = marker(); - import_strict2.default.match(textOf2(await responses(echoTurn(GPT, mark))), new RegExp(mark)); - const forced = turnOf({ model: GPT, tools: [viewImageTool], toolRequired: true, messages: [message(USER, text("Look at a.png."))] }); - const [only] = callsOf(await responses(forced)); - import_strict2.default.equal(only?.name, "view_image"); - }); - (0, import_node_test.it)("sends a tool result's image in the request Responses accepts", async () => { - import_strict2.default.match(textOf2(await responses(imageTurn(GPT))).toLowerCase(), /red/); - }); - } - if (CHAT_GPT) { - (0, import_node_test.it)("streams an answer on Chat Completions", async () => { - const mark = marker(); - import_strict2.default.match(textOf2(await chat(echoTurn(CHAT_GPT, mark))), new RegExp(mark)); - }); - } -}); -(0, import_node_test.describe)("a Model Provider Service", () => { - if (MPS && MPS_MODEL) { - const header = { [PROVIDER_HEADER]: MPS }; - (0, import_node_test.it)("serves its Claude models on Anthropic Messages, with tool calls", async () => { - const mark = marker(); - import_strict2.default.match(textOf2(await anthropic(echoTurn(MPS_MODEL, mark), 256, header)), new RegExp(mark)); - const forced = turnOf({ model: MPS_MODEL, tools: [viewImageTool], toolRequired: true, messages: [message(USER, text("Look at a.png."))] }); - const [only] = callsOf(await anthropic(forced, 256, header)); - import_strict2.default.equal(only?.name, "view_image"); - }); - (0, import_node_test.it)("needs the routing header, and isn't served on the mlflow route", async () => { - const turn = echoTurn(MPS_MODEL, marker()); - await rejectsWith(anthropic(turn, 64), "AI Gateway returned"); - await rejectsWith(chat(turn, MLFLOW, header), "AI Gateway returned"); - }); - } - if (OPENAI_MPS && OPENAI_MPS_MODEL) { - const header = { [PROVIDER_HEADER]: OPENAI_MPS }; - (0, import_node_test.it)("serves its OpenAI models on the gateway's OpenAI route", async () => { - const mark = marker(); - import_strict2.default.match(textOf2(await responses(echoTurn(OPENAI_MPS_MODEL, mark), OPENAI, header)), new RegExp(mark)); - }); - (0, import_node_test.it)("isn't served on the mlflow route", async () => { - await rejectsWith(responses(echoTurn(OPENAI_MPS_MODEL, marker()), MLFLOW, header), "AI Gateway returned"); - }); - } -}); From b44bec72fcb14fb16b96a0094aeb19f9c7f370c9 Mon Sep 17 00:00:00 2001 From: Xi Lyu Date: Fri, 30 Oct 2026 11:05:24 +0000 Subject: [PATCH 3/4] Run the live gateway checks in the VS Code extension job They take the workspace and bearer the Python e2e tests take, and skip without them, so a fork's pull request stays green. The step can't fail the job: it calls real models, and provider quota must not turn the required test check red. Co-authored-by: Isaac --- .github/workflows/ci.yml | 52 +++++++-------------- vscode-extension/README.md | 4 +- vscode-extension/test/live/requests.live.ts | 28 ++++++----- 3 files changed, 37 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a497f8332..682f542e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,40 +117,24 @@ jobs: echo "::error::The VS Code extension bundle is stale. Run \`just vscode-build\` and commit the result." exit 1 } - - vscode-extension-live: - name: VS Code extension · live gateway - runs-on: ubuntu-latest - needs: vscode-extension - # Needs the workspace secrets, which pull requests from forks don't get, and the merge queue - # reruns nothing that already ran on the PR. - if: ${{ github.event_name != 'merge_group' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} - timeout-minutes: 10 - defaults: - run: - working-directory: vscode-extension - env: - UG_LIVE_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} - # See the e2e shards: a hosted runner has no cached login, so CI supplies a pre-fetched bearer. - UG_LIVE_TOKEN: ${{ secrets.DATABRICKS_BEARER }} - UG_LIVE_CLAUDE_MODEL: system.ai.claude-haiku-4-5 - UG_LIVE_GPT_MODEL: system.ai.gpt-5-4-nano - # The services and targets the integration CUJs use (scripts/run_integration.py). - UG_LIVE_MPS: main.ucode.ci_e2e_anthropic_nonrelay_mps - UG_LIVE_MPS_MODEL: claude-haiku-4-5-20251001 - UG_LIVE_OPENAI_MPS: main.ucode.ci_openai_mps - UG_LIVE_OPENAI_MPS_MODEL: gpt-5-nano - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 - with: - node-version: 22 - cache: npm - cache-dependency-path: vscode-extension/package-lock.json - - run: npm ci --no-audit --no-fund - - run: npm run live + # Last, so the offline checks fail fast before this spends anything on real models. It calls + # the gateway, so it can fail on provider quota or rate limits; that must not turn the required + # `test` check red, and the step is still marked failed in the run. + - name: Check the requests against a real gateway + continue-on-error: true + if: ${{ github.event_name != 'merge_group' }} + env: + UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} + # See the e2e shards: a hosted runner has no cached login, so CI supplies a pre-fetched bearer. + DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }} + UG_LIVE_CLAUDE_MODEL: system.ai.claude-haiku-4-5 + UG_LIVE_GPT_MODEL: system.ai.gpt-5-4-nano + # The services and targets the integration CUJs use (scripts/run_integration.py). + UG_LIVE_MPS: main.ucode.ci_e2e_anthropic_nonrelay_mps + UG_LIVE_MPS_MODEL: claude-haiku-4-5-20251001 + UG_LIVE_OPENAI_MPS: main.ucode.ci_openai_mps + UG_LIVE_OPENAI_MPS_MODEL: gpt-5-nano + run: npm run live e2e-shards: name: ${{ matrix.name }} · ${{ matrix.os == 'windows-latest' && 'Windows' || 'Linux' }} diff --git a/vscode-extension/README.md b/vscode-extension/README.md index daead9241..b5b07caa8 100644 --- a/vscode-extension/README.md +++ b/vscode-extension/README.md @@ -35,6 +35,8 @@ UG_LIVE_GPT_MODEL=system.ai.gpt-5-nano \ npm run live ``` -`UG_LIVE_TOKEN` works instead of `UG_LIVE_AUTH_COMMAND`. Setting `UG_LIVE_MPS` and `UG_LIVE_MPS_MODEL` +CI runs it as the last step of the `VS Code extension` job, from `UCODE_TEST_WORKSPACE` and +`DATABRICKS_BEARER`; with neither set the suite skips itself, so a fork's pull request stays green. +`UG_LIVE_WORKSPACE` and `UG_LIVE_TOKEN` override them for a one-off local run. Setting `UG_LIVE_MPS` and `UG_LIVE_MPS_MODEL` (and `UG_LIVE_OPENAI_MPS` / `UG_LIVE_OPENAI_MPS_MODEL`) adds the Model Provider Service cases; a case whose variables are unset isn't registered. The models cost a few cents per run. diff --git a/vscode-extension/test/live/requests.live.ts b/vscode-extension/test/live/requests.live.ts index bf5ec9cab..4d6ac1de9 100644 --- a/vscode-extension/test/live/requests.live.ts +++ b/vscode-extension/test/live/requests.live.ts @@ -1,8 +1,9 @@ // Sends the requests the extension builds to a real AI Gateway and checks the replies it gets back. // Run with `npm run live`; the offline suite globs `test/*.test.ts` and never collects this. // -// Needs UG_LIVE_WORKSPACE and either UG_LIVE_AUTH_COMMAND or UG_LIVE_TOKEN. The models and Model -// Provider Services come from the UG_LIVE_* variables below; a case whose variables are unset isn't +// Needs UCODE_TEST_WORKSPACE and DATABRICKS_BEARER, as the Python e2e tests do; without them the +// suite skips rather than fails, so a fork's pull request stays green. The models and Model Provider +// Services come from the UG_LIVE_* variables below; a case whose variables are unset isn't // registered, so the CI job decides what gets covered. import assert from "node:assert/strict"; import { describe, it } from "node:test"; @@ -24,11 +25,10 @@ import { LanguageModelDataPart, LanguageModelTextPart, LanguageModelToolCallPart const env = (name: string): string => process.env[name] ?? ""; -const WORKSPACE = env("UG_LIVE_WORKSPACE").replace(/\/+$/, ""); -const AUTH_COMMAND = env("UG_LIVE_AUTH_COMMAND") || (env("UG_LIVE_TOKEN") && `printf %s "$UG_LIVE_TOKEN"`); -if (!WORKSPACE || !AUTH_COMMAND) { - throw new Error("Set UG_LIVE_WORKSPACE and UG_LIVE_AUTH_COMMAND (or UG_LIVE_TOKEN) to run the live tests."); -} +// The same variables the Python e2e tests take, with UG_LIVE_* overrides for running one by hand. +const WORKSPACE = (env("UG_LIVE_WORKSPACE") || env("UCODE_TEST_WORKSPACE")).replace(/\/+$/, ""); +const TOKEN_VAR = env("UG_LIVE_TOKEN") ? "UG_LIVE_TOKEN" : "DATABRICKS_BEARER"; +const AUTH_COMMAND = env("UG_LIVE_AUTH_COMMAND") || (env(TOKEN_VAR) && `printf %s "$${TOKEN_VAR}"`); const MLFLOW = "/ai-gateway/mlflow/v1"; const OPENAI = "/ai-gateway/openai/v1"; @@ -100,8 +100,12 @@ const MPS_MODEL = env("UG_LIVE_MPS_MODEL"); const OPENAI_MPS = env("UG_LIVE_OPENAI_MPS"); const OPENAI_MPS_MODEL = env("UG_LIVE_OPENAI_MPS_MODEL"); +if (!WORKSPACE || !AUTH_COMMAND) { + it.skip("set UCODE_TEST_WORKSPACE and DATABRICKS_BEARER to run the live tests", () => {}); +} + describe("the workspace's own models", () => { - if (CLAUDE) { + if (CLAUDE && WORKSPACE && AUTH_COMMAND) { it("streams a Claude answer on Anthropic Messages, and reads the prompt cache on a repeat", async () => { const mark = marker(); // Long enough that the gateway caches it; the cache point is on the system prompt. @@ -138,7 +142,7 @@ describe("the workspace's own models", () => { }); } - if (GPT) { + if (GPT && WORKSPACE && AUTH_COMMAND) { it("streams a GPT answer and a tool call on Responses", async () => { const mark = marker(); assert.match(textOf(await responses(echoTurn(GPT, mark))), new RegExp(mark)); @@ -153,7 +157,7 @@ describe("the workspace's own models", () => { }); } - if (CHAT_GPT) { + if (CHAT_GPT && WORKSPACE && AUTH_COMMAND) { it("streams an answer on Chat Completions", async () => { const mark = marker(); assert.match(textOf(await chat(echoTurn(CHAT_GPT, mark))), new RegExp(mark)); @@ -162,7 +166,7 @@ describe("the workspace's own models", () => { }); describe("a Model Provider Service", () => { - if (MPS && MPS_MODEL) { + if (MPS && MPS_MODEL && WORKSPACE && AUTH_COMMAND) { const header = { [PROVIDER_HEADER]: MPS }; it("serves its Claude models on Anthropic Messages, with tool calls", async () => { @@ -182,7 +186,7 @@ describe("a Model Provider Service", () => { }); } - if (OPENAI_MPS && OPENAI_MPS_MODEL) { + if (OPENAI_MPS && OPENAI_MPS_MODEL && WORKSPACE && AUTH_COMMAND) { const header = { [PROVIDER_HEADER]: OPENAI_MPS }; it("serves its OpenAI models on the gateway's OpenAI route", async () => { From 10db270916698d23839a4770a6b996b5ecd4b2d7 Mon Sep 17 00:00:00 2001 From: Xi Lyu Date: Fri, 30 Oct 2026 11:10:55 +0000 Subject: [PATCH 4/4] Default the live checks' models and services instead of passing them in CI The integration CUJs' models and services are the right default, as scripts/run_integration.py already defaults its provider flags to them, so the CI step passes only the two secrets. Also pad the cached prompt past Anthropic's 2048-token minimum for Haiku, which the old padding sat just under. Co-authored-by: Isaac --- .github/workflows/ci.yml | 7 ------ vscode-extension/README.md | 13 ++++++----- vscode-extension/test/live/requests.live.ts | 25 +++++++++++++-------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 682f542e2..e116cf2cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,13 +127,6 @@ jobs: UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} # See the e2e shards: a hosted runner has no cached login, so CI supplies a pre-fetched bearer. DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }} - UG_LIVE_CLAUDE_MODEL: system.ai.claude-haiku-4-5 - UG_LIVE_GPT_MODEL: system.ai.gpt-5-4-nano - # The services and targets the integration CUJs use (scripts/run_integration.py). - UG_LIVE_MPS: main.ucode.ci_e2e_anthropic_nonrelay_mps - UG_LIVE_MPS_MODEL: claude-haiku-4-5-20251001 - UG_LIVE_OPENAI_MPS: main.ucode.ci_openai_mps - UG_LIVE_OPENAI_MPS_MODEL: gpt-5-nano run: npm run live e2e-shards: diff --git a/vscode-extension/README.md b/vscode-extension/README.md index b5b07caa8..a8de86e00 100644 --- a/vscode-extension/README.md +++ b/vscode-extension/README.md @@ -30,13 +30,16 @@ routing header. It takes the workspace and models from the environment: ``` UG_LIVE_WORKSPACE=https:// \ UG_LIVE_AUTH_COMMAND='ug auth-token --host https://' \ -UG_LIVE_CLAUDE_MODEL=system.ai.claude-haiku-4-5 \ -UG_LIVE_GPT_MODEL=system.ai.gpt-5-nano \ +UG_LIVE_MPS= UG_LIVE_OPENAI_MPS= \ npm run live ``` CI runs it as the last step of the `VS Code extension` job, from `UCODE_TEST_WORKSPACE` and `DATABRICKS_BEARER`; with neither set the suite skips itself, so a fork's pull request stays green. -`UG_LIVE_WORKSPACE` and `UG_LIVE_TOKEN` override them for a one-off local run. Setting `UG_LIVE_MPS` and `UG_LIVE_MPS_MODEL` -(and `UG_LIVE_OPENAI_MPS` / `UG_LIVE_OPENAI_MPS_MODEL`) adds the Model Provider Service cases; a case -whose variables are unset isn't registered. The models cost a few cents per run. +`UG_LIVE_WORKSPACE` and `UG_LIVE_TOKEN` override them for a one-off local run. + +The models and Model Provider Services default to the ones the integration CUJs use, so CI passes +only the two secrets. Any of `UG_LIVE_CLAUDE_MODEL`, `UG_LIVE_GPT_MODEL`, `UG_LIVE_MPS`, +`UG_LIVE_MPS_MODEL`, `UG_LIVE_OPENAI_MPS` and `UG_LIVE_OPENAI_MPS_MODEL` overrides one; setting it to +the empty string leaves that case out, which is what another workspace needs for the services it +doesn't have. Each case needs its model or service to be set; the rest are left out. A run costs a few cents. diff --git a/vscode-extension/test/live/requests.live.ts b/vscode-extension/test/live/requests.live.ts index 4d6ac1de9..0de45076e 100644 --- a/vscode-extension/test/live/requests.live.ts +++ b/vscode-extension/test/live/requests.live.ts @@ -91,14 +91,20 @@ const responses = (turn: Turn, base = MLFLOW, headers: Record = const chat = (turn: Turn, base = MLFLOW, headers: Record = {}) => collect(chatCompletionsReplies(send(`${base}/chat/completions`, chatCompletionsRequest(turn), headers))); -const CLAUDE = env("UG_LIVE_CLAUDE_MODEL"); -const GPT = env("UG_LIVE_GPT_MODEL"); -// Unset in CI: every OpenAI-type model there is GPT-5 or newer, which ug routes to Responses. +// What to exercise, defaulted to the models and services the integration CUJs use (the same values +// scripts/run_integration.py defaults its --claude-provider and --codex-provider flags to). Override +// any of these for another workspace, or set one to "" to leave that case out. +const setting = (name: string, fallback: string): string => (name in process.env ? env(name) : fallback); + +const CLAUDE = setting("UG_LIVE_CLAUDE_MODEL", "system.ai.claude-haiku-4-5"); +const GPT = setting("UG_LIVE_GPT_MODEL", "system.ai.gpt-5-4-nano"); +// No default: every OpenAI-type model on the CI workspace is GPT-5 or newer, which ug routes to +// Responses, so nothing there exercises Chat Completions. const CHAT_GPT = env("UG_LIVE_CHAT_MODEL"); -const MPS = env("UG_LIVE_MPS"); -const MPS_MODEL = env("UG_LIVE_MPS_MODEL"); -const OPENAI_MPS = env("UG_LIVE_OPENAI_MPS"); -const OPENAI_MPS_MODEL = env("UG_LIVE_OPENAI_MPS_MODEL"); +const MPS = setting("UG_LIVE_MPS", "main.ucode.ci_e2e_anthropic_nonrelay_mps"); +const MPS_MODEL = setting("UG_LIVE_MPS_MODEL", "claude-haiku-4-5-20251001"); +const OPENAI_MPS = setting("UG_LIVE_OPENAI_MPS", "main.ucode.ci_openai_mps"); +const OPENAI_MPS_MODEL = setting("UG_LIVE_OPENAI_MPS_MODEL", "gpt-5-nano"); if (!WORKSPACE || !AUTH_COMMAND) { it.skip("set UCODE_TEST_WORKSPACE and DATABRICKS_BEARER to run the live tests", () => {}); @@ -108,8 +114,9 @@ describe("the workspace's own models", () => { if (CLAUDE && WORKSPACE && AUTH_COMMAND) { it("streams a Claude answer on Anthropic Messages, and reads the prompt cache on a repeat", async () => { const mark = marker(); - // Long enough that the gateway caches it; the cache point is on the system prompt. - const system = text(`Answer in as few words as possible. ${"Ignore this sentence. ".repeat(400)}`); + // Anthropic caches a prompt only above a minimum length, 2048 tokens for Haiku models, so this + // has to clear that with room to spare; the cache point itself sits on the system prompt. + const system = text(`Answer in as few words as possible. ${"Ignore this sentence. ".repeat(1500)}`); const turn = turnOf({ model: CLAUDE, messages: [message(SYSTEM, system), message(USER, text(`Reply with exactly: ${mark}`))] }); assert.match(textOf(await anthropic(turn, 64)), new RegExp(mark));