diff --git a/.github/workflows/test-drift.yml b/.github/workflows/test-drift.yml index 66a4fb52..f101e537 100644 --- a/.github/workflows/test-drift.yml +++ b/.github/workflows/test-drift.yml @@ -85,6 +85,11 @@ jobs: # job can classify an exit-5 quarantine distinctly instead of folding it # into the generic "job failed → HTTP drift" fallback (M5 backstop). exit_code: ${{ steps.drift.outputs.exit_code }} + # Probe-model retirement preflight results, for the `notify` job: Slack + # lines for retired / soon-retiring probe models, and whether any is + # already retired (which stopped the run before drift was graded). + model_alert: ${{ steps.model_preflight.outputs.retirement_alert }} + model_retired: ${{ steps.model_preflight.outputs.retired }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: { persist-credentials: false } @@ -180,6 +185,23 @@ jobs: PREFLIGHT npx tsx "$PREFLIGHT_FILE" + # Probe-model RETIREMENT preflight. Asks each provider whether every model + # the drift probes pin (src/__tests__/drift/probe-models.ts) is still + # served. A retired one fails here with "probe model X retired on DATE — + # update " instead of as a confusing drift failure three nights + # running (o4-mini, 2026-10-23). A shutdown date inside 30 days is a + # warning in the job summary and in Slack via the `notify` job. No key, no + # retirement API, or a transient error is an informational skip. + - name: Preflight — probe model retirement + id: model_preflight + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + run: npx tsx scripts/drift-model-preflight.ts + # Provision a local Ollama daemon so the OLLAMA_HOST-gated live drift leg # (src/__tests__/drift/ollama.drift.ts) actually runs. Ollama needs NO API # key — the "credential" is just a running daemon. We pull the smallest @@ -453,6 +475,8 @@ jobs: DRIFT_SUMMARY: ${{ needs.drift.outputs.summary }} DRIFT_RUNS: ${{ needs.drift.outputs.runs }} DRIFT_EXIT_CODE: ${{ needs.drift.outputs.exit_code }} + MODEL_ALERT: ${{ needs.drift.outputs.model_alert }} + MODEL_RETIRED: ${{ needs.drift.outputs.model_retired }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} run: | @@ -463,6 +487,7 @@ jobs: INFRA_ERROR=false QUARANTINE=false LIVE_TIMEOUT=false + PROBE_RETIRED=false # Determine what happened in each job. An exit_code of 5 means the # collector QUARANTINED unparseable output — the drift job concludes @@ -478,7 +503,13 @@ jobs: # would fall through to the good→good "stay quiet" path and nobody would # learn that a surface stopped being checked. Classified before the # generic fallbacks so it is never reported as real drift. - if [ "$DRIFT_EXIT_CODE" = "5" ]; then + # + # A retired probe model stops the drift job at its preflight, before + # anything is graded. The job concludes `failure`, so without this + # branch it would be reported as "providers changed response formats". + if [ "$MODEL_RETIRED" = "true" ]; then + PROBE_RETIRED=true + elif [ "$DRIFT_EXIT_CODE" = "5" ]; then QUARANTINE=true elif [ "$DRIFT_EXIT_CODE" = "6" ]; then LIVE_TIMEOUT=true @@ -522,7 +553,15 @@ jobs: # and needs manual triage. Classified distinctly — NOT real HTTP API # drift — so nobody chases a phantom provider format change. The # DRIFT_SUMMARY already carries the quarantine detail block (C5.2). - if [ "$QUARANTINE" = "true" ]; then + ALERT_SHOWN=false + if [ "$PROBE_RETIRED" = "true" ]; then + EMOJI="🛑" + MSG="*Drift probe model retired* in aimock — the preflight stopped the run before any drift was graded (not API drift). Update the probe model:${NL}${MODEL_ALERT}${NL}${RUN_URL}" + ALERT_SHOWN=true + if [ "$AGUI_DRIFT" = "true" ]; then + MSG="${MSG}${NL}AG-UI schema drift was also detected — canonical ag-ui types changed." + fi + elif [ "$QUARANTINE" = "true" ]; then EMOJI="🔬" MSG="*Drift collector quarantined unparseable output* in aimock — manual triage required (not confirmed API drift).${DETAIL}${NL}${RUN_URL}" # Live surface silent (collector exit 6): a live leg observed nothing at @@ -552,11 +591,22 @@ jobs: elif [ "$PREV" = "failure" ]; then EMOJI="✅" MSG="Drift tests passing again — required checks pass.${DETAIL}" + # Good → good, but a probe model retires soon — warn every day until + # it is updated (a shutdown date inside the preflight's window). + elif [ -n "$MODEL_ALERT" ]; then + EMOJI="⚠️" + MSG="*Drift probe model retiring soon* in aimock — update it before its shutdown date:${NL}${MODEL_ALERT}${NL}${RUN_URL}" + ALERT_SHOWN=true # Good → good — stay quiet else exit 0 fi + # Any other alert still carries a pending probe-model retirement. + if [ -n "$MODEL_ALERT" ] && [ "$ALERT_SHOWN" = "false" ]; then + MSG="${MSG}${NL}${MODEL_ALERT}" + fi + PAYLOAD=$(jq -n --arg text "${EMOJI} ${MSG}" '{text: $text}') curl -sf -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index ca062632..a6712c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Client disconnects on those surfaces are journaled as `interrupted` with `interruptReason: "client aborted"` (HTTP `status: 0` when nothing was sent); aimock-caused teardowns record `"disconnect"`. Journal snapshots of hung-up requests now carry these fields - WebSocket: a client TCP FIN without a close frame now closes the connection at once (1006) and ends the turn - Daily drift: the OpenAI Responses K9 reasoning-exhaustion probes (HTTP and WebSocket) request `gpt-5.4-mini`. OpenAI retired `o4-mini` on 2026-10-23 (its `/v1/models` `shutdown_date`), and it has answered `404 model_not_found` since. The probe model is one constant, `OPENAI_K9_PROBE_MODEL`. The usage check accepts `output_tokens` equal to `output_tokens_details.reasoning_tokens` (gpt-5.4-mini spends the 32-token cap on reasoning; o4-mini reported 0). The event-order check is unchanged, and the retained o4-mini replay fixtures are still graded against o4-mini. Drift harness only; aimock behavior is unchanged (#584). +- Daily drift: a preflight checks every model the live drift probes pin (`src/__tests__/drift/probe-models.ts`) against the provider's own retirement data before the run. A retired model fails the run with `probe model X retired on DATE — update ` instead of looking like API drift; a shutdown date within 30 days is a warning in the job summary and in Slack. The Gemini embeddings canary now probes `gemini-embedding-001`; `text-embedding-004` answers 404. Drift harness only; aimock behavior is unchanged. - Docker image: the base image is pinned to `node:22.23.4-alpine` by digest. Node 22.23.4 fixes CVE-2026-111497 (Brotli decompression memory bug); the 1.46.0 image shipped Node 22.23.3. ## [1.46.0] - 2026-10-20 diff --git a/DRIFT.md b/DRIFT.md index accae150..5a1e4745 100644 --- a/DRIFT.md +++ b/DRIFT.md @@ -105,6 +105,8 @@ When a `critical` drift is detected: **A deprecation needs nothing from you.** The daily sync records it in `deprecatedFamilies` and aimock keeps mocking the family, so clients pinned to a retired model id keep working. The only thing worth doing by hand is the cheap live test model: if `src/__tests__/drift/providers.ts` names a model that no longer exists, the live drift legs cannot run at all, so point them at a current one and re-run `pnpm test:drift`. +**A retired probe model is caught before the run.** Every model a live drift probe pins is listed in `src/__tests__/drift/probe-models.ts`. Before the daily `drift` job runs the suite, `scripts/drift-model-preflight.ts` asks each provider about each one: OpenAI `GET /v1/models/{id}` (`shutdown_date`), Anthropic `GET /v1/models/{id}` (`lifecycle`, `retires_at`), Gemini `GET /v1beta/models/{id}` (404 when gone; no date published), Cohere `GET /v1/models/{id}` (`is_deprecated`) and the OpenRouter listing (`expiration_date`). A model that is gone or past its shutdown date fails the step with `probe model X retired on DATE — update `, and Slack says "probe model retired", not "API drift". A shutdown date within 30 days (`DRIFT_PREFLIGHT_WARN_DAYS`) is a warning in the job summary and in Slack. Bedrock, Vertex, Ollama and fal models are listed but skipped: the step has no retirement API it can ask for them. Legs that pick their model from the live listing (`resolveLiveModel`) are not listed; they move on their own. + ## Adding a New Provider 1. Add the provider's SDK as a devDependency in `package.json` diff --git a/scripts/drift-model-preflight.ts b/scripts/drift-model-preflight.ts new file mode 100644 index 00000000..8642277d --- /dev/null +++ b/scripts/drift-model-preflight.ts @@ -0,0 +1,380 @@ +/// + +/** + * Drift probe-model retirement preflight + * + * Before the nightly drift run, asks each provider whether every model the + * drift probes pin (`src/__tests__/drift/probe-models.ts`) is still served, so + * a retirement is caught weeks ahead and named plainly instead of surfacing as + * a confusing drift failure. + * + * - gone, or past its shutdown date → `::error::` and exit 1 + * - shutdown date within the warning window (default 30 days), or marked + * deprecated → `::warning::`, step summary, and the `retirement_alert` + * step output the `notify` job posts to Slack + * - no key, no retirement API, auth or transient error → informational skip + * + * Retirement fields used, each verified with a live request on 2026-10-26: + * OpenAI GET /v1/models/{id} `shutdown_date` (YYYY-MM-DD | null); 404 when gone + * Anthropic GET /v1/models/{id} `lifecycle`, `retires_at`; 404 when retired + * Gemini GET /v1beta/models/{id} no date field; 404 when gone + * Cohere GET /v1/models/{id} `is_deprecated` (absent when active); 404 when gone + * OpenRouter GET /api/v1/models `expiration_date` (YYYY-MM-DD | null); absent when gone + * Bedrock's GetFoundationModel reports only `modelLifecycle.status` + * (ACTIVE | LEGACY) and needs AWS credentials this step does not hold; Vertex, + * Ollama and fal expose nothing usable. Those are skipped with a note. + * + * CLI usage (in CI): + * npx tsx scripts/drift-model-preflight.ts + * Env: OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY, COHERE_API_KEY, + * OPENROUTER_API_KEY; DRIFT_PREFLIGHT_WARN_DAYS overrides the 30-day window. + */ + +import { randomBytes } from "node:crypto"; +import { appendFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { DRIFT_PROBE_MODELS, type DriftProbeModel } from "../src/__tests__/drift/probe-models.js"; + +export const DEFAULT_WARN_DAYS = 30; +const REGISTRY_PATH = "src/__tests__/drift/probe-models.ts"; +const DAY_MS = 86_400_000; +const REQUEST_TIMEOUT_MS = 15_000; + +/** What a provider said about one model. */ +export type ModelStatus = + /** Served. `retiresOn` is the published shutdown date, if any. */ + | { kind: "served"; retiresOn?: string; deprecated?: boolean; note?: string } + /** The provider no longer serves it. */ + | { kind: "gone"; detail: string } + /** No answer usable for a retirement decision. Never fails the run. */ + | { kind: "unknown"; detail: string }; + +export type Verdict = "ok" | "warn" | "retired" | "skip"; + +export interface ProbeResult { + probe: DriftProbeModel; + verdict: Verdict; + message: string; +} + +// --------------------------------------------------------------------------- +// Date logic +// --------------------------------------------------------------------------- + +/** Whole UTC days from `now` until `date` (negative once it has passed). */ +export function daysUntil(date: string, now: Date): number { + const target = Date.parse(date.length === 10 ? `${date}T00:00:00Z` : date); + if (Number.isNaN(target)) throw new Error(`unparseable retirement date: ${date}`); + const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); + const t = new Date(target); + const day = Date.UTC(t.getUTCFullYear(), t.getUTCMonth(), t.getUTCDate()); + return Math.round((day - today) / DAY_MS); +} + +/** A model is retired ON its shutdown date, and warned about inside the window. */ +export function classifyRetirement( + retiresOn: string | undefined, + now: Date, + warnDays: number, +): "ok" | "warn" | "retired" { + if (!retiresOn) return "ok"; + const days = daysUntil(retiresOn, now); + if (days <= 0) return "retired"; + return days <= warnDays ? "warn" : "ok"; +} + +/** Where to fix a probe model: its constant when it has one, else its pin. */ +export function fixHint(probe: DriftProbeModel): string { + if (probe.constant) return `update ${probe.constant} in ${probe.source}`; + if (probe.source === REGISTRY_PATH) return `update ${REGISTRY_PATH}`; + return `update the pin in ${probe.source} and its entry in ${REGISTRY_PATH}`; +} + +export function evaluate( + probe: DriftProbeModel, + status: ModelStatus, + now: Date, + warnDays: number, +): ProbeResult { + const name = `probe model ${probe.model} (${probe.provider})`; + if (status.kind === "unknown") { + return { probe, verdict: "skip", message: `${name}: skipped — ${status.detail}` }; + } + if (status.kind === "gone") { + return { + probe, + verdict: "retired", + message: `${name} is retired (${status.detail}) — ${fixHint(probe)}`, + }; + } + const date = status.retiresOn?.slice(0, 10); + const level = classifyRetirement(date, now, warnDays); + if (level === "retired") { + return { + probe, + verdict: "retired", + message: `${name} retired on ${date} — ${fixHint(probe)}`, + }; + } + if (level === "warn") { + const days = daysUntil(date!, now); + return { + probe, + verdict: "warn", + message: `${name} retires on ${date} (in ${days} day${days === 1 ? "" : "s"}) — ${fixHint(probe)}`, + }; + } + if (status.deprecated) { + return { + probe, + verdict: "warn", + message: `${name} is deprecated with no retirement date published — ${fixHint(probe)}`, + }; + } + const when = date ? `shutdown ${date}` : (status.note ?? "no shutdown date"); + return { probe, verdict: "ok", message: `${name}: served (${when})` }; +} + +// --------------------------------------------------------------------------- +// Provider lookups +// --------------------------------------------------------------------------- + +export type FetchLike = (url: string, init?: RequestInit) => Promise; + +interface LookupContext { + fetch: FetchLike; + env: Record; + /** Memo for listing-based providers, so a listing is fetched once per run. */ + cache: Map>; +} + +const KEY_ENV: Record = { + openai: "OPENAI_API_KEY", + anthropic: "ANTHROPIC_API_KEY", + gemini: "GOOGLE_API_KEY", + cohere: "COHERE_API_KEY", + openrouter: "OPENROUTER_API_KEY", +}; + +const NO_API_NOTE: Record = { + bedrock: + "no retirement API reachable here (GetFoundationModel gives only ACTIVE/LEGACY and needs AWS credentials)", + vertex: "no retirement API reachable here", + ollama: "local daemon model; nothing to retire upstream", + fal: "fal exposes no model retirement API", +}; + +/** Turn a non-2xx answer into a status. 404 is the only "gone" signal. */ +async function nonOk(res: Response): Promise { + const body = (await res.text().catch(() => "")).replace(/\s+/g, " ").slice(0, 200); + if (res.status === 404) return { kind: "gone", detail: `HTTP 404: ${body}` }; + return { kind: "unknown", detail: `HTTP ${res.status} from the models API` }; +} + +async function getJson( + ctx: LookupContext, + url: string, + headers: Record, +): Promise { + return ctx.fetch(url, { headers, signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) }); +} + +type Lookup = (model: string, key: string, ctx: LookupContext) => Promise; + +const LOOKUPS: Record = { + async openai(model, key, ctx) { + const res = await getJson( + ctx, + `https://api.openai.com/v1/models/${encodeURIComponent(model)}`, + { + Authorization: `Bearer ${key}`, + }, + ); + if (!res.ok) return nonOk(res); + const body = (await res.json()) as { shutdown_date?: string | null }; + if (!("shutdown_date" in body)) { + return { kind: "unknown", detail: "response has no shutdown_date field" }; + } + return { kind: "served", retiresOn: body.shutdown_date ?? undefined }; + }, + + async anthropic(model, key, ctx) { + const res = await getJson( + ctx, + `https://api.anthropic.com/v1/models/${encodeURIComponent(model)}`, + { + "x-api-key": key, + "anthropic-version": "2023-06-01", + }, + ); + if (!res.ok) return nonOk(res); + const body = (await res.json()) as { lifecycle?: string; retires_at?: string | null }; + if (body.lifecycle === "retired") return { kind: "gone", detail: "lifecycle: retired" }; + if (!("lifecycle" in body) && !("retires_at" in body)) { + return { kind: "unknown", detail: "response has no lifecycle or retires_at field" }; + } + return { + kind: "served", + retiresOn: body.retires_at ?? undefined, + deprecated: body.lifecycle === "deprecated", + }; + }, + + async gemini(model, key, ctx) { + const res = await getJson( + ctx, + `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}`, + { "x-goog-api-key": key }, + ); + if (!res.ok) return nonOk(res); + return { kind: "served", note: "Gemini publishes no shutdown date" }; + }, + + async cohere(model, key, ctx) { + const res = await getJson( + ctx, + `https://api.cohere.com/v1/models/${encodeURIComponent(model)}`, + { + Authorization: `Bearer ${key}`, + }, + ); + if (!res.ok) return nonOk(res); + const body = (await res.json()) as { is_deprecated?: boolean }; + return { + kind: "served", + deprecated: body.is_deprecated === true, + note: "Cohere publishes no shutdown date", + }; + }, + + async openrouter(model, key, ctx) { + let listing = ctx.cache.get("openrouter") as + | Promise + | undefined; + if (!listing) { + listing = getJson(ctx, "https://openrouter.ai/api/v1/models", { + Authorization: `Bearer ${key}`, + }).then(async (res) => { + if (!res.ok) return res; + return ((await res.json()) as { data: { id: string; expiration_date?: string | null }[] }) + .data; + }); + ctx.cache.set("openrouter", listing); + } + const models = await listing; + if (!Array.isArray(models)) { + return { kind: "unknown", detail: `HTTP ${models.status} from the models API` }; + } + const entry = models.find((m) => m.id === model); + if (!entry) return { kind: "gone", detail: "not in the /api/v1/models listing" }; + return { kind: "served", retiresOn: entry.expiration_date ?? undefined }; + }, +}; + +export async function lookup(probe: DriftProbeModel, ctx: LookupContext): Promise { + const check = LOOKUPS[probe.provider]; + if (!check) return { kind: "unknown", detail: NO_API_NOTE[probe.provider] ?? "no API" }; + const keyEnv = KEY_ENV[probe.provider]; + const key = ctx.env[keyEnv]; + if (!key) return { kind: "unknown", detail: `${keyEnv} is not set` }; + try { + return await check(probe.model, key, ctx); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { kind: "unknown", detail: `request failed: ${msg}` }; + } +} + +export async function runPreflight( + probes: readonly DriftProbeModel[], + opts: { fetch: FetchLike; env: Record; now: Date; warnDays: number }, +): Promise { + const ctx: LookupContext = { fetch: opts.fetch, env: opts.env, cache: new Map() }; + return Promise.all( + probes.map(async (probe) => evaluate(probe, await lookup(probe, ctx), opts.now, opts.warnDays)), + ); +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +const ICON: Record = { ok: "✅", warn: "⚠️", retired: "❌", skip: "➖" }; + +export function stepSummary(results: ProbeResult[], warnDays: number): string { + const rows = results.map( + (r) => `| ${ICON[r.verdict]} | ${r.probe.provider} | \`${r.probe.model}\` | ${r.message} |`, + ); + return [ + `### Drift probe-model retirement preflight (warning window: ${warnDays} days)`, + "", + "| | Provider | Model | Result |", + "|---|---|---|---|", + ...rows, + "", + ].join("\n"); +} + +/** Slack mrkdwn lines for the `notify` job: retirements first, then warnings. */ +export function alertText(results: ProbeResult[]): string { + const pick = (verdict: Verdict) => results.filter((r) => r.verdict === verdict); + return [...pick("retired"), ...pick("warn")].map((r) => `• ${r.message}`).join("\n"); +} + +function writeGithubOutput(name: string, value: string): void { + const outPath = process.env.GITHUB_OUTPUT; + if (!outPath) return; + let delimiter = `EOF_${randomBytes(8).toString("hex")}`; + while (value.includes(delimiter)) { + delimiter = `EOF_${randomBytes(8).toString("hex")}`; + } + appendFileSync(outPath, `${name}<<${delimiter}\n${value}\n${delimiter}\n`, "utf-8"); +} + +export function parseWarnDays(raw: string | undefined): number { + if (raw === undefined || raw === "") return DEFAULT_WARN_DAYS; + const n = Number(raw); + if (!Number.isInteger(n) || n < 0) { + throw new Error(`DRIFT_PREFLIGHT_WARN_DAYS must be a whole number of days, got "${raw}"`); + } + return n; +} + +async function main(): Promise { + const warnDays = parseWarnDays(process.env.DRIFT_PREFLIGHT_WARN_DAYS); + const results = await runPreflight(DRIFT_PROBE_MODELS, { + fetch: (url, init) => fetch(url, init), + env: process.env, + now: new Date(), + warnDays, + }); + + for (const r of results) { + if (r.verdict === "retired") console.log(`::error title=probe-model-retired::${r.message}`); + else if (r.verdict === "warn") + console.log(`::warning title=probe-model-retiring::${r.message}`); + else console.log(`preflight: ${r.message}`); + } + + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, stepSummary(results, warnDays), "utf-8"); + } + const retired = results.some((r) => r.verdict === "retired"); + writeGithubOutput("retirement_alert", alertText(results)); + writeGithubOutput("retired", String(retired)); + + if (retired) { + console.log( + "::error::Probe-model preflight failed: a drift probe model is retired. Drift was not run — this is not API drift.", + ); + process.exit(1); + } + console.log("preflight: no drift probe model is retired"); +} + +// Only run as a CLI — importing this module (e.g. from tests) does not run main(). +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + await main(); +} diff --git a/src/__tests__/drift-model-preflight.test.ts b/src/__tests__/drift-model-preflight.test.ts new file mode 100644 index 00000000..8eafd7da --- /dev/null +++ b/src/__tests__/drift-model-preflight.test.ts @@ -0,0 +1,273 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + alertText, + classifyRetirement, + daysUntil, + evaluate, + parseWarnDays, + runPreflight, + stepSummary, + type FetchLike, +} from "../../scripts/drift-model-preflight.js"; +import { DRIFT_PROBE_MODELS, type DriftProbeModel } from "./drift/probe-models.js"; + +const NOW = new Date("2026-10-26T18:00:00Z"); +const k9: DriftProbeModel = { + provider: "openai", + model: "o4-mini", + source: "src/__tests__/drift/probe-models.ts", + constant: "OPENAI_K9_PROBE_MODEL", +}; + +describe("date logic (fixed dates)", () => { + it("counts whole UTC days, whatever the time of day", () => { + expect(daysUntil("2026-10-26", NOW)).toBe(0); + expect(daysUntil("2026-10-27", NOW)).toBe(1); + expect(daysUntil("2026-10-23", NOW)).toBe(-3); + expect(daysUntil("2026-11-25T23:59:59Z", NOW)).toBe(30); + }); + + it("is retired ON the shutdown date and after it", () => { + expect(classifyRetirement("2026-10-23", NOW, 30)).toBe("retired"); + expect(classifyRetirement("2026-10-26", NOW, 30)).toBe("retired"); + }); + + it("warns inside the window, inclusive, and not outside it", () => { + expect(classifyRetirement("2026-10-27", NOW, 30)).toBe("warn"); + expect(classifyRetirement("2026-11-25", NOW, 30)).toBe("warn"); + expect(classifyRetirement("2026-11-26", NOW, 30)).toBe("ok"); + expect(classifyRetirement("2026-12-11", NOW, 30)).toBe("ok"); + expect(classifyRetirement("2026-12-11", NOW, 46)).toBe("warn"); + }); + + it("has nothing to say without a date", () => { + expect(classifyRetirement(undefined, NOW, 30)).toBe("ok"); + }); + + it("refuses a date it cannot read instead of passing it", () => { + expect(() => daysUntil("soon", NOW)).toThrow(/unparseable/); + }); + + it("reads the warning window from the environment", () => { + expect(parseWarnDays(undefined)).toBe(30); + expect(parseWarnDays("")).toBe(30); + expect(parseWarnDays("60")).toBe(60); + expect(() => parseWarnDays("-1")).toThrow(); + expect(() => parseWarnDays("ten")).toThrow(); + }); +}); + +describe("evaluate", () => { + it("names the model, the date and the constant to change when it is retired", () => { + const r = evaluate(k9, { kind: "served", retiresOn: "2026-10-23" }, NOW, 30); + expect(r.verdict).toBe("retired"); + expect(r.message).toBe( + "probe model o4-mini (openai) retired on 2026-10-23 — update OPENAI_K9_PROBE_MODEL in src/__tests__/drift/probe-models.ts", + ); + }); + + it("treats a gone model as retired and points at the pin", () => { + const pinned: DriftProbeModel = { + provider: "gemini", + model: "text-embedding-004", + source: "src/__tests__/drift/gemini.drift.ts", + }; + const r = evaluate(pinned, { kind: "gone", detail: "HTTP 404: not found" }, NOW, 30); + expect(r.verdict).toBe("retired"); + expect(r.message).toContain("text-embedding-004 (gemini) is retired (HTTP 404"); + expect(r.message).toContain( + "update the pin in src/__tests__/drift/gemini.drift.ts and its entry in src/__tests__/drift/probe-models.ts", + ); + }); + + it("warns with the days left", () => { + const r = evaluate( + { ...k9, model: "gpt-5-mini" }, + { kind: "served", retiresOn: "2026-11-01" }, + NOW, + 30, + ); + expect(r.verdict).toBe("warn"); + expect(r.message).toContain("gpt-5-mini (openai) retires on 2026-11-01 (in 6 days)"); + }); + + it("uses only the date part of an ISO timestamp", () => { + const r = evaluate(k9, { kind: "served", retiresOn: "2026-11-02T00:00:00Z" }, NOW, 30); + expect(r.message).toContain("retires on 2026-11-02 (in 7 days)"); + }); + + it("warns on a deprecation that has no date", () => { + const r = evaluate(k9, { kind: "served", deprecated: true }, NOW, 30); + expect(r.verdict).toBe("warn"); + expect(r.message).toContain("deprecated with no retirement date published"); + }); + + it("skips, never fails, when the provider gave no usable answer", () => { + const r = evaluate(k9, { kind: "unknown", detail: "HTTP 503 from the models API" }, NOW, 30); + expect(r.verdict).toBe("skip"); + }); +}); + +/** A fake provider API answering by URL. */ +function fakeFetch(routes: Record): FetchLike { + return async (url) => { + const route = routes[url]; + if (!route) throw new Error(`unexpected request: ${url}`); + const [status, body] = route; + return new Response(typeof body === "string" ? body : JSON.stringify(body), { status }); + }; +} + +const ALL_KEYS = { + OPENAI_API_KEY: "k", + ANTHROPIC_API_KEY: "k", + GOOGLE_API_KEY: "k", + COHERE_API_KEY: "k", + OPENROUTER_API_KEY: "k", +}; + +describe("provider lookups", () => { + const probes: DriftProbeModel[] = [ + { provider: "openai", model: "o4-mini", source: "a.ts" }, + { provider: "openai", model: "gpt-5-mini", source: "a.ts" }, + { provider: "openai", model: "gpt-4o-mini", source: "a.ts" }, + { provider: "anthropic", model: "claude-3-haiku-20240307", source: "b.ts" }, + { provider: "anthropic", model: "claude-old", source: "b.ts" }, + { provider: "gemini", model: "gemini-2.5-flash", source: "c.ts" }, + { provider: "cohere", model: "rerank-v3.5", source: "d.ts" }, + { provider: "openrouter", model: "qwen/qwen3-32b", source: "e.ts" }, + { provider: "openrouter", model: "gone/model", source: "e.ts" }, + { provider: "ollama", model: "qwen2:0.5b", source: "f.yml" }, + ]; + const routes: Record = { + "https://api.openai.com/v1/models/o4-mini": [ + 200, + { id: "o4-mini", shutdown_date: "2026-10-23" }, + ], + "https://api.openai.com/v1/models/gpt-5-mini": [200, { shutdown_date: "2026-11-20" }], + "https://api.openai.com/v1/models/gpt-4o-mini": [200, { shutdown_date: null }], + "https://api.anthropic.com/v1/models/claude-3-haiku-20240307": [ + 404, + { type: "error", error: { type: "not_found_error" } }, + ], + "https://api.anthropic.com/v1/models/claude-old": [ + 200, + { lifecycle: "deprecated", retires_at: "2026-11-05T00:00:00Z" }, + ], + "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash": [200, {}], + "https://api.cohere.com/v1/models/rerank-v3.5": [200, { name: "rerank-v3.5" }], + "https://openrouter.ai/api/v1/models": [ + 200, + { data: [{ id: "qwen/qwen3-32b", expiration_date: null }] }, + ], + }; + + it("maps each provider's real retirement field to a verdict", async () => { + const results = await runPreflight(probes, { + fetch: fakeFetch(routes), + env: ALL_KEYS, + now: NOW, + warnDays: 30, + }); + expect(results.map((r) => [r.probe.model, r.verdict])).toEqual([ + ["o4-mini", "retired"], + ["gpt-5-mini", "warn"], + ["gpt-4o-mini", "ok"], + ["claude-3-haiku-20240307", "retired"], + ["claude-old", "warn"], + ["gemini-2.5-flash", "ok"], + ["rerank-v3.5", "ok"], + ["qwen/qwen3-32b", "ok"], + ["gone/model", "retired"], + ["qwen2:0.5b", "skip"], + ]); + }); + + it("fetches the OpenRouter listing once for any number of models", async () => { + let calls = 0; + const counting: FetchLike = async (url, init) => { + if (url.startsWith("https://openrouter.ai")) calls++; + return fakeFetch(routes)(url, init); + }; + await runPreflight( + probes.filter((p) => p.provider === "openrouter"), + { fetch: counting, env: ALL_KEYS, now: NOW, warnDays: 30 }, + ); + expect(calls).toBe(1); + }); + + it("skips on a missing key, an auth error, a 5xx or a network error", async () => { + const url = "https://api.openai.com/v1/models/gpt-4o-mini"; + const one = (status: number): FetchLike => fakeFetch({ [url]: [status, "nope"] }); + const probe = [probes[2]]; + const run = (fetch: FetchLike, env: Record = ALL_KEYS) => + runPreflight(probe, { fetch, env, now: NOW, warnDays: 30 }).then((r) => r[0]); + + expect((await run(one(200), {})).message).toContain("OPENAI_API_KEY is not set"); + expect((await run(one(401))).verdict).toBe("skip"); + expect((await run(one(429))).verdict).toBe("skip"); + expect((await run(one(503))).verdict).toBe("skip"); + const down: FetchLike = async () => { + throw new TypeError("fetch failed"); + }; + expect((await run(down)).message).toContain("request failed: fetch failed"); + }); + + it("skips when the response lacks the retirement field instead of calling it served", async () => { + const r = await runPreflight([probes[2]], { + fetch: fakeFetch({ "https://api.openai.com/v1/models/gpt-4o-mini": [200, { id: "x" }] }), + env: ALL_KEYS, + now: NOW, + warnDays: 30, + }); + expect(r[0].verdict).toBe("skip"); + }); +}); + +describe("reporting", () => { + const results = [ + evaluate(k9, { kind: "served", retiresOn: "2026-11-10" }, NOW, 30), + evaluate(k9, { kind: "served", retiresOn: "2026-10-23" }, NOW, 30), + evaluate(k9, { kind: "served" }, NOW, 30), + evaluate(k9, { kind: "unknown", detail: "x" }, NOW, 30), + ]; + + it("alerts on retirements first, then warnings, and nothing else", () => { + const lines = alertText(results).split("\n"); + expect(lines).toHaveLength(2); + expect(lines[0]).toMatch(/^• probe model o4-mini \(openai\) retired on 2026-10-23/); + expect(lines[1]).toMatch(/^• probe model o4-mini \(openai\) retires on 2026-11-10/); + }); + + it("alert text is empty on a clean run", () => { + expect(alertText([results[2], results[3]])).toBe(""); + }); + + it("writes one summary row per model", () => { + const summary = stepSummary(results, 30); + expect(summary).toContain("warning window: 30 days"); + expect( + summary.split("\n").filter((l) => l.startsWith("| ") && l.includes("`o4-mini`")), + ).toHaveLength(4); + }); +}); + +describe("probe-models.ts stays honest", () => { + const root = resolve(__dirname, "../.."); + + it("every entry is still pinned in the source file it names", () => { + for (const p of DRIFT_PROBE_MODELS) { + const text = readFileSync(resolve(root, p.source), "utf8"); + expect(text, `${p.model} is no longer in ${p.source}`).toContain(p.model); + } + }); + + it("lists each provider/model pair once", () => { + const keys = DRIFT_PROBE_MODELS.map((p) => `${p.provider}/${p.model}`); + expect(new Set(keys).size).toBe(keys.length); + }); +}); diff --git a/src/__tests__/drift/gemini.drift.ts b/src/__tests__/drift/gemini.drift.ts index 17cb6339..9a1bb85a 100644 --- a/src/__tests__/drift/gemini.drift.ts +++ b/src/__tests__/drift/gemini.drift.ts @@ -755,7 +755,7 @@ describe("Gemini thinking token shapes", () => { describe.skipIf(!GOOGLE_API_KEY)("Gemini Embeddings canary", () => { it("canary: verify embeddings endpoint exists and response shape", async () => { const res = await fetch( - `https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent?key=${GOOGLE_API_KEY}`, + `https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:embedContent?key=${GOOGLE_API_KEY}`, { method: "POST", headers: { "content-type": "application/json" }, diff --git a/src/__tests__/drift/helpers.ts b/src/__tests__/drift/helpers.ts index 82b72fb6..132c3a98 100644 --- a/src/__tests__/drift/helpers.ts +++ b/src/__tests__/drift/helpers.ts @@ -318,12 +318,10 @@ export async function collectMockWSMessages( // OpenAI Responses K9 (reasoning-only max_output_tokens exhaustion) // --------------------------------------------------------------------------- -/** - * The OpenAI model the live K9 exhaustion probes (HTTP and WS) request. Change - * only this line when OpenAI retires it: `o4-mini` hit its `/v1/models` - * `shutdown_date` of 2026-10-23 and has answered `404 model_not_found` since. - */ -export const OPENAI_K9_PROBE_MODEL = "gpt-5.4-mini"; +// The K9 probe model lives with every other pinned drift probe model in +// probe-models.ts, which the retirement preflight reads without loading this +// module (it starts servers). +export { OPENAI_K9_PROBE_MODEL } from "./probe-models.js"; /** * The model the retained K9 replay fixtures were captured from (2026-10-10). diff --git a/src/__tests__/drift/probe-models.ts b/src/__tests__/drift/probe-models.ts new file mode 100644 index 00000000..2c05e659 --- /dev/null +++ b/src/__tests__/drift/probe-models.ts @@ -0,0 +1,114 @@ +/** + * Every provider model the live drift probes PIN, in one place. + * + * `scripts/drift-model-preflight.ts` reads this list before the nightly drift + * run and asks each provider whether the model is still served. A retired probe + * model otherwise surfaces as a confusing drift failure (`o4-mini` hit its + * `shutdown_date` on 2026-10-23 and the K9 probe then failed for three nights + * as "drift"). + * + * Side-effect-free data module (no `describe`/`beforeAll`), so the preflight + * script and the unit test import it without registering a drift suite. + * + * NOT LISTED: legs that choose their model from the provider's live listing at + * run time (`resolveLiveModel` / `resolveLiveModelCandidates` in providers.ts — + * the OpenAI chat and Responses tool legs, the Anthropic text leg, Cohere chat, + * Gemini Live). They move to a served model on their own. Mock-only legs that + * never send their model upstream are not listed either. + * + * WHEN YOU PIN A NEW PROBE MODEL, add it here. `probe-models.test.ts` checks + * that each entry still appears in the `source` file it names, so an entry + * cannot outlive its pin unnoticed. + */ + +/** Providers whose model APIs report retirement, so the preflight can ask. */ +export type RetirementCheckedProvider = "openai" | "anthropic" | "gemini" | "cohere" | "openrouter"; + +/** Providers with no retirement API reachable from the preflight. */ +export type UncheckedProvider = "bedrock" | "vertex" | "ollama" | "fal"; + +export interface DriftProbeModel { + provider: RetirementCheckedProvider | UncheckedProvider; + /** The exact id the probe sends upstream. */ + model: string; + /** Repo-relative file that pins the id: the place to edit when it retires. */ + source: string; + /** The exported constant to change, when the pin is one. */ + constant?: string; +} + +/** + * The OpenAI model the live K9 exhaustion probes (HTTP and WS) request, via + * helpers.ts. Change only this line when OpenAI retires it: `o4-mini` hit its + * `/v1/models` `shutdown_date` of 2026-10-23 and has answered + * `404 model_not_found` since. + */ +export const OPENAI_K9_PROBE_MODEL = "gpt-5.4-mini"; + +export const DRIFT_PROBE_MODELS: readonly DriftProbeModel[] = [ + // OpenAI + { provider: "openai", model: "gpt-4o-mini", source: "src/__tests__/drift/providers.ts" }, + { + provider: "openai", + model: "text-embedding-3-small", + source: "src/__tests__/drift/providers.ts", + }, + { provider: "openai", model: "whisper-1", source: "src/__tests__/drift/transcription.drift.ts" }, + { provider: "openai", model: "gpt-realtime-mini", source: "src/__tests__/drift/ws-providers.ts" }, + { + provider: "openai", + model: "gpt-4.1-mini", + source: "src/__tests__/drift/openai-responses.drift.ts", + }, + { + provider: "openai", + model: OPENAI_K9_PROBE_MODEL, + source: "src/__tests__/drift/probe-models.ts", + constant: "OPENAI_K9_PROBE_MODEL", + }, + // Anthropic + { + provider: "anthropic", + model: "claude-haiku-4-5-20251001", + source: "src/__tests__/drift/providers.ts", + }, + { + provider: "anthropic", + model: "claude-sonnet-4-5-20250929", + source: "src/__tests__/drift/anthropic.drift.ts", + }, + { + provider: "anthropic", + model: "claude-sonnet-4-5", + source: "src/__tests__/drift/anthropic.drift.ts", + }, + // Gemini Developer API + { provider: "gemini", model: "gemini-2.5-flash", source: "src/__tests__/drift/providers.ts" }, + { + provider: "gemini", + model: "gemini-embedding-001", + source: "src/__tests__/drift/gemini.drift.ts", + }, + // Cohere + { provider: "cohere", model: "rerank-v3.5", source: "src/__tests__/drift/rerank.drift.ts" }, + // OpenRouter + { + provider: "openrouter", + model: "qwen/qwen3-32b", + source: "src/__tests__/drift/openrouter-chat.drift.ts", + }, + // No retirement API the preflight can reach (listed so the run says so). + { + provider: "bedrock", + model: "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + source: ".github/workflows/test-drift.yml", + }, + { provider: "vertex", model: "gemini-2.5-flash", source: ".github/workflows/test-drift.yml" }, + { provider: "ollama", model: "qwen2:0.5b", source: ".github/workflows/test-drift.yml" }, + { provider: "ollama", model: "qwen3:0.6b", source: ".github/workflows/test-drift.yml" }, + { + provider: "fal", + model: "fal-ai/flux/schnell", + source: "src/__tests__/drift/fal-queue.drift.ts", + }, +]; diff --git a/src/__tests__/test-drift-workflow.test.ts b/src/__tests__/test-drift-workflow.test.ts index 240b9f2b..d026c4e4 100644 --- a/src/__tests__/test-drift-workflow.test.ts +++ b/src/__tests__/test-drift-workflow.test.ts @@ -848,3 +848,85 @@ describe("test-drift.yml — a hang-up is annotated as a close, not as a timeout expect(run.stdio).not.toContain("nullms"); }); }); + +// --------------------------------------------------------------------------- +// A retired probe model is named as one, never as drift. +// +// o4-mini hit its OpenAI shutdown_date on 2026-10-23 and the K9 probe that +// pinned it failed the nightly run for three nights, alerted as "providers +// changed response formats". The `Preflight — probe model retirement` step now +// stops the job first, and `notify` routes on its outputs. +// --------------------------------------------------------------------------- +describe("test-drift.yml — a retired probe model is not reported as drift", () => { + const RETIRED_LINE = + "• probe model o4-mini (openai) retired on 2026-10-23 — update OPENAI_K9_PROBE_MODEL in src/__tests__/drift/probe-models.ts"; + const WARN_LINE = + "• probe model gpt-5-mini (openai) retires on 2026-12-11 (in 20 days) — update the pin in src/__tests__/drift/providers.ts and its entry in src/__tests__/drift/probe-models.ts"; + + it("the preflight runs in the drift job before any drift test, with the keys it queries", () => { + const order = stepsOfJob(DRIFT_JOB); + const idx = order.findIndex((s) => s.name === "Preflight — probe model retirement"); + expect(idx, "no probe-model preflight step").toBeGreaterThan(-1); + expect(idx).toBeLessThan(order.findIndex((s) => s.name === KEY_STEP)); + expect(order[idx].id).toBe("model_preflight"); + expect(order[idx].run).toContain("npx tsx scripts/drift-model-preflight.ts"); + expect(Object.keys(order[idx].env).sort()).toEqual([ + "ANTHROPIC_API_KEY", + "COHERE_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "OPENROUTER_API_KEY", + ]); + expect(wf).toContain("model_alert: ${{ steps.model_preflight.outputs.retirement_alert }}"); + expect(wf).toContain("model_retired: ${{ steps.model_preflight.outputs.retired }}"); + }); + + it("EXECUTED: a retired probe model alerts as a retirement, not as HTTP API drift", () => { + const { posted } = observeNotify({ + DRIFT_EXIT_CODE: "", + DRIFT_RESULT: "failure", + PREV: "success", + MODEL_RETIRED: "true", + MODEL_ALERT: RETIRED_LINE, + }); + expect(posted).toContain("Drift probe model retired"); + expect(posted).toContain("o4-mini (openai) retired on 2026-10-23"); + expect(posted).not.toContain("HTTP API drift detected"); + }); + + it("EXECUTED NEGATIVE CONTROL: the same failed job without a retirement is still drift", () => { + const { posted } = observeNotify({ + DRIFT_EXIT_CODE: "", + DRIFT_RESULT: "failure", + PREV: "success", + MODEL_RETIRED: "false", + MODEL_ALERT: "", + }); + expect(posted).toContain("HTTP API drift detected"); + expect(posted).not.toContain("probe model"); + }); + + it("EXECUTED: a soon-retiring probe model posts on an otherwise quiet day", () => { + const { posted } = observeNotify({ + DRIFT_EXIT_CODE: "0", + DRIFT_RESULT: "success", + PREV: "success", + MODEL_RETIRED: "false", + MODEL_ALERT: WARN_LINE, + }); + expect(posted).toContain("Drift probe model retiring soon"); + expect(posted).toContain("gpt-5-mini (openai) retires on 2026-12-11"); + }); + + it("EXECUTED: a soon-retiring probe model rides along on a real drift alert", () => { + const { posted } = observeNotify({ + DRIFT_EXIT_CODE: "2", + DRIFT_RESULT: "failure", + PREV: "success", + MODEL_RETIRED: "false", + MODEL_ALERT: WARN_LINE, + }); + expect(posted).toContain("HTTP API drift detected"); + expect(posted).toContain("gpt-5-mini (openai) retires on 2026-12-11"); + }); +});