From 59abdeeb71c3f06df272eab8689f3a2cc89a7c30 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Fri, 26 Jun 2026 19:01:56 +0200 Subject: [PATCH 01/14] feat(appkit): add agent eval framework with native mlflow evaluation runs eve-style eval authoring (defineEval + t-context + matchers) discovered from config/agents//evals/*.eval.ts and run via 'appkit agent eval' against a running app. Streams per-eval progress and gates CI via exit code. When Databricks creds + an experiment are set, it creates a real MLflow evaluation run (mlflow.runType=genai_evaluate): each eval's trace links to the run, pass/fail is written as feedback assessments, and aggregate metrics are logged. All via the MLflow REST API. Signed-off-by: MarioCadenas --- .../config/agents/query/evals/smoke.eval.ts | 14 ++ .../config/agents/query/evals/sum.eval.ts | 13 ++ .../agents/query/evals/tool-call.eval.ts | 16 ++ packages/appkit/src/beta.ts | 2 + packages/appkit/src/evals/define-eval.ts | 32 +++ packages/appkit/src/evals/discover.ts | 76 +++++++ packages/appkit/src/evals/http-driver.ts | 148 ++++++++++++++ packages/appkit/src/evals/index.ts | 40 ++++ packages/appkit/src/evals/matchers.ts | 25 +++ packages/appkit/src/evals/mlflow-report.ts | 113 +++++++++++ packages/appkit/src/evals/mlflow-rest.ts | 39 ++++ packages/appkit/src/evals/mlflow-run.ts | 117 +++++++++++ packages/appkit/src/evals/report.ts | 76 +++++++ packages/appkit/src/evals/run-eval.ts | 155 ++++++++++++++ packages/appkit/src/evals/run-evals.ts | 166 +++++++++++++++ .../appkit/src/evals/tests/discover.test.ts | 42 ++++ .../appkit/src/evals/tests/matchers.test.ts | 19 ++ .../src/evals/tests/mlflow-report.test.ts | 57 ++++++ .../appkit/src/evals/tests/mlflow-run.test.ts | 30 +++ .../appkit/src/evals/tests/report.test.ts | 54 +++++ .../src/evals/tests/resolve-default.test.ts | 25 +++ .../appkit/src/evals/tests/run-eval.test.ts | 117 +++++++++++ packages/appkit/src/evals/types.ts | 128 ++++++++++++ .../shared/src/cli/commands/agent/eval.ts | 191 ++++++++++++++++++ .../shared/src/cli/commands/agent/index.ts | 19 ++ packages/shared/src/cli/index.ts | 2 + 26 files changed, 1716 insertions(+) create mode 100644 apps/dev-playground/config/agents/query/evals/smoke.eval.ts create mode 100644 apps/dev-playground/config/agents/query/evals/sum.eval.ts create mode 100644 apps/dev-playground/config/agents/query/evals/tool-call.eval.ts create mode 100644 packages/appkit/src/evals/define-eval.ts create mode 100644 packages/appkit/src/evals/discover.ts create mode 100644 packages/appkit/src/evals/http-driver.ts create mode 100644 packages/appkit/src/evals/index.ts create mode 100644 packages/appkit/src/evals/matchers.ts create mode 100644 packages/appkit/src/evals/mlflow-report.ts create mode 100644 packages/appkit/src/evals/mlflow-rest.ts create mode 100644 packages/appkit/src/evals/mlflow-run.ts create mode 100644 packages/appkit/src/evals/report.ts create mode 100644 packages/appkit/src/evals/run-eval.ts create mode 100644 packages/appkit/src/evals/run-evals.ts create mode 100644 packages/appkit/src/evals/tests/discover.test.ts create mode 100644 packages/appkit/src/evals/tests/matchers.test.ts create mode 100644 packages/appkit/src/evals/tests/mlflow-report.test.ts create mode 100644 packages/appkit/src/evals/tests/mlflow-run.test.ts create mode 100644 packages/appkit/src/evals/tests/report.test.ts create mode 100644 packages/appkit/src/evals/tests/resolve-default.test.ts create mode 100644 packages/appkit/src/evals/tests/run-eval.test.ts create mode 100644 packages/appkit/src/evals/types.ts create mode 100644 packages/shared/src/cli/commands/agent/eval.ts create mode 100644 packages/shared/src/cli/commands/agent/index.ts diff --git a/apps/dev-playground/config/agents/query/evals/smoke.eval.ts b/apps/dev-playground/config/agents/query/evals/smoke.eval.ts new file mode 100644 index 000000000..de3f2d4d4 --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/smoke.eval.ts @@ -0,0 +1,14 @@ +import { defineEval } from "@databricks/appkit/beta"; + +/** + * Example eval. The agent defaults to this directory's name (`query`). + * Run with: + * pnpm exec appkit agent eval query --root apps/dev-playground --url http://localhost:8000 + */ +export default defineEval({ + description: "Query dispatcher responds to a greeting", + async test(t) { + await t.send("Hi there!"); + t.succeeded(); // gate: the turn completed + }, +}); diff --git a/apps/dev-playground/config/agents/query/evals/sum.eval.ts b/apps/dev-playground/config/agents/query/evals/sum.eval.ts new file mode 100644 index 000000000..9bc905e6d --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/sum.eval.ts @@ -0,0 +1,13 @@ +import { defineEval, includes } from "@databricks/appkit/beta"; + +export default defineEval({ + description: "Helper agent smoke test", + // Target the default code-defined agent. Drop this to use the `query` agent + // (the parent directory name). + agent: "helper", + async test(t) { + await t.send("What is 2 + 2?"); + t.succeeded(); // gate: the turn completed + t.check(t.reply, includes("4")).soft(); // tracked metric, won't fail the gate + }, +}); diff --git a/apps/dev-playground/config/agents/query/evals/tool-call.eval.ts b/apps/dev-playground/config/agents/query/evals/tool-call.eval.ts new file mode 100644 index 000000000..47221330a --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/tool-call.eval.ts @@ -0,0 +1,16 @@ +import { defineEval } from "@databricks/appkit/beta"; + +/** + * Tool-call eval: the default `helper` agent should call its `get_weather` + * tool when asked about the weather. (Targets `helper` explicitly — the parent + * directory only determines discovery, not which agent runs.) + */ +export default defineEval({ + description: "Helper agent calls the get_weather tool", + agent: "helper", + async test(t) { + await t.send("What's the weather in Brooklyn?"); + t.succeeded(); + t.calledTool("get_weather"); + }, +}); diff --git a/packages/appkit/src/beta.ts b/packages/appkit/src/beta.ts index 4de7ba79c..b7c457cae 100644 --- a/packages/appkit/src/beta.ts +++ b/packages/appkit/src/beta.ts @@ -83,6 +83,8 @@ export { varchar, } from "./database/schema-builder"; +// Agent evaluation (eve-style authoring, reports to MLflow) +export * from "./evals"; // Agent types export type { AgentDefinition, diff --git a/packages/appkit/src/evals/define-eval.ts b/packages/appkit/src/evals/define-eval.ts new file mode 100644 index 000000000..9b8ff9072 --- /dev/null +++ b/packages/appkit/src/evals/define-eval.ts @@ -0,0 +1,32 @@ +import type { EvalConfig, EvalDefinition } from "./types"; + +/** + * Define an agent eval. Default-export the result from a + * `config/agents//evals/*.eval.ts` file. + * + * @example + * ```ts + * import { defineEval, includes } from "@databricks/appkit/beta"; + * + * export default defineEval({ + * description: "Weather agent basic coverage", + * async test(t) { + * await t.send("What's the weather in Brooklyn?"); + * t.succeeded(); + * t.calledTool("get_weather"); + * t.check(t.reply, includes("Sunny")); + * }, + * }); + * ``` + */ +export function defineEval(def: EvalDefinition): EvalDefinition { + if (typeof def.test !== "function") { + throw new Error("defineEval: `test` must be a function"); + } + return def; +} + +/** Define per-directory eval config. Default-export from `evals.config.ts`. */ +export function defineEvalConfig(config: EvalConfig): EvalConfig { + return config; +} diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts new file mode 100644 index 000000000..e60ae4030 --- /dev/null +++ b/packages/appkit/src/evals/discover.ts @@ -0,0 +1,76 @@ +import { readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +/** An eval file found under `config/agents//evals/`. */ +export interface DiscoveredEval { + /** Absolute path to the `*.eval.ts` file. */ + file: string; + /** Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). */ + id: string; + /** The agent id (the `config/agents/` directory name). */ + agent: string; +} + +function isDir(p: string): boolean { + try { + return statSync(p).isDirectory(); + } catch { + return false; + } +} + +/** Recursively collect `*.eval.ts` files (skips `evals.config.ts`). */ +function walkEvalFiles(dir: string): string[] { + const out: string[] = []; + let entries: string[]; + try { + entries = readdirSync(dir); + } catch { + return out; + } + for (const entry of entries) { + const full = path.join(dir, entry); + if (isDir(full)) { + out.push(...walkEvalFiles(full)); + } else if (entry.endsWith(".eval.ts")) { + out.push(full); + } + } + return out; +} + +/** + * Discover evals under `/config/agents//evals/`. The agent id + * is the directory name; the eval id is the file path relative to that evals + * dir with `.eval.ts` stripped. Returns a stable, sorted list. + */ +export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { + const agentsDir = path.join(rootDir, "config", "agents"); + const out: DiscoveredEval[] = []; + + let agents: string[]; + try { + agents = readdirSync(agentsDir).filter((n) => + isDir(path.join(agentsDir, n)), + ); + } catch { + return out; + } + + for (const agent of agents) { + const evalsDir = path.join(agentsDir, agent, "evals"); + if (!isDir(evalsDir)) continue; + for (const file of walkEvalFiles(evalsDir)) { + const id = path + .relative(evalsDir, file) + .replace(/\.eval\.ts$/, "") + .split(path.sep) + .join("/"); + out.push({ file, id, agent }); + } + } + + return out.sort( + (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id), + ); +} diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts new file mode 100644 index 000000000..0803a6e37 --- /dev/null +++ b/packages/appkit/src/evals/http-driver.ts @@ -0,0 +1,148 @@ +import type { DriveResult, EvalDriver } from "./types"; + +export interface HttpDriverOptions { + /** Base URL of the running app, e.g. `http://localhost:3000`. */ + baseUrl: string; + /** Agent alias to target. Omit to use the app's default agent. */ + agent?: string; + /** Extra request headers (e.g. auth for a deployed app). */ + headers?: Record; + /** Chat endpoint path. Defaults to `/api/agents/chat`. */ + path?: string; + /** MLflow run id to link each turn's trace to (for evaluation runs). */ + mlflowRunId?: string; +} + +/** Parse a single Responses-API SSE `data:` payload into the running totals. */ +function applyEvent( + event: Record, + state: { + reply: string; + toolCalls: string[]; + seen: Set; + ok: boolean; + traceId?: string; + }, + setThread: (id: string) => void, +): void { + const type = event.type; + if ( + type === "response.output_text.delta" && + typeof event.delta === "string" + ) { + state.reply += event.delta; + return; + } + if ( + type === "response.output_item.added" || + type === "response.output_item.done" + ) { + const item = event.item as + | { type?: string; name?: string; call_id?: string } + | undefined; + if (item?.type === "function_call" && item.name) { + const key = item.call_id ?? item.name; + if (!state.seen.has(key)) { + state.seen.add(key); + state.toolCalls.push(item.name); + } + } + return; + } + if (type === "error" || type === "response.failed") { + state.ok = false; + return; + } + if (type === "appkit.metadata") { + const data = event.data as + | { threadId?: string; mlflowTraceId?: string } + | undefined; + if (data?.threadId) setThread(data.threadId); + if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId; + } +} + +/** + * Drives an agent by POSTing to a running app's chat endpoint and parsing the + * SSE response. Keeps the thread id across `send`s so multi-turn evals share a + * conversation. Agent/stream errors surface as `succeeded: false` rather than + * throwing, so `t.succeeded()` can assert on them. + */ +export function createHttpDriver(options: HttpDriverOptions): EvalDriver { + const chatPath = options.path ?? "/api/agents/chat"; + let threadId: string | undefined; + + return { + async send(message: string): Promise { + let res: Response; + try { + res = await fetch(`${options.baseUrl}${chatPath}`, { + method: "POST", + headers: { "content-type": "application/json", ...options.headers }, + body: JSON.stringify({ + message, + ...(options.agent ? { agent: options.agent } : {}), + ...(threadId ? { threadId } : {}), + ...(options.mlflowRunId + ? { mlflowRunId: options.mlflowRunId } + : {}), + }), + }); + } catch { + return { reply: "", toolCalls: [], succeeded: false }; + } + + if (!res.ok || !res.body) { + return { + reply: "", + toolCalls: [], + succeeded: false, + sessionId: threadId, + }; + } + + const state = { + reply: "", + toolCalls: [] as string[], + seen: new Set(), + ok: true, + traceId: undefined as string | undefined, + }; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice(6).trim(); + if (!data || data === "[DONE]") continue; + try { + applyEvent(JSON.parse(data), state, (id) => { + threadId = id; + }); + } catch { + // skip malformed event lines + } + } + } + } finally { + reader.releaseLock(); + } + + return { + reply: state.reply, + toolCalls: state.toolCalls, + succeeded: state.ok, + sessionId: threadId, + traceId: state.traceId, + }; + }, + }; +} diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts new file mode 100644 index 000000000..7143d993a --- /dev/null +++ b/packages/appkit/src/evals/index.ts @@ -0,0 +1,40 @@ +export { defineEval, defineEvalConfig } from "./define-eval"; +export { type DiscoveredEval, discoverEvalFiles } from "./discover"; +export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; +export { equals, includes, matches } from "./matchers"; +export { + type Assessment, + buildAssessment, + type MlflowReportOptions, + type ReportOutcome, + reportToMlflow, +} from "./mlflow-report"; +export { + type EvalSummary, + evalGlyph, + formatEvalDetail, + formatEvalHeadline, + formatEvalResults, + formatSummaryLine, + summarize, +} from "./report"; +export { type RunEvalOptions, runEval } from "./run-eval"; +export { + type EvalProgress, + type EvalRunSummary, + type RunEvalsOptions, + runEvalsInDir, +} from "./run-evals"; +export type { + AssertionHandle, + AssertionResult, + DriveResult, + EvalConfig, + EvalDefinition, + EvalDriver, + EvalResult, + Matcher, + MatchResult, + Severity, + TestContext, +} from "./types"; diff --git a/packages/appkit/src/evals/matchers.ts b/packages/appkit/src/evals/matchers.ts new file mode 100644 index 000000000..1e57796f9 --- /dev/null +++ b/packages/appkit/src/evals/matchers.ts @@ -0,0 +1,25 @@ +import type { Matcher } from "./types"; + +/** Passes when the value contains `substring`. */ +export function includes(substring: string): Matcher { + return (value) => ({ + pass: value.includes(substring), + detail: `expected to include ${JSON.stringify(substring)}`, + }); +} + +/** Passes when the value equals `expected` exactly. */ +export function equals(expected: string): Matcher { + return (value) => ({ + pass: value === expected, + detail: `expected to equal ${JSON.stringify(expected)}`, + }); +} + +/** Passes when the value matches `pattern`. */ +export function matches(pattern: RegExp): Matcher { + return (value) => ({ + pass: pattern.test(value), + detail: `expected to match ${pattern}`, + }); +} diff --git a/packages/appkit/src/evals/mlflow-report.ts b/packages/appkit/src/evals/mlflow-report.ts new file mode 100644 index 000000000..f801bbd65 --- /dev/null +++ b/packages/appkit/src/evals/mlflow-report.ts @@ -0,0 +1,113 @@ +import { normalizeHost } from "./mlflow-rest"; +import type { EvalResult } from "./types"; + +/** A Feedback assessment in the MLflow REST proto-JSON shape. */ +export interface Assessment { + trace_id: string; + assessment_name: string; + source: { source_type: "CODE" | "HUMAN" | "LLM_JUDGE"; source_id: string }; + feedback: { value: unknown }; + rationale?: string; + metadata?: Record; +} + +export interface MlflowReportOptions { + /** Databricks workspace host (scheme optional — normalized). */ + host: string; + /** Bearer token for the MLflow REST API. */ + token: string; +} + +export interface ReportOutcome { + written: number; + skipped: number; + failures: Array<{ traceId: string; status?: number; error?: string }>; +} + +/** + * Build the single pass/fail Feedback assessment for an eval result. Returns + * undefined when there's no trace to attach to or the eval was skipped. + */ +export function buildAssessment(result: EvalResult): Assessment | undefined { + if (!result.traceId || result.skipped) return undefined; + + const failed = result.assertions.filter((a) => !a.pass); + const rationale = result.error + ? `error: ${result.error}` + : failed.length + ? failed + .map( + (a) => + `${a.severity}:${a.label}${a.detail ? ` (${a.detail})` : ""}`, + ) + .join("; ") + : "all assertions passed"; + + return { + trace_id: result.traceId, + assessment_name: "appkit_eval", + source: { source_type: "CODE", source_id: "appkit-eval" }, + feedback: { value: result.passed }, + rationale, + metadata: { eval_id: result.id }, + }; +} + +async function postAssessment( + host: string, + token: string, + assessment: Assessment, +): Promise<{ ok: boolean; status?: number; error?: string }> { + const url = `${normalizeHost(host)}/api/3.0/mlflow/traces/${encodeURIComponent( + assessment.trace_id, + )}/assessments`; + try { + const res = await fetch(url, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ assessment }), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { ok: false, status: res.status, error: text.slice(0, 500) }; + } + return { ok: true }; + } catch (err) { + return { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +/** + * Write one pass/fail assessment per eval result to the Databricks MLflow REST + * API. Never throws — failures are collected so the run still reports. + */ +export async function reportToMlflow( + results: EvalResult[], + options: MlflowReportOptions, +): Promise { + const outcome: ReportOutcome = { written: 0, skipped: 0, failures: [] }; + for (const result of results) { + const assessment = buildAssessment(result); + if (!assessment) { + outcome.skipped++; + continue; + } + const res = await postAssessment(options.host, options.token, assessment); + if (res.ok) { + outcome.written++; + } else { + outcome.failures.push({ + traceId: assessment.trace_id, + status: res.status, + error: res.error, + }); + } + } + return outcome; +} diff --git a/packages/appkit/src/evals/mlflow-rest.ts b/packages/appkit/src/evals/mlflow-rest.ts new file mode 100644 index 000000000..a89912d24 --- /dev/null +++ b/packages/appkit/src/evals/mlflow-rest.ts @@ -0,0 +1,39 @@ +/** Shared helpers for talking to the Databricks/MLflow REST API. */ + +export interface MlflowRestOptions { + /** Databricks workspace host (scheme optional — normalized). */ + host: string; + /** Bearer token for the MLflow REST API. */ + token: string; +} + +/** Ensure the host has a scheme (Databricks env often lacks `https://`). */ +export function normalizeHost(raw: string): string { + const h = raw.trim().replace(/\/+$/, ""); + return /^https?:\/\//i.test(h) ? h : `https://${h}`; +} + +/** + * POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or throws + * with the status + response text so callers can surface a precise error. + */ +export async function mlflowPost( + options: MlflowRestOptions, + path: string, + body: unknown, +): Promise { + const res = await fetch(`${normalizeHost(options.host)}${path}`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${options.token}`, + }, + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`${path} -> ${res.status} ${text.slice(0, 500)}`); + } + const text = await res.text(); + return (text ? JSON.parse(text) : {}) as T; +} diff --git a/packages/appkit/src/evals/mlflow-run.ts b/packages/appkit/src/evals/mlflow-run.ts new file mode 100644 index 000000000..5806a322a --- /dev/null +++ b/packages/appkit/src/evals/mlflow-run.ts @@ -0,0 +1,117 @@ +import { type MlflowRestOptions, mlflowPost } from "./mlflow-rest"; +import type { EvalResult } from "./types"; + +/** Run tag value that makes a run appear under the experiment's "Evaluation runs". */ +const GENAI_EVALUATE_RUN_TYPE = "genai_evaluate"; + +interface CreateRunResponse { + run: { info: { run_id?: string; run_uuid?: string } }; +} + +interface MlflowMetric { + key: string; + value: number; + timestamp: number; + step: number; +} + +/** + * Create an MLflow run tagged as a GenAI evaluation, so it shows under the + * experiment's "Evaluation runs". Returns the run id; link traces to it via the + * `mlflow.sourceRun` trace metadata and log results before finishing. + */ +export async function createEvalRun( + options: MlflowRestOptions & { + experimentId: string; + runName?: string; + startTime: number; + }, +): Promise { + const created = await mlflowPost( + options, + "/api/2.0/mlflow/runs/create", + { + experiment_id: options.experimentId, + start_time: options.startTime, + ...(options.runName ? { run_name: options.runName } : {}), + }, + ); + const runId = created.run?.info?.run_id ?? created.run?.info?.run_uuid; + if (!runId) { + throw new Error("runs/create returned no run id"); + } + await mlflowPost(options, "/api/2.0/mlflow/runs/set-tag", { + run_id: runId, + key: "mlflow.runType", + value: GENAI_EVALUATE_RUN_TYPE, + }); + return runId; +} + +/** Aggregate per-eval results into MLflow run metrics. */ +export function aggregateMetrics( + results: EvalResult[], + timestamp: number, +): MlflowMetric[] { + const scored = results.filter((r) => !r.skipped); + const passed = scored.filter((r) => r.passed).length; + return [ + { key: "eval/total", value: results.length, timestamp, step: 0 }, + { key: "eval/scored", value: scored.length, timestamp, step: 0 }, + { key: "eval/passed", value: passed, timestamp, step: 0 }, + { + key: "eval/pass_rate", + value: scored.length ? passed / scored.length : 0, + timestamp, + step: 0, + }, + ]; +} + +export interface FinishOutcome { + finished: boolean; + /** Metric logging is best-effort; set when it failed (the run is still finished). */ + metricsError?: string; + /** Set when the FINISHED update itself failed (run may be left RUNNING). */ + finishError?: string; +} + +/** + * Log aggregate metrics (best-effort) and mark the eval run FINISHED. Never + * throws — a metric-logging failure must not prevent the run from being closed, + * or it would be left stuck in RUNNING forever. + */ +export async function finishEvalRun( + options: MlflowRestOptions & { + runId: string; + results: EvalResult[]; + endTime: number; + }, +): Promise { + const outcome: FinishOutcome = { finished: false }; + + const metrics = aggregateMetrics(options.results, options.endTime); + if (metrics.length) { + try { + await mlflowPost(options, "/api/2.0/mlflow/runs/log-batch", { + run_id: options.runId, + metrics, + }); + } catch (err) { + outcome.metricsError = err instanceof Error ? err.message : String(err); + } + } + + try { + await mlflowPost(options, "/api/2.0/mlflow/runs/update", { + run_id: options.runId, + status: "FINISHED", + end_time: options.endTime, + }); + outcome.finished = true; + } catch (err) { + outcome.finishError = err instanceof Error ? err.message : String(err); + } + + return outcome; +} diff --git a/packages/appkit/src/evals/report.ts b/packages/appkit/src/evals/report.ts new file mode 100644 index 000000000..2e42c5b1c --- /dev/null +++ b/packages/appkit/src/evals/report.ts @@ -0,0 +1,76 @@ +import type { EvalResult } from "./types"; + +export interface EvalSummary { + total: number; + passed: number; + failed: number; + skipped: number; + /** True when no eval failed (skips don't count as failures). */ + allPassed: boolean; +} + +export function summarize(results: EvalResult[]): EvalSummary { + let passed = 0; + let failed = 0; + let skipped = 0; + for (const r of results) { + if (r.skipped) skipped++; + else if (r.passed) passed++; + else failed++; + } + return { + total: results.length, + passed, + failed, + skipped, + allPassed: failed === 0, + }; +} + +/** Status glyph for a single eval result. */ +export function evalGlyph(result: EvalResult): string { + if (result.skipped) return "−"; + return result.passed ? "✓" : "✗"; +} + +/** The one-line header for a single eval result (no failure detail). */ +export function formatEvalHeadline(result: EvalResult): string { + if (result.skipped) { + return `− ${result.id} (skipped${ + result.skipped.reason ? `: ${result.skipped.reason}` : "" + })`; + } + return `${evalGlyph(result)} ${result.id}${ + result.description ? ` — ${result.description}` : "" + }`; +} + +/** Indented detail lines for a failing eval (error + failing assertions). */ +export function formatEvalDetail(result: EvalResult): string[] { + const lines: string[] = []; + if (result.error) lines.push(` error: ${result.error}`); + for (const a of result.assertions) { + if (a.pass) continue; + const tag = a.severity === "soft" ? "soft" : "gate"; + lines.push(` ✗ [${tag}] ${a.label}${a.detail ? ` — ${a.detail}` : ""}`); + } + return lines; +} + +/** The final PASS/FAIL summary line. */ +export function formatSummaryLine(results: EvalResult[]): string { + const s = summarize(results); + return `${s.allPassed ? "PASS" : "FAIL"} — ${s.passed} passed, ${s.failed} failed, ${s.skipped} skipped (${s.total} total)`; +} + +/** Render all results as a human-readable console report (non-streaming). */ +export function formatEvalResults(results: EvalResult[]): string { + const lines: string[] = []; + for (const r of results) { + lines.push(formatEvalHeadline(r)); + lines.push(...formatEvalDetail(r)); + } + lines.push(""); + lines.push(formatSummaryLine(results)); + return lines.join("\n"); +} diff --git a/packages/appkit/src/evals/run-eval.ts b/packages/appkit/src/evals/run-eval.ts new file mode 100644 index 000000000..637508f20 --- /dev/null +++ b/packages/appkit/src/evals/run-eval.ts @@ -0,0 +1,155 @@ +import type { + AssertionHandle, + AssertionResult, + EvalDefinition, + EvalDriver, + EvalResult, + Matcher, + TestContext, +} from "./types"; + +/** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */ +class SkipSignal extends Error { + constructor(public reason?: string) { + super("eval skipped"); + this.name = "SkipSignal"; + } +} + +export interface RunEvalOptions { + /** Stable id for the eval (e.g. its file path relative to the evals dir). */ + id: string; + /** Drives the agent and returns reply/tool-calls/success per `send`. */ + driver: EvalDriver; + /** When true, soft assertion failures also fail the eval. */ + strict?: boolean; +} + +/** + * Run a single eval against a driver. Never throws for assertion or agent + * failures — those become a non-passing {@link EvalResult}. Only a malformed + * eval definition surfaces as `result.error`. + */ +export async function runEval( + def: EvalDefinition, + options: RunEvalOptions, +): Promise { + const assertions: AssertionResult[] = []; + let reply = ""; + let toolCalls: string[] = []; + let sessionId: string | undefined; + let lastTraceId: string | undefined; + let lastSucceeded = false; + + const record = ( + label: string, + pass: boolean, + score?: number, + detail?: string, + ): AssertionHandle => { + const result: AssertionResult = { + label, + severity: "gate", + pass, + score, + detail, + }; + assertions.push(result); + const handle: AssertionHandle = { + gate() { + result.severity = "gate"; + return handle; + }, + soft() { + result.severity = "soft"; + return handle; + }, + atLeast(threshold: number) { + result.severity = "soft"; + result.pass = (result.score ?? (result.pass ? 1 : 0)) >= threshold; + return handle; + }, + }; + return handle; + }; + + const t: TestContext = { + async send(message) { + const r = await options.driver.send(message); + reply = r.reply; + toolCalls = r.toolCalls; + sessionId = r.sessionId; + lastSucceeded = r.succeeded; + if (r.traceId) lastTraceId = r.traceId; + }, + get reply() { + return reply; + }, + get toolCalls() { + return toolCalls; + }, + get sessionId() { + return sessionId; + }, + succeeded() { + return record( + "succeeded", + lastSucceeded, + undefined, + lastSucceeded ? undefined : "agent turn did not complete successfully", + ); + }, + calledTool(name) { + return record( + `calledTool(${name})`, + toolCalls.includes(name), + undefined, + `expected tool "${name}" to be called (called: ${ + toolCalls.length ? toolCalls.join(", ") : "none" + })`, + ); + }, + check(value: string, matcher: Matcher) { + const m = matcher(value); + return record("check", m.pass, m.score, m.detail); + }, + skip(reason) { + throw new SkipSignal(reason); + }, + }; + + try { + await def.test(t); + } catch (err) { + if (err instanceof SkipSignal) { + return { + id: options.id, + description: def.description, + skipped: { reason: err.reason }, + assertions, + passed: true, + traceId: lastTraceId, + }; + } + return { + id: options.id, + description: def.description, + assertions, + passed: false, + error: err instanceof Error ? err.message : String(err), + traceId: lastTraceId, + }; + } + + const passed = assertions.every( + (a) => a.pass || (a.severity === "soft" && !options.strict), + ); + + return { + id: options.id, + description: def.description, + assertions, + passed, + traceId: lastTraceId, + }; +} diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts new file mode 100644 index 000000000..4cbd51c01 --- /dev/null +++ b/packages/appkit/src/evals/run-evals.ts @@ -0,0 +1,166 @@ +import { pathToFileURL } from "node:url"; +import { discoverEvalFiles } from "./discover"; +import { createHttpDriver } from "./http-driver"; +import { type ReportOutcome, reportToMlflow } from "./mlflow-report"; +import { createEvalRun, type FinishOutcome, finishEvalRun } from "./mlflow-run"; +import { runEval } from "./run-eval"; +import type { EvalDefinition, EvalResult } from "./types"; + +export interface RunEvalsOptions { + /** Project root containing `config/agents/`. Defaults to `process.cwd()`. */ + rootDir?: string; + /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */ + baseUrl: string; + /** Substring filter on `/` (or an exact agent id). */ + filter?: string; + /** Soft assertion failures also fail the eval. */ + strict?: boolean; + /** Extra request headers for the driver (e.g. auth for a deployed app). */ + headers?: Record; + /** + * When set, create a native MLflow "Evaluation run": each eval's trace is + * linked to the run, pass/fail is written as feedback, and aggregate metrics + * are logged. Requires Databricks creds + the target experiment. + */ + mlflow?: { host: string; token: string; experimentId: string }; + /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */ + now?: number; + /** Progress callback, invoked as evals are discovered, started, and finished. */ + onEvent?: (event: EvalProgress) => void; +} + +export type EvalProgress = + | { type: "discovered"; total: number } + | { type: "run-created"; runId: string } + | { type: "start"; id: string; index: number; total: number } + | { type: "result"; result: EvalResult; index: number; total: number }; + +export interface EvalRunSummary { + results: EvalResult[]; + /** Present when an MLflow evaluation run was created. */ + mlflow?: { runId: string; report: ReportOutcome; finish: FinishOutcome }; +} + +/** + * Load a `*.eval.ts` file and return its default-exported {@link EvalDefinition}. + * Uses tsx's programmatic loader so TypeScript eval files run without a build + * step. The specifier is indirected so the type checker doesn't try to resolve + * tsx's internal entry. + */ +async function loadEval(file: string): Promise { + const tsxApi = "tsx/esm/api"; + let tsImport: (specifier: string, parentURL: string) => Promise; + try { + ({ tsImport } = (await import(tsxApi)) as { + tsImport: (specifier: string, parentURL: string) => Promise; + }); + } catch { + throw new Error( + "Running .eval.ts files requires `tsx`. Install it as a dev dependency (`pnpm add -D tsx`).", + ); + } + + const mod = await tsImport(pathToFileURL(file).href, import.meta.url); + const def = resolveEvalDefault(mod); + if (!def) { + throw new Error(`${file}: must default-export defineEval({ test })`); + } + return def; +} + +/** + * Unwrap the eval default export across module-interop shapes. Depending on + * whether the eval file is treated as ESM or CJS, the value lands at + * `mod.default` (ESM), `mod.default.default` (CJS `__esModule` double-wrap), or + * `mod` itself. Returns the first candidate that looks like an eval. + */ +export function resolveEvalDefault(mod: unknown): EvalDefinition | undefined { + const seen = new Set(); + let candidate: unknown = mod; + for (let i = 0; i < 4 && candidate && !seen.has(candidate); i++) { + if (typeof (candidate as EvalDefinition).test === "function") { + return candidate as EvalDefinition; + } + seen.add(candidate); + candidate = (candidate as { default?: unknown }).default; + } + return undefined; +} + +/** + * Discover, load, and run every eval under each agent's `evals/` dir, driving + * the agents on a running app. Never throws for an individual eval — load/run + * failures become non-passing {@link EvalResult}s. + */ +export async function runEvalsInDir( + options: RunEvalsOptions, +): Promise { + const root = options.rootDir ?? process.cwd(); + const now = options.now ?? Date.now(); + let discovered = discoverEvalFiles(root); + + if (options.filter) { + const f = options.filter; + discovered = discovered.filter( + (d) => d.agent === f || `${d.agent}/${d.id}`.includes(f), + ); + } + + const emit = options.onEvent ?? (() => {}); + const total = discovered.length; + emit({ type: "discovered", total }); + + // Create the MLflow evaluation run up front so each eval's trace can be + // linked to it as it runs. + let runId: string | undefined; + if (options.mlflow) { + runId = await createEvalRun({ + ...options.mlflow, + runName: `appkit-eval ${new Date(now).toISOString()}`, + startTime: now, + }); + emit({ type: "run-created", runId }); + } + + const results: EvalResult[] = []; + for (let index = 0; index < discovered.length; index++) { + const d = discovered[index]; + const id = `${d.agent}/${d.id}`; + emit({ type: "start", id, index, total }); + + let result: EvalResult; + try { + const def = await loadEval(d.file); + const driver = createHttpDriver({ + baseUrl: options.baseUrl, + agent: def.agent ?? d.agent, + headers: options.headers, + mlflowRunId: runId, + }); + result = await runEval(def, { id, driver, strict: options.strict }); + } catch (err) { + result = { + id, + assertions: [], + passed: false, + error: err instanceof Error ? err.message : String(err), + }; + } + + results.push(result); + emit({ type: "result", result, index, total }); + } + + if (options.mlflow && runId) { + const report = await reportToMlflow(results, options.mlflow); + const finish = await finishEvalRun({ + ...options.mlflow, + runId, + results, + endTime: options.now ?? Date.now(), + }); + return { results, mlflow: { runId, report, finish } }; + } + + return { results }; +} diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts new file mode 100644 index 000000000..6eb7c2208 --- /dev/null +++ b/packages/appkit/src/evals/tests/discover.test.ts @@ -0,0 +1,42 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { discoverEvalFiles } from "../discover"; + +let root: string; + +function write(rel: string, content = "export default {}") { + const full = path.join(root, rel); + mkdirSync(path.dirname(full), { recursive: true }); + writeFileSync(full, content); +} + +beforeEach(() => { + root = mkdtempSync(path.join(tmpdir(), "appkit-evals-")); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("discoverEvalFiles", () => { + test("finds *.eval.ts per agent, derives id + agent, ignores config", () => { + write("config/agents/support/evals/basic.eval.ts"); + write("config/agents/support/evals/nested/deep.eval.ts"); + write("config/agents/support/evals/evals.config.ts"); + write("config/agents/analyst/evals/sql.eval.ts"); + write("config/agents/no-evals/agent.md", "# agent"); + + const found = discoverEvalFiles(root); + + expect(found.map((f) => `${f.agent}/${f.id}`)).toEqual([ + "analyst/sql", + "support/basic", + "support/nested/deep", + ]); + }); + + test("returns empty when there is no config/agents dir", () => { + expect(discoverEvalFiles(root)).toEqual([]); + }); +}); diff --git a/packages/appkit/src/evals/tests/matchers.test.ts b/packages/appkit/src/evals/tests/matchers.test.ts new file mode 100644 index 000000000..03653b520 --- /dev/null +++ b/packages/appkit/src/evals/tests/matchers.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "vitest"; +import { equals, includes, matches } from "../matchers"; + +describe("eval matchers", () => { + test("includes", () => { + expect(includes("Sunny")("It is Sunny today").pass).toBe(true); + expect(includes("Rainy")("It is Sunny today").pass).toBe(false); + }); + + test("equals", () => { + expect(equals("yes")("yes").pass).toBe(true); + expect(equals("yes")("Yes").pass).toBe(false); + }); + + test("matches", () => { + expect(matches(/^\d+ rows$/)("42 rows").pass).toBe(true); + expect(matches(/^\d+ rows$/)("forty rows").pass).toBe(false); + }); +}); diff --git a/packages/appkit/src/evals/tests/mlflow-report.test.ts b/packages/appkit/src/evals/tests/mlflow-report.test.ts new file mode 100644 index 000000000..141556400 --- /dev/null +++ b/packages/appkit/src/evals/tests/mlflow-report.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, test } from "vitest"; +import { buildAssessment } from "../mlflow-report"; +import type { EvalResult } from "../types"; + +describe("buildAssessment", () => { + test("builds a pass assessment with trace id and source", () => { + const result: EvalResult = { + id: "support/weather", + traceId: "tr-123", + assertions: [{ label: "succeeded", severity: "gate", pass: true }], + passed: true, + }; + const a = buildAssessment(result); + expect(a).toEqual({ + trace_id: "tr-123", + assessment_name: "appkit_eval", + source: { source_type: "CODE", source_id: "appkit-eval" }, + feedback: { value: true }, + rationale: "all assertions passed", + metadata: { eval_id: "support/weather" }, + }); + }); + + test("fail assessment summarizes failing assertions in the rationale", () => { + const a = buildAssessment({ + id: "support/x", + traceId: "tr-9", + assertions: [ + { label: "succeeded", severity: "gate", pass: true }, + { + label: "calledTool(get_weather)", + severity: "gate", + pass: false, + detail: "not called", + }, + ], + passed: false, + }); + expect(a?.feedback.value).toBe(false); + expect(a?.rationale).toContain("gate:calledTool(get_weather) (not called)"); + }); + + test("returns undefined without a trace id or when skipped", () => { + expect( + buildAssessment({ id: "x", assertions: [], passed: true }), + ).toBeUndefined(); + expect( + buildAssessment({ + id: "x", + traceId: "tr-1", + assertions: [], + passed: true, + skipped: { reason: "no data" }, + }), + ).toBeUndefined(); + }); +}); diff --git a/packages/appkit/src/evals/tests/mlflow-run.test.ts b/packages/appkit/src/evals/tests/mlflow-run.test.ts new file mode 100644 index 000000000..8c4318cd6 --- /dev/null +++ b/packages/appkit/src/evals/tests/mlflow-run.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "vitest"; +import { aggregateMetrics } from "../mlflow-run"; +import type { EvalResult } from "../types"; + +describe("aggregateMetrics", () => { + test("counts total/scored/passed and computes pass_rate (skips excluded)", () => { + const results: EvalResult[] = [ + { id: "a", assertions: [], passed: true }, + { id: "b", assertions: [], passed: false }, + { id: "c", assertions: [], passed: true, skipped: { reason: "x" } }, + ]; + const metrics = aggregateMetrics(results, 1000); + const byKey = Object.fromEntries(metrics.map((m) => [m.key, m.value])); + expect(byKey["eval/total"]).toBe(3); + expect(byKey["eval/scored"]).toBe(2); + expect(byKey["eval/passed"]).toBe(1); + expect(byKey["eval/pass_rate"]).toBe(0.5); + expect(metrics.every((m) => m.timestamp === 1000 && m.step === 0)).toBe( + true, + ); + }); + + test("pass_rate is 0 when nothing is scored", () => { + const metrics = aggregateMetrics( + [{ id: "a", assertions: [], passed: true, skipped: {} }], + 1, + ); + expect(metrics.find((m) => m.key === "eval/pass_rate")?.value).toBe(0); + }); +}); diff --git a/packages/appkit/src/evals/tests/report.test.ts b/packages/appkit/src/evals/tests/report.test.ts new file mode 100644 index 000000000..c02c68843 --- /dev/null +++ b/packages/appkit/src/evals/tests/report.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "vitest"; +import { formatEvalResults, summarize } from "../report"; +import type { EvalResult } from "../types"; + +const results: EvalResult[] = [ + { + id: "a/pass", + assertions: [{ label: "succeeded", severity: "gate", pass: true }], + passed: true, + }, + { + id: "a/fail", + assertions: [ + { + label: "calledTool(x)", + severity: "gate", + pass: false, + detail: "not called", + }, + ], + passed: false, + }, + { + id: "a/skip", + assertions: [], + passed: true, + skipped: { reason: "no data" }, + }, +]; + +describe("eval reporting", () => { + test("summarize counts pass/fail/skip and allPassed", () => { + expect(summarize(results)).toEqual({ + total: 3, + passed: 1, + failed: 1, + skipped: 1, + allPassed: false, + }); + }); + + test("summarize allPassed is true when nothing failed", () => { + expect(summarize([results[0], results[2]]).allPassed).toBe(true); + }); + + test("formatEvalResults shows status, failing assertions, and a summary line", () => { + const out = formatEvalResults(results); + expect(out).toContain("✓ a/pass"); + expect(out).toContain("✗ a/fail"); + expect(out).toContain("[gate] calledTool(x) — not called"); + expect(out).toContain("a/skip (skipped: no data)"); + expect(out).toContain("FAIL — 1 passed, 1 failed, 1 skipped (3 total)"); + }); +}); diff --git a/packages/appkit/src/evals/tests/resolve-default.test.ts b/packages/appkit/src/evals/tests/resolve-default.test.ts new file mode 100644 index 000000000..40079226c --- /dev/null +++ b/packages/appkit/src/evals/tests/resolve-default.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, test } from "vitest"; +import { resolveEvalDefault } from "../run-evals"; + +const def = { description: "x", test: async () => {} }; + +describe("resolveEvalDefault (module interop)", () => { + test("pure ESM: mod.default", () => { + expect(resolveEvalDefault({ default: def })).toBe(def); + }); + + test("CJS __esModule double-wrap: mod.default.default", () => { + expect( + resolveEvalDefault({ default: { __esModule: true, default: def } }), + ).toBe(def); + }); + + test("direct: mod is the def", () => { + expect(resolveEvalDefault(def)).toBe(def); + }); + + test("no default export → undefined", () => { + expect(resolveEvalDefault({ default: { notTest: 1 } })).toBeUndefined(); + expect(resolveEvalDefault(null)).toBeUndefined(); + }); +}); diff --git a/packages/appkit/src/evals/tests/run-eval.test.ts b/packages/appkit/src/evals/tests/run-eval.test.ts new file mode 100644 index 000000000..abc780302 --- /dev/null +++ b/packages/appkit/src/evals/tests/run-eval.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "vitest"; +import { defineEval } from "../define-eval"; +import { includes } from "../matchers"; +import { runEval } from "../run-eval"; +import type { DriveResult, EvalDriver } from "../types"; + +/** Driver that returns a fixed result for every send. */ +function fakeDriver(result: Partial): EvalDriver { + return { + send: async () => ({ + reply: "", + toolCalls: [], + succeeded: true, + ...result, + }), + }; +} + +describe("runEval", () => { + test("passes when all gate assertions pass", async () => { + const def = defineEval({ + async test(t) { + await t.send("weather?"); + t.succeeded(); + t.calledTool("get_weather"); + t.check(t.reply, includes("Sunny")); + }, + }); + const result = await runEval(def, { + id: "weather", + driver: fakeDriver({ + reply: "It is Sunny", + toolCalls: ["get_weather"], + succeeded: true, + }), + }); + expect(result.passed).toBe(true); + expect(result.assertions).toHaveLength(3); + expect(result.assertions.every((a) => a.pass)).toBe(true); + }); + + test("fails when a gate assertion fails", async () => { + const def = defineEval({ + async test(t) { + await t.send("weather?"); + t.calledTool("get_weather"); + }, + }); + const result = await runEval(def, { + id: "no-tool", + driver: fakeDriver({ reply: "I don't know", toolCalls: [] }), + }); + expect(result.passed).toBe(false); + expect(result.assertions[0].pass).toBe(false); + }); + + test("soft failures don't fail the eval unless strict", async () => { + const def = defineEval({ + async test(t) { + await t.send("hi"); + t.calledTool("get_weather").soft(); + }, + }); + const lenient = await runEval(def, { + id: "soft", + driver: fakeDriver({ toolCalls: [] }), + }); + expect(lenient.passed).toBe(true); + + const strict = await runEval(def, { + id: "soft", + driver: fakeDriver({ toolCalls: [] }), + strict: true, + }); + expect(strict.passed).toBe(false); + }); + + test("succeeded() reflects the driver's success flag", async () => { + const def = defineEval({ + async test(t) { + await t.send("hi"); + t.succeeded(); + }, + }); + const result = await runEval(def, { + id: "failed-turn", + driver: fakeDriver({ succeeded: false }), + }); + expect(result.passed).toBe(false); + }); + + test("t.skip marks the eval skipped and passing", async () => { + const def = defineEval({ + test(t) { + t.skip("no fixture data"); + }, + }); + const result = await runEval(def, { + id: "skipped", + driver: fakeDriver({}), + }); + expect(result.skipped?.reason).toBe("no fixture data"); + expect(result.passed).toBe(true); + expect(result.assertions).toHaveLength(0); + }); + + test("a thrown error becomes a non-passing result, not an exception", async () => { + const def = defineEval({ + async test() { + throw new Error("boom"); + }, + }); + const result = await runEval(def, { id: "boom", driver: fakeDriver({}) }); + expect(result.passed).toBe(false); + expect(result.error).toBe("boom"); + }); +}); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts new file mode 100644 index 000000000..c6ef02e1a --- /dev/null +++ b/packages/appkit/src/evals/types.ts @@ -0,0 +1,128 @@ +/** + * Agent evaluation primitives — an eve-style authoring API that runs against + * AppKit agents and reports to Databricks MLflow. + * + * Evals live in `config/agents//evals/*.eval.ts`, each default-exporting a + * {@link EvalDefinition} via {@link defineEval}. A runner drives the agent + * (today over HTTP against a running app), and the `test` function asserts on + * the reply and tool usage with deterministic matchers (and, later, LLM judges). + */ + +/** Result of a deterministic matcher run against a value. */ +export interface MatchResult { + pass: boolean; + /** Optional 0..1 score for scored matchers (similarity, judges). */ + score?: number; + /** Human-readable explanation, shown on failure. */ + detail?: string; +} + +/** A deterministic matcher: inspects a string value and returns a result. */ +export type Matcher = (value: string) => MatchResult; + +/** Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). */ +export type Severity = "gate" | "soft"; + +/** A single recorded assertion outcome. */ +export interface AssertionResult { + label: string; + severity: Severity; + pass: boolean; + score?: number; + detail?: string; +} + +/** + * Chainable handle returned by every assertion to control its severity. + * Mirrors eve: assertions are gates by default; `.soft()` demotes to a tracked + * metric; `.atLeast(n)` is a soft, score-thresholded assertion. + */ +export interface AssertionHandle { + /** Promote to a hard gate — failure fails the eval (non-zero exit). */ + gate(): AssertionHandle; + /** Demote to a tracked metric — doesn't fail unless running with `strict`. */ + soft(): AssertionHandle; + /** Soft assertion that passes only when the score is at least `threshold`. */ + atLeast(threshold: number): AssertionHandle; +} + +/** What a driver returns for a single `t.send`. */ +export interface DriveResult { + /** The final assistant message text. */ + reply: string; + /** Names of tools the agent called during the turn. */ + toolCalls: string[]; + /** Whether the turn completed without an agent/stream error. */ + succeeded: boolean; + /** Thread/session id, when the driver exposes one. */ + sessionId?: string; + /** MLflow trace id for the turn, when tracing is enabled on the app. */ + traceId?: string; +} + +/** + * Abstraction over how the agent is driven. The HTTP driver posts to a running + * app's agents endpoint; future drivers (in-process) implement the same shape. + */ +export interface EvalDriver { + send(message: string): Promise; +} + +/** The `t` context passed to an eval's `test` function. */ +export interface TestContext { + /** Send a user message to the agent and capture its response. */ + send(message: string): Promise; + /** The last assistant reply. */ + readonly reply: string; + /** Tools called during the last turn. */ + readonly toolCalls: string[]; + /** The current session/thread id, if any. */ + readonly sessionId: string | undefined; + /** Assert the last turn completed successfully (gate by default). */ + succeeded(): AssertionHandle; + /** Assert a tool was called during the run (gate by default). */ + calledTool(name: string): AssertionHandle; + /** Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. */ + check(value: string, matcher: Matcher): AssertionHandle; + /** Skip this eval with an optional reason. */ + skip(reason?: string): never; +} + +/** A single eval, default-exported from a `*.eval.ts` file. */ +export interface EvalDefinition { + /** Short human description, shown in reports. */ + description?: string; + /** Target agent id. Defaults to the eval's parent `config/agents/` dir. */ + agent?: string; + /** Free-form tags for filtering. */ + tags?: string[]; + /** Per-eval timeout. */ + timeoutMs?: number; + /** The eval body: drive the agent and assert on its behavior. */ + test(t: TestContext): Promise | void; +} + +/** Per-directory config from `evals.config.ts`. */ +export interface EvalConfig { + /** LLM judge config. Defaults to the agent's own serving endpoint. */ + judge?: { model?: string }; + /** Max evals to run concurrently. */ + maxConcurrency?: number; + /** Default per-eval timeout. */ + timeoutMs?: number; +} + +/** The outcome of running one eval. */ +export interface EvalResult { + id: string; + description?: string; + /** Set when the eval called `t.skip`. */ + skipped?: { reason?: string }; + assertions: AssertionResult[]; + /** True when all gates passed (and, under strict, all soft assertions too). */ + passed: boolean; + /** Set when the eval threw before completing. */ + error?: string; + /** MLflow trace id of the eval's last turn, for attaching assessments. */ + traceId?: string; +} diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts new file mode 100644 index 000000000..ee6924230 --- /dev/null +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -0,0 +1,191 @@ +import { Command } from "commander"; + +interface EvalRunSummary { + results: unknown[]; + mlflow?: { + runId: string; + report: { + written: number; + skipped: number; + failures: Array<{ traceId: string; status?: number; error?: string }>; + }; + finish: { finished: boolean; metricsError?: string; finishError?: string }; + }; +} + +type EvalProgress = + | { type: "discovered"; total: number } + | { type: "run-created"; runId: string } + | { type: "start"; id: string; index: number; total: number } + | { type: "result"; result: unknown; index: number; total: number }; + +/** Subset of `@databricks/appkit/beta`'s eval runner used by this command. */ +interface EvalRunner { + runEvalsInDir(opts: { + rootDir?: string; + baseUrl: string; + filter?: string; + strict?: boolean; + headers?: Record; + mlflow?: { host: string; token: string; experimentId: string }; + onEvent?: (event: EvalProgress) => void; + }): Promise; + evalGlyph(result: unknown): string; + formatEvalDetail(result: unknown): string[]; + formatSummaryLine(results: unknown[]): string; + summarize(results: unknown[]): { allPassed: boolean }; +} + +/** + * Loaded at runtime from the consuming project so this command (which ships in + * `@databricks/shared`) doesn't take a build-time dependency on appkit. The + * specifier is a variable so the type checker treats it as `any`. + */ +async function loadRunner(): Promise { + const spec = "@databricks/appkit/beta"; + try { + return (await import(spec)) as unknown as EvalRunner; + } catch (err) { + throw new Error( + "Could not load @databricks/appkit. Run `appkit agent eval` from a " + + "project with @databricks/appkit installed. " + + `Cause: ${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +function parseHeaders(values: string[]): Record { + const headers: Record = {}; + for (const v of values) { + const i = v.indexOf(":"); + if (i === -1) continue; + headers[v.slice(0, i).trim()] = v.slice(i + 1).trim(); + } + return headers; +} + +interface EvalOptions { + url: string; + strict?: boolean; + root?: string; + header?: string[]; + databricksHost?: string; + databricksToken?: string; + experiment?: string; +} + +async function runAgentEval( + filter: string | undefined, + opts: EvalOptions, +): Promise { + const runner = await loadRunner(); + + // Create a native MLflow "Evaluation run" when Databricks creds + an + // experiment are available (traces live in the app; the run + scores are + // driven from here, so this side needs creds). + const host = opts.databricksHost ?? process.env.DATABRICKS_HOST; + const token = opts.databricksToken ?? process.env.DATABRICKS_TOKEN; + const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID; + const mlflow = + host && token && experimentId ? { host, token, experimentId } : undefined; + + // Stream progress as evals run, instead of going silent until the end. + const onEvent = (event: EvalProgress): void => { + switch (event.type) { + case "discovered": + console.log( + `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${opts.url}\n`, + ); + break; + case "run-created": + console.log(`MLflow evaluation run: ${event.runId}\n`); + break; + case "start": + process.stdout.write( + `▸ [${event.index + 1}/${event.total}] ${event.id} … `, + ); + break; + case "result": { + process.stdout.write(`${runner.evalGlyph(event.result)}\n`); + for (const line of runner.formatEvalDetail(event.result)) { + console.log(line); + } + break; + } + } + }; + + const summary = await runner.runEvalsInDir({ + rootDir: opts.root, + baseUrl: opts.url, + filter, + strict: opts.strict, + headers: opts.header ? parseHeaders(opts.header) : undefined, + mlflow, + onEvent, + }); + console.log(`\n${runner.formatSummaryLine(summary.results)}`); + + if (summary.mlflow) { + const { report, finish } = summary.mlflow; + console.log( + `MLflow: ${report.written} assessment(s) written` + + (report.skipped ? `, ${report.skipped} skipped` : "") + + (report.failures.length ? `, ${report.failures.length} failed` : ""), + ); + for (const f of report.failures) { + console.error( + ` ✗ trace ${f.traceId}: ${f.status ?? ""} ${f.error ?? ""}`.trim(), + ); + } + if (finish.metricsError) { + console.error(` ⚠ metrics not logged: ${finish.metricsError}`); + } + if (!finish.finished) { + console.error( + ` ✗ run left RUNNING — failed to finish: ${finish.finishError ?? "unknown"}`, + ); + } + } else { + console.log( + "\nMLflow evaluation run skipped — set DATABRICKS_HOST + DATABRICKS_TOKEN" + + " + MLFLOW_EXPERIMENT_ID (or the matching flags) to create one.", + ); + } + + if (!runner.summarize(summary.results).allPassed) { + process.exitCode = 1; + } +} + +export const agentEvalCommand = new Command("eval") + .description( + "Run agent evals (config/agents//evals/*.eval.ts) against a running app", + ) + .argument( + "[filter]", + "Only run evals whose / contains this substring (or an exact agent id)", + ) + .option("--url ", "Base URL of the running app", "http://localhost:3000") + .option("--strict", "Fail on soft-assertion misses too", false) + .option( + "--root ", + "Project root containing config/agents/ (default: cwd)", + ) + .option( + "--header ", + "Extra request header as 'Key: value' (repeatable)", + ) + .option( + "--databricks-host ", + "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)", + ) + .option( + "--databricks-token ", + "Databricks token for writing MLflow assessments (default: DATABRICKS_TOKEN)", + ) + .option( + "--experiment ", + "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)", + ) + .action(runAgentEval); diff --git a/packages/shared/src/cli/commands/agent/index.ts b/packages/shared/src/cli/commands/agent/index.ts new file mode 100644 index 000000000..3f31a92d8 --- /dev/null +++ b/packages/shared/src/cli/commands/agent/index.ts @@ -0,0 +1,19 @@ +import { Command } from "commander"; +import { agentEvalCommand } from "./eval"; + +/** + * Parent command for agent development operations. + * Subcommands: + * - eval: Run agent evals against a running app + */ +export const agentCommand = new Command("agent") + .description("Agent development commands") + .addCommand(agentEvalCommand) + .addHelpText( + "after", + ` +Examples: + $ appkit agent eval + $ appkit agent eval support --strict + $ appkit agent eval --url https://my-app.databricksapps.com`, + ); diff --git a/packages/shared/src/cli/index.ts b/packages/shared/src/cli/index.ts index 8c4649419..a228b1f90 100644 --- a/packages/shared/src/cli/index.ts +++ b/packages/shared/src/cli/index.ts @@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url"; import { Command } from "commander"; +import { agentCommand } from "./commands/agent/index.js"; import { codemodCommand } from "./commands/codemod/index.js"; import { docsCommand } from "./commands/docs.js"; import { doctorCommand } from "./commands/doctor/index.js"; @@ -38,5 +39,6 @@ cmd.addCommand(doctorCommand); // is still in development (registry + add work end-to-end but aren't announced). cmd.addCommand(registryCommand, { hidden: true }); cmd.addCommand(addCommand, { hidden: true }); +cmd.addCommand(agentCommand); await cmd.parseAsync(); From 53da13e5eb25f661cd5be13cdee39da2aa216012 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Mon, 29 Jun 2026 17:55:16 +0200 Subject: [PATCH 02/14] feat(appkit): add llm-as-judge and mlflow trace/eval parity Extend the agent eval framework and tighten MLflow output to match the native `mlflow.genai.evaluate` experience: - LLM-as-judge via autoevals (factuality, closedQA, custom), pointed at a Databricks serving endpoint; exposed through `t.judge.*`. - One Feedback assessment per assertion (judges as LLM_JUDGE with score + rationale) plus an overall `appkit_eval`; assessment names sanitized to `[A-Za-z0-9_-]` since the API rejects dots. - Trace-table parity: set Request/Response previews and the `mlflow.traceName` tag (the Trace-name column reads the tag, not the span name). - Eval runs carry `mlflow.source.name`/`type` tags so linked traces show Source and Run name; live chat traces have no run so those stay empty. - Example judge eval under config/agents/query/evals. Signed-off-by: MarioCadenas --- .../config/agents/query/evals/judge.eval.ts | 25 ++ .../api/appkit/Function.buildAssessment.md | 18 ++ .../api/appkit/Function.createHttpDriver.md | 20 ++ docs/docs/api/appkit/Function.defineEval.md | 34 ++ .../api/appkit/Function.defineEvalConfig.md | 17 + .../api/appkit/Function.discoverEvalFiles.md | 19 ++ docs/docs/api/appkit/Function.equals.md | 17 + docs/docs/api/appkit/Function.evalGlyph.md | 17 + .../api/appkit/Function.formatEvalDetail.md | 17 + .../api/appkit/Function.formatEvalHeadline.md | 17 + .../api/appkit/Function.formatEvalResults.md | 17 + .../api/appkit/Function.formatSummaryLine.md | 17 + docs/docs/api/appkit/Function.includes.md | 17 + docs/docs/api/appkit/Function.matches.md | 17 + .../api/appkit/Function.reportToMlflow.md | 19 ++ docs/docs/api/appkit/Function.runEval.md | 20 ++ .../docs/api/appkit/Function.runEvalsInDir.md | 19 ++ docs/docs/api/appkit/Function.summarize.md | 15 + .../api/appkit/Interface.AssertionHandle.md | 53 ++++ .../api/appkit/Interface.AssertionResult.md | 43 +++ docs/docs/api/appkit/Interface.Assessment.md | 74 +++++ .../api/appkit/Interface.DiscoveredEval.md | 33 ++ docs/docs/api/appkit/Interface.DriveResult.md | 53 ++++ docs/docs/api/appkit/Interface.EvalConfig.md | 41 +++ .../api/appkit/Interface.EvalDefinition.md | 63 ++++ docs/docs/api/appkit/Interface.EvalDriver.md | 22 ++ docs/docs/api/appkit/Interface.EvalResult.md | 75 +++++ .../api/appkit/Interface.EvalRunSummary.md | 41 +++ docs/docs/api/appkit/Interface.EvalSummary.md | 43 +++ .../api/appkit/Interface.HttpDriverOptions.md | 51 +++ docs/docs/api/appkit/Interface.MatchResult.md | 31 ++ .../appkit/Interface.MlflowReportOptions.md | 21 ++ .../api/appkit/Interface.ReportOutcome.md | 47 +++ .../api/appkit/Interface.RunEvalOptions.md | 31 ++ .../api/appkit/Interface.RunEvalsOptions.md | 115 +++++++ docs/docs/api/appkit/Interface.TestContext.md | 128 ++++++++ .../docs/api/appkit/TypeAlias.EvalProgress.md | 25 ++ docs/docs/api/appkit/TypeAlias.Matcher.md | 17 + docs/docs/api/appkit/TypeAlias.Severity.md | 7 + docs/docs/api/appkit/index.md | 86 +++-- docs/docs/api/appkit/typedoc-sidebar.ts | 295 +++++++++--------- packages/appkit/package.json | 1 + packages/appkit/src/evals/index.ts | 9 +- packages/appkit/src/evals/judge.ts | 108 +++++++ packages/appkit/src/evals/mlflow-report.ts | 88 ++++-- packages/appkit/src/evals/mlflow-run.ts | 17 +- packages/appkit/src/evals/run-eval.ts | 40 +++ packages/appkit/src/evals/run-evals.ts | 10 + packages/appkit/src/evals/tests/judge.test.ts | 22 ++ .../src/evals/tests/mlflow-report.test.ts | 82 +++-- packages/appkit/src/evals/types.ts | 21 ++ packages/appkit/src/plugins/agents/agents.ts | 14 + packages/appkit/src/plugins/agents/mlflow.ts | 27 ++ .../shared/src/cli/commands/agent/eval.ts | 14 + pnpm-lock.yaml | 201 ++++++++++-- 55 files changed, 2109 insertions(+), 282 deletions(-) create mode 100644 apps/dev-playground/config/agents/query/evals/judge.eval.ts create mode 100644 docs/docs/api/appkit/Function.buildAssessment.md create mode 100644 docs/docs/api/appkit/Function.createHttpDriver.md create mode 100644 docs/docs/api/appkit/Function.defineEval.md create mode 100644 docs/docs/api/appkit/Function.defineEvalConfig.md create mode 100644 docs/docs/api/appkit/Function.discoverEvalFiles.md create mode 100644 docs/docs/api/appkit/Function.equals.md create mode 100644 docs/docs/api/appkit/Function.evalGlyph.md create mode 100644 docs/docs/api/appkit/Function.formatEvalDetail.md create mode 100644 docs/docs/api/appkit/Function.formatEvalHeadline.md create mode 100644 docs/docs/api/appkit/Function.formatEvalResults.md create mode 100644 docs/docs/api/appkit/Function.formatSummaryLine.md create mode 100644 docs/docs/api/appkit/Function.includes.md create mode 100644 docs/docs/api/appkit/Function.matches.md create mode 100644 docs/docs/api/appkit/Function.reportToMlflow.md create mode 100644 docs/docs/api/appkit/Function.runEval.md create mode 100644 docs/docs/api/appkit/Function.runEvalsInDir.md create mode 100644 docs/docs/api/appkit/Function.summarize.md create mode 100644 docs/docs/api/appkit/Interface.AssertionHandle.md create mode 100644 docs/docs/api/appkit/Interface.AssertionResult.md create mode 100644 docs/docs/api/appkit/Interface.Assessment.md create mode 100644 docs/docs/api/appkit/Interface.DiscoveredEval.md create mode 100644 docs/docs/api/appkit/Interface.DriveResult.md create mode 100644 docs/docs/api/appkit/Interface.EvalConfig.md create mode 100644 docs/docs/api/appkit/Interface.EvalDefinition.md create mode 100644 docs/docs/api/appkit/Interface.EvalDriver.md create mode 100644 docs/docs/api/appkit/Interface.EvalResult.md create mode 100644 docs/docs/api/appkit/Interface.EvalRunSummary.md create mode 100644 docs/docs/api/appkit/Interface.EvalSummary.md create mode 100644 docs/docs/api/appkit/Interface.HttpDriverOptions.md create mode 100644 docs/docs/api/appkit/Interface.MatchResult.md create mode 100644 docs/docs/api/appkit/Interface.MlflowReportOptions.md create mode 100644 docs/docs/api/appkit/Interface.ReportOutcome.md create mode 100644 docs/docs/api/appkit/Interface.RunEvalOptions.md create mode 100644 docs/docs/api/appkit/Interface.RunEvalsOptions.md create mode 100644 docs/docs/api/appkit/Interface.TestContext.md create mode 100644 docs/docs/api/appkit/TypeAlias.EvalProgress.md create mode 100644 docs/docs/api/appkit/TypeAlias.Matcher.md create mode 100644 docs/docs/api/appkit/TypeAlias.Severity.md create mode 100644 packages/appkit/src/evals/judge.ts create mode 100644 packages/appkit/src/evals/tests/judge.test.ts diff --git a/apps/dev-playground/config/agents/query/evals/judge.eval.ts b/apps/dev-playground/config/agents/query/evals/judge.eval.ts new file mode 100644 index 000000000..039045bdd --- /dev/null +++ b/apps/dev-playground/config/agents/query/evals/judge.eval.ts @@ -0,0 +1,25 @@ +import { defineEval } from "@databricks/appkit/beta"; + +/** + * LLM-as-judge eval: scores the helper agent's weather answer with a judge + * model (via autoevals → a Databricks serving endpoint). + * + * Requires a judge model: + * appkit agent eval query --root apps/dev-playground --url http://localhost:8001 \ + * --judge-model databricks-claude-sonnet-4-5 + * (or set APPKIT_JUDGE_MODEL). Without it, t.judge.* errors with a clear message. + */ +export default defineEval({ + description: "Helper weather answer is relevant (LLM judge)", + agent: "helper", + async test(t) { + await t.send("What's the weather in Brooklyn?"); + t.succeeded(); + // closedQA needs no ground truth — it judges the reply against a question. + ( + await t.judge.closedQA( + "Does the response describe weather conditions for Brooklyn?", + ) + ).atLeast(0.5); + }, +}); diff --git a/docs/docs/api/appkit/Function.buildAssessment.md b/docs/docs/api/appkit/Function.buildAssessment.md new file mode 100644 index 000000000..62c08b2bd --- /dev/null +++ b/docs/docs/api/appkit/Function.buildAssessment.md @@ -0,0 +1,18 @@ +# Function: buildAssessment() + +```ts +function buildAssessment(result: EvalResult): Assessment | undefined; +``` + +Build the single pass/fail Feedback assessment for an eval result. Returns +undefined when there's no trace to attach to or the eval was skipped. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +[`Assessment`](Interface.Assessment.md) \| `undefined` diff --git a/docs/docs/api/appkit/Function.createHttpDriver.md b/docs/docs/api/appkit/Function.createHttpDriver.md new file mode 100644 index 000000000..b373be10b --- /dev/null +++ b/docs/docs/api/appkit/Function.createHttpDriver.md @@ -0,0 +1,20 @@ +# Function: createHttpDriver() + +```ts +function createHttpDriver(options: HttpDriverOptions): EvalDriver; +``` + +Drives an agent by POSTing to a running app's chat endpoint and parsing the +SSE response. Keeps the thread id across `send`s so multi-turn evals share a +conversation. Agent/stream errors surface as `succeeded: false` rather than +throwing, so `t.succeeded()` can assert on them. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`HttpDriverOptions`](Interface.HttpDriverOptions.md) | + +## Returns + +[`EvalDriver`](Interface.EvalDriver.md) diff --git a/docs/docs/api/appkit/Function.defineEval.md b/docs/docs/api/appkit/Function.defineEval.md new file mode 100644 index 000000000..a47e86629 --- /dev/null +++ b/docs/docs/api/appkit/Function.defineEval.md @@ -0,0 +1,34 @@ +# Function: defineEval() + +```ts +function defineEval(def: EvalDefinition): EvalDefinition; +``` + +Define an agent eval. Default-export the result from a +`config/agents//evals/*.eval.ts` file. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `def` | [`EvalDefinition`](Interface.EvalDefinition.md) | + +## Returns + +[`EvalDefinition`](Interface.EvalDefinition.md) + +## Example + +```ts +import { defineEval, includes } from "@databricks/appkit/beta"; + +export default defineEval({ + description: "Weather agent basic coverage", + async test(t) { + await t.send("What's the weather in Brooklyn?"); + t.succeeded(); + t.calledTool("get_weather"); + t.check(t.reply, includes("Sunny")); + }, +}); +``` diff --git a/docs/docs/api/appkit/Function.defineEvalConfig.md b/docs/docs/api/appkit/Function.defineEvalConfig.md new file mode 100644 index 000000000..2710a598c --- /dev/null +++ b/docs/docs/api/appkit/Function.defineEvalConfig.md @@ -0,0 +1,17 @@ +# Function: defineEvalConfig() + +```ts +function defineEvalConfig(config: EvalConfig): EvalConfig; +``` + +Define per-directory eval config. Default-export from `evals.config.ts`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `config` | [`EvalConfig`](Interface.EvalConfig.md) | + +## Returns + +[`EvalConfig`](Interface.EvalConfig.md) diff --git a/docs/docs/api/appkit/Function.discoverEvalFiles.md b/docs/docs/api/appkit/Function.discoverEvalFiles.md new file mode 100644 index 000000000..746b45e5f --- /dev/null +++ b/docs/docs/api/appkit/Function.discoverEvalFiles.md @@ -0,0 +1,19 @@ +# Function: discoverEvalFiles() + +```ts +function discoverEvalFiles(rootDir: string): DiscoveredEval[]; +``` + +Discover evals under `/config/agents//evals/`. The agent id +is the directory name; the eval id is the file path relative to that evals +dir with `.eval.ts` stripped. Returns a stable, sorted list. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `rootDir` | `string` | + +## Returns + +[`DiscoveredEval`](Interface.DiscoveredEval.md)[] diff --git a/docs/docs/api/appkit/Function.equals.md b/docs/docs/api/appkit/Function.equals.md new file mode 100644 index 000000000..6d93b8377 --- /dev/null +++ b/docs/docs/api/appkit/Function.equals.md @@ -0,0 +1,17 @@ +# Function: equals() + +```ts +function equals(expected: string): Matcher; +``` + +Passes when the value equals `expected` exactly. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `expected` | `string` | + +## Returns + +[`Matcher`](TypeAlias.Matcher.md) diff --git a/docs/docs/api/appkit/Function.evalGlyph.md b/docs/docs/api/appkit/Function.evalGlyph.md new file mode 100644 index 000000000..edef1483d --- /dev/null +++ b/docs/docs/api/appkit/Function.evalGlyph.md @@ -0,0 +1,17 @@ +# Function: evalGlyph() + +```ts +function evalGlyph(result: EvalResult): string; +``` + +Status glyph for a single eval result. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.formatEvalDetail.md b/docs/docs/api/appkit/Function.formatEvalDetail.md new file mode 100644 index 000000000..f07f83a1f --- /dev/null +++ b/docs/docs/api/appkit/Function.formatEvalDetail.md @@ -0,0 +1,17 @@ +# Function: formatEvalDetail() + +```ts +function formatEvalDetail(result: EvalResult): string[]; +``` + +Indented detail lines for a failing eval (error + failing assertions). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +`string`[] diff --git a/docs/docs/api/appkit/Function.formatEvalHeadline.md b/docs/docs/api/appkit/Function.formatEvalHeadline.md new file mode 100644 index 000000000..f6d22ee8d --- /dev/null +++ b/docs/docs/api/appkit/Function.formatEvalHeadline.md @@ -0,0 +1,17 @@ +# Function: formatEvalHeadline() + +```ts +function formatEvalHeadline(result: EvalResult): string; +``` + +The one-line header for a single eval result (no failure detail). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.formatEvalResults.md b/docs/docs/api/appkit/Function.formatEvalResults.md new file mode 100644 index 000000000..2607a52b7 --- /dev/null +++ b/docs/docs/api/appkit/Function.formatEvalResults.md @@ -0,0 +1,17 @@ +# Function: formatEvalResults() + +```ts +function formatEvalResults(results: EvalResult[]): string; +``` + +Render all results as a human-readable console report (non-streaming). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.formatSummaryLine.md b/docs/docs/api/appkit/Function.formatSummaryLine.md new file mode 100644 index 000000000..355f92958 --- /dev/null +++ b/docs/docs/api/appkit/Function.formatSummaryLine.md @@ -0,0 +1,17 @@ +# Function: formatSummaryLine() + +```ts +function formatSummaryLine(results: EvalResult[]): string; +``` + +The final PASS/FAIL summary line. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.includes.md b/docs/docs/api/appkit/Function.includes.md new file mode 100644 index 000000000..3ff1ab7d5 --- /dev/null +++ b/docs/docs/api/appkit/Function.includes.md @@ -0,0 +1,17 @@ +# Function: includes() + +```ts +function includes(substring: string): Matcher; +``` + +Passes when the value contains `substring`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `substring` | `string` | + +## Returns + +[`Matcher`](TypeAlias.Matcher.md) diff --git a/docs/docs/api/appkit/Function.matches.md b/docs/docs/api/appkit/Function.matches.md new file mode 100644 index 000000000..6839fa5ce --- /dev/null +++ b/docs/docs/api/appkit/Function.matches.md @@ -0,0 +1,17 @@ +# Function: matches() + +```ts +function matches(pattern: RegExp): Matcher; +``` + +Passes when the value matches `pattern`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `pattern` | `RegExp` | + +## Returns + +[`Matcher`](TypeAlias.Matcher.md) diff --git a/docs/docs/api/appkit/Function.reportToMlflow.md b/docs/docs/api/appkit/Function.reportToMlflow.md new file mode 100644 index 000000000..5d3c3b7b9 --- /dev/null +++ b/docs/docs/api/appkit/Function.reportToMlflow.md @@ -0,0 +1,19 @@ +# Function: reportToMlflow() + +```ts +function reportToMlflow(results: EvalResult[], options: MlflowReportOptions): Promise; +``` + +Write one pass/fail assessment per eval result to the Databricks MLflow REST +API. Never throws — failures are collected so the run still reports. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | +| `options` | [`MlflowReportOptions`](Interface.MlflowReportOptions.md) | + +## Returns + +`Promise`\<[`ReportOutcome`](Interface.ReportOutcome.md)\> diff --git a/docs/docs/api/appkit/Function.runEval.md b/docs/docs/api/appkit/Function.runEval.md new file mode 100644 index 000000000..9f0fd1a05 --- /dev/null +++ b/docs/docs/api/appkit/Function.runEval.md @@ -0,0 +1,20 @@ +# Function: runEval() + +```ts +function runEval(def: EvalDefinition, options: RunEvalOptions): Promise; +``` + +Run a single eval against a driver. Never throws for assertion or agent +failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed +eval definition surfaces as `result.error`. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `def` | [`EvalDefinition`](Interface.EvalDefinition.md) | +| `options` | [`RunEvalOptions`](Interface.RunEvalOptions.md) | + +## Returns + +`Promise`\<[`EvalResult`](Interface.EvalResult.md)\> diff --git a/docs/docs/api/appkit/Function.runEvalsInDir.md b/docs/docs/api/appkit/Function.runEvalsInDir.md new file mode 100644 index 000000000..903559b1b --- /dev/null +++ b/docs/docs/api/appkit/Function.runEvalsInDir.md @@ -0,0 +1,19 @@ +# Function: runEvalsInDir() + +```ts +function runEvalsInDir(options: RunEvalsOptions): Promise; +``` + +Discover, load, and run every eval under each agent's `evals/` dir, driving +the agents on a running app. Never throws for an individual eval — load/run +failures become non-passing [EvalResult](Interface.EvalResult.md)s. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`RunEvalsOptions`](Interface.RunEvalsOptions.md) | + +## Returns + +`Promise`\<[`EvalRunSummary`](Interface.EvalRunSummary.md)\> diff --git a/docs/docs/api/appkit/Function.summarize.md b/docs/docs/api/appkit/Function.summarize.md new file mode 100644 index 000000000..959817e62 --- /dev/null +++ b/docs/docs/api/appkit/Function.summarize.md @@ -0,0 +1,15 @@ +# Function: summarize() + +```ts +function summarize(results: EvalResult[]): EvalSummary; +``` + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `results` | [`EvalResult`](Interface.EvalResult.md)[] | + +## Returns + +[`EvalSummary`](Interface.EvalSummary.md) diff --git a/docs/docs/api/appkit/Interface.AssertionHandle.md b/docs/docs/api/appkit/Interface.AssertionHandle.md new file mode 100644 index 000000000..02e3c746b --- /dev/null +++ b/docs/docs/api/appkit/Interface.AssertionHandle.md @@ -0,0 +1,53 @@ +# Interface: AssertionHandle + +Chainable handle returned by every assertion to control its severity. +Mirrors eve: assertions are gates by default; `.soft()` demotes to a tracked +metric; `.atLeast(n)` is a soft, score-thresholded assertion. + +## Methods + +### atLeast() + +```ts +atLeast(threshold: number): AssertionHandle; +``` + +Soft assertion that passes only when the score is at least `threshold`. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `threshold` | `number` | + +#### Returns + +`AssertionHandle` + +*** + +### gate() + +```ts +gate(): AssertionHandle; +``` + +Promote to a hard gate — failure fails the eval (non-zero exit). + +#### Returns + +`AssertionHandle` + +*** + +### soft() + +```ts +soft(): AssertionHandle; +``` + +Demote to a tracked metric — doesn't fail unless running with `strict`. + +#### Returns + +`AssertionHandle` diff --git a/docs/docs/api/appkit/Interface.AssertionResult.md b/docs/docs/api/appkit/Interface.AssertionResult.md new file mode 100644 index 000000000..df3e986bf --- /dev/null +++ b/docs/docs/api/appkit/Interface.AssertionResult.md @@ -0,0 +1,43 @@ +# Interface: AssertionResult + +A single recorded assertion outcome. + +## Properties + +### detail? + +```ts +optional detail: string; +``` + +*** + +### label + +```ts +label: string; +``` + +*** + +### pass + +```ts +pass: boolean; +``` + +*** + +### score? + +```ts +optional score: number; +``` + +*** + +### severity + +```ts +severity: Severity; +``` diff --git a/docs/docs/api/appkit/Interface.Assessment.md b/docs/docs/api/appkit/Interface.Assessment.md new file mode 100644 index 000000000..d3537cf00 --- /dev/null +++ b/docs/docs/api/appkit/Interface.Assessment.md @@ -0,0 +1,74 @@ +# Interface: Assessment + +A Feedback assessment in the MLflow REST proto-JSON shape. + +## Properties + +### assessment\_name + +```ts +assessment_name: string; +``` + +*** + +### feedback + +```ts +feedback: { + value: unknown; +}; +``` + +#### value + +```ts +value: unknown; +``` + +*** + +### metadata? + +```ts +optional metadata: Record; +``` + +*** + +### rationale? + +```ts +optional rationale: string; +``` + +*** + +### source + +```ts +source: { + source_id: string; + source_type: "CODE" | "HUMAN" | "LLM_JUDGE"; +}; +``` + +#### source\_id + +```ts +source_id: string; +``` + +#### source\_type + +```ts +source_type: "CODE" | "HUMAN" | "LLM_JUDGE"; +``` + +*** + +### trace\_id + +```ts +trace_id: string; +``` diff --git a/docs/docs/api/appkit/Interface.DiscoveredEval.md b/docs/docs/api/appkit/Interface.DiscoveredEval.md new file mode 100644 index 000000000..376aa4c4a --- /dev/null +++ b/docs/docs/api/appkit/Interface.DiscoveredEval.md @@ -0,0 +1,33 @@ +# Interface: DiscoveredEval + +An eval file found under `config/agents//evals/`. + +## Properties + +### agent + +```ts +agent: string; +``` + +The agent id (the `config/agents/` directory name). + +*** + +### file + +```ts +file: string; +``` + +Absolute path to the `*.eval.ts` file. + +*** + +### id + +```ts +id: string; +``` + +Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). diff --git a/docs/docs/api/appkit/Interface.DriveResult.md b/docs/docs/api/appkit/Interface.DriveResult.md new file mode 100644 index 000000000..15625c1ac --- /dev/null +++ b/docs/docs/api/appkit/Interface.DriveResult.md @@ -0,0 +1,53 @@ +# Interface: DriveResult + +What a driver returns for a single `t.send`. + +## Properties + +### reply + +```ts +reply: string; +``` + +The final assistant message text. + +*** + +### sessionId? + +```ts +optional sessionId: string; +``` + +Thread/session id, when the driver exposes one. + +*** + +### succeeded + +```ts +succeeded: boolean; +``` + +Whether the turn completed without an agent/stream error. + +*** + +### toolCalls + +```ts +toolCalls: string[]; +``` + +Names of tools the agent called during the turn. + +*** + +### traceId? + +```ts +optional traceId: string; +``` + +MLflow trace id for the turn, when tracing is enabled on the app. diff --git a/docs/docs/api/appkit/Interface.EvalConfig.md b/docs/docs/api/appkit/Interface.EvalConfig.md new file mode 100644 index 000000000..eb8227fcd --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalConfig.md @@ -0,0 +1,41 @@ +# Interface: EvalConfig + +Per-directory config from `evals.config.ts`. + +## Properties + +### judge? + +```ts +optional judge: { + model?: string; +}; +``` + +LLM judge config. Defaults to the agent's own serving endpoint. + +#### model? + +```ts +optional model: string; +``` + +*** + +### maxConcurrency? + +```ts +optional maxConcurrency: number; +``` + +Max evals to run concurrently. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Default per-eval timeout. diff --git a/docs/docs/api/appkit/Interface.EvalDefinition.md b/docs/docs/api/appkit/Interface.EvalDefinition.md new file mode 100644 index 000000000..e579ce345 --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalDefinition.md @@ -0,0 +1,63 @@ +# Interface: EvalDefinition + +A single eval, default-exported from a `*.eval.ts` file. + +## Properties + +### agent? + +```ts +optional agent: string; +``` + +Target agent id. Defaults to the eval's parent `config/agents/` dir. + +*** + +### description? + +```ts +optional description: string; +``` + +Short human description, shown in reports. + +*** + +### tags? + +```ts +optional tags: string[]; +``` + +Free-form tags for filtering. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Per-eval timeout. + +## Methods + +### test() + +```ts +test(t: TestContext): void | Promise; +``` + +The eval body: drive the agent and assert on its behavior. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `t` | [`TestContext`](Interface.TestContext.md) | + +#### Returns + +`void` \| `Promise`\<`void`\> diff --git a/docs/docs/api/appkit/Interface.EvalDriver.md b/docs/docs/api/appkit/Interface.EvalDriver.md new file mode 100644 index 000000000..69d81a05a --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalDriver.md @@ -0,0 +1,22 @@ +# Interface: EvalDriver + +Abstraction over how the agent is driven. The HTTP driver posts to a running +app's agents endpoint; future drivers (in-process) implement the same shape. + +## Methods + +### send() + +```ts +send(message: string): Promise; +``` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `message` | `string` | + +#### Returns + +`Promise`\<[`DriveResult`](Interface.DriveResult.md)\> diff --git a/docs/docs/api/appkit/Interface.EvalResult.md b/docs/docs/api/appkit/Interface.EvalResult.md new file mode 100644 index 000000000..94a6c9aff --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalResult.md @@ -0,0 +1,75 @@ +# Interface: EvalResult + +The outcome of running one eval. + +## Properties + +### assertions + +```ts +assertions: AssertionResult[]; +``` + +*** + +### description? + +```ts +optional description: string; +``` + +*** + +### error? + +```ts +optional error: string; +``` + +Set when the eval threw before completing. + +*** + +### id + +```ts +id: string; +``` + +*** + +### passed + +```ts +passed: boolean; +``` + +True when all gates passed (and, under strict, all soft assertions too). + +*** + +### skipped? + +```ts +optional skipped: { + reason?: string; +}; +``` + +Set when the eval called `t.skip`. + +#### reason? + +```ts +optional reason: string; +``` + +*** + +### traceId? + +```ts +optional traceId: string; +``` + +MLflow trace id of the eval's last turn, for attaching assessments. diff --git a/docs/docs/api/appkit/Interface.EvalRunSummary.md b/docs/docs/api/appkit/Interface.EvalRunSummary.md new file mode 100644 index 000000000..ec9176c00 --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalRunSummary.md @@ -0,0 +1,41 @@ +# Interface: EvalRunSummary + +## Properties + +### mlflow? + +```ts +optional mlflow: { + finish: FinishOutcome; + report: ReportOutcome; + runId: string; +}; +``` + +Present when an MLflow evaluation run was created. + +#### finish + +```ts +finish: FinishOutcome; +``` + +#### report + +```ts +report: ReportOutcome; +``` + +#### runId + +```ts +runId: string; +``` + +*** + +### results + +```ts +results: EvalResult[]; +``` diff --git a/docs/docs/api/appkit/Interface.EvalSummary.md b/docs/docs/api/appkit/Interface.EvalSummary.md new file mode 100644 index 000000000..3c8fc1e6e --- /dev/null +++ b/docs/docs/api/appkit/Interface.EvalSummary.md @@ -0,0 +1,43 @@ +# Interface: EvalSummary + +## Properties + +### allPassed + +```ts +allPassed: boolean; +``` + +True when no eval failed (skips don't count as failures). + +*** + +### failed + +```ts +failed: number; +``` + +*** + +### passed + +```ts +passed: number; +``` + +*** + +### skipped + +```ts +skipped: number; +``` + +*** + +### total + +```ts +total: number; +``` diff --git a/docs/docs/api/appkit/Interface.HttpDriverOptions.md b/docs/docs/api/appkit/Interface.HttpDriverOptions.md new file mode 100644 index 000000000..8bb1f25e0 --- /dev/null +++ b/docs/docs/api/appkit/Interface.HttpDriverOptions.md @@ -0,0 +1,51 @@ +# Interface: HttpDriverOptions + +## Properties + +### agent? + +```ts +optional agent: string; +``` + +Agent alias to target. Omit to use the app's default agent. + +*** + +### baseUrl + +```ts +baseUrl: string; +``` + +Base URL of the running app, e.g. `http://localhost:3000`. + +*** + +### headers? + +```ts +optional headers: Record; +``` + +Extra request headers (e.g. auth for a deployed app). + +*** + +### mlflowRunId? + +```ts +optional mlflowRunId: string; +``` + +MLflow run id to link each turn's trace to (for evaluation runs). + +*** + +### path? + +```ts +optional path: string; +``` + +Chat endpoint path. Defaults to `/api/agents/chat`. diff --git a/docs/docs/api/appkit/Interface.MatchResult.md b/docs/docs/api/appkit/Interface.MatchResult.md new file mode 100644 index 000000000..a8744ef07 --- /dev/null +++ b/docs/docs/api/appkit/Interface.MatchResult.md @@ -0,0 +1,31 @@ +# Interface: MatchResult + +Result of a deterministic matcher run against a value. + +## Properties + +### detail? + +```ts +optional detail: string; +``` + +Human-readable explanation, shown on failure. + +*** + +### pass + +```ts +pass: boolean; +``` + +*** + +### score? + +```ts +optional score: number; +``` + +Optional 0..1 score for scored matchers (similarity, judges). diff --git a/docs/docs/api/appkit/Interface.MlflowReportOptions.md b/docs/docs/api/appkit/Interface.MlflowReportOptions.md new file mode 100644 index 000000000..22733c8be --- /dev/null +++ b/docs/docs/api/appkit/Interface.MlflowReportOptions.md @@ -0,0 +1,21 @@ +# Interface: MlflowReportOptions + +## Properties + +### host + +```ts +host: string; +``` + +Databricks workspace host (scheme optional — normalized). + +*** + +### token + +```ts +token: string; +``` + +Bearer token for the MLflow REST API. diff --git a/docs/docs/api/appkit/Interface.ReportOutcome.md b/docs/docs/api/appkit/Interface.ReportOutcome.md new file mode 100644 index 000000000..f42136a9b --- /dev/null +++ b/docs/docs/api/appkit/Interface.ReportOutcome.md @@ -0,0 +1,47 @@ +# Interface: ReportOutcome + +## Properties + +### failures + +```ts +failures: { + error?: string; + status?: number; + traceId: string; +}[]; +``` + +#### error? + +```ts +optional error: string; +``` + +#### status? + +```ts +optional status: number; +``` + +#### traceId + +```ts +traceId: string; +``` + +*** + +### skipped + +```ts +skipped: number; +``` + +*** + +### written + +```ts +written: number; +``` diff --git a/docs/docs/api/appkit/Interface.RunEvalOptions.md b/docs/docs/api/appkit/Interface.RunEvalOptions.md new file mode 100644 index 000000000..16a6b8190 --- /dev/null +++ b/docs/docs/api/appkit/Interface.RunEvalOptions.md @@ -0,0 +1,31 @@ +# Interface: RunEvalOptions + +## Properties + +### driver + +```ts +driver: EvalDriver; +``` + +Drives the agent and returns reply/tool-calls/success per `send`. + +*** + +### id + +```ts +id: string; +``` + +Stable id for the eval (e.g. its file path relative to the evals dir). + +*** + +### strict? + +```ts +optional strict: boolean; +``` + +When true, soft assertion failures also fail the eval. diff --git a/docs/docs/api/appkit/Interface.RunEvalsOptions.md b/docs/docs/api/appkit/Interface.RunEvalsOptions.md new file mode 100644 index 000000000..5fe5bbd9c --- /dev/null +++ b/docs/docs/api/appkit/Interface.RunEvalsOptions.md @@ -0,0 +1,115 @@ +# Interface: RunEvalsOptions + +## Properties + +### baseUrl + +```ts +baseUrl: string; +``` + +Base URL of the running app to drive, e.g. `http://localhost:3000`. + +*** + +### filter? + +```ts +optional filter: string; +``` + +Substring filter on `/` (or an exact agent id). + +*** + +### headers? + +```ts +optional headers: Record; +``` + +Extra request headers for the driver (e.g. auth for a deployed app). + +*** + +### mlflow? + +```ts +optional mlflow: { + experimentId: string; + host: string; + token: string; +}; +``` + +When set, create a native MLflow "Evaluation run": each eval's trace is +linked to the run, pass/fail is written as feedback, and aggregate metrics +are logged. Requires Databricks creds + the target experiment. + +#### experimentId + +```ts +experimentId: string; +``` + +#### host + +```ts +host: string; +``` + +#### token + +```ts +token: string; +``` + +*** + +### now? + +```ts +optional now: number; +``` + +Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. + +*** + +### onEvent()? + +```ts +optional onEvent: (event: EvalProgress) => void; +``` + +Progress callback, invoked as evals are discovered, started, and finished. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `event` | [`EvalProgress`](TypeAlias.EvalProgress.md) | + +#### Returns + +`void` + +*** + +### rootDir? + +```ts +optional rootDir: string; +``` + +Project root containing `config/agents/`. Defaults to `process.cwd()`. + +*** + +### strict? + +```ts +optional strict: boolean; +``` + +Soft assertion failures also fail the eval. diff --git a/docs/docs/api/appkit/Interface.TestContext.md b/docs/docs/api/appkit/Interface.TestContext.md new file mode 100644 index 000000000..5f249952b --- /dev/null +++ b/docs/docs/api/appkit/Interface.TestContext.md @@ -0,0 +1,128 @@ +# Interface: TestContext + +The `t` context passed to an eval's `test` function. + +## Properties + +### reply + +```ts +readonly reply: string; +``` + +The last assistant reply. + +*** + +### sessionId + +```ts +readonly sessionId: string | undefined; +``` + +The current session/thread id, if any. + +*** + +### toolCalls + +```ts +readonly toolCalls: string[]; +``` + +Tools called during the last turn. + +## Methods + +### calledTool() + +```ts +calledTool(name: string): AssertionHandle; +``` + +Assert a tool was called during the run (gate by default). + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `name` | `string` | + +#### Returns + +[`AssertionHandle`](Interface.AssertionHandle.md) + +*** + +### check() + +```ts +check(value: string, matcher: Matcher): AssertionHandle; +``` + +Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `value` | `string` | +| `matcher` | [`Matcher`](TypeAlias.Matcher.md) | + +#### Returns + +[`AssertionHandle`](Interface.AssertionHandle.md) + +*** + +### send() + +```ts +send(message: string): Promise; +``` + +Send a user message to the agent and capture its response. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `message` | `string` | + +#### Returns + +`Promise`\<`void`\> + +*** + +### skip() + +```ts +skip(reason?: string): never; +``` + +Skip this eval with an optional reason. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `reason?` | `string` | + +#### Returns + +`never` + +*** + +### succeeded() + +```ts +succeeded(): AssertionHandle; +``` + +Assert the last turn completed successfully (gate by default). + +#### Returns + +[`AssertionHandle`](Interface.AssertionHandle.md) diff --git a/docs/docs/api/appkit/TypeAlias.EvalProgress.md b/docs/docs/api/appkit/TypeAlias.EvalProgress.md new file mode 100644 index 000000000..d1814ba6a --- /dev/null +++ b/docs/docs/api/appkit/TypeAlias.EvalProgress.md @@ -0,0 +1,25 @@ +# Type Alias: EvalProgress + +```ts +type EvalProgress = + | { + total: number; + type: "discovered"; +} + | { + runId: string; + type: "run-created"; +} + | { + id: string; + index: number; + total: number; + type: "start"; +} + | { + index: number; + result: EvalResult; + total: number; + type: "result"; +}; +``` diff --git a/docs/docs/api/appkit/TypeAlias.Matcher.md b/docs/docs/api/appkit/TypeAlias.Matcher.md new file mode 100644 index 000000000..f22e48787 --- /dev/null +++ b/docs/docs/api/appkit/TypeAlias.Matcher.md @@ -0,0 +1,17 @@ +# Type Alias: Matcher() + +```ts +type Matcher = (value: string) => MatchResult; +``` + +A deterministic matcher: inspects a string value and returns a result. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `value` | `string` | + +## Returns + +[`MatchResult`](Interface.MatchResult.md) diff --git a/docs/docs/api/appkit/TypeAlias.Severity.md b/docs/docs/api/appkit/TypeAlias.Severity.md new file mode 100644 index 000000000..575bad861 --- /dev/null +++ b/docs/docs/api/appkit/TypeAlias.Severity.md @@ -0,0 +1,7 @@ +# Type Alias: Severity + +```ts +type Severity = "gate" | "soft"; +``` + +Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index 884db87f6..a21025585 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -26,7 +26,6 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [PolicyDeniedError](Class.PolicyDeniedError.md) | Thrown when a policy denies an action. | | [ResourceRegistry](Class.ResourceRegistry.md) | Central registry for tracking plugin resource requirements. Deduplication uses type + resourceKey (machine-stable); alias is for display only. | | [ServerError](Class.ServerError.md) | Error thrown when server lifecycle operations fail. Use for server start/stop issues, configuration conflicts, etc. | -| [SupervisorApiAdapter](Class.SupervisorApiAdapter.md) | Adapter that calls the Databricks AI Gateway Responses API (`/ai-gateway/mlflow/v1/responses`). | | [TunnelError](Class.TunnelError.md) | Error thrown when remote tunnel operations fail. Use for tunnel connection issues, message parsing failures, etc. | | [ValidationError](Class.ValidationError.md) | Error thrown when input validation fails. Use for invalid parameters, missing required fields, or type mismatches. | @@ -40,21 +39,28 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [AgentRunContext](Interface.AgentRunContext.md) | - | | [AgentsPluginConfig](Interface.AgentsPluginConfig.md) | Base configuration interface for AppKit plugins | | [AgentToolDefinition](Interface.AgentToolDefinition.md) | - | +| [AssertionHandle](Interface.AssertionHandle.md) | Chainable handle returned by every assertion to control its severity. Mirrors eve: assertions are gates by default; `.soft()` demotes to a tracked metric; `.atLeast(n)` is a soft, score-thresholded assertion. | +| [AssertionResult](Interface.AssertionResult.md) | A single recorded assertion outcome. | +| [Assessment](Interface.Assessment.md) | A Feedback assessment in the MLflow REST proto-JSON shape. | | [AutoInheritToolsConfig](Interface.AutoInheritToolsConfig.md) | Auto-inherit configuration. When enabled for a given agent origin, agents with no explicit `tools:` declaration receive every registered ToolProvider plugin tool whose author marked `autoInheritable: true`. Tools without that flag — destructive, state-mutating, or privilege-sensitive — never spread automatically and must be wired via `tools:` (object or function form in code, `plugin:NAME` entries in markdown frontmatter). | | [BasePluginConfig](Interface.BasePluginConfig.md) | Base configuration interface for AppKit plugins | | [CacheConfig](Interface.CacheConfig.md) | Configuration for the CacheInterceptor. Controls TTL, size limits, storage backend, and probabilistic cleanup. | | [DatabaseCredential](Interface.DatabaseCredential.md) | Database credentials with OAuth token for Postgres connection | -| [DatabaseRegistry](Interface.DatabaseRegistry.md) | CANONICAL augmentation target. Empty by default; the generated `database.d.ts` augments it via `declare module "@databricks/appkit" { interface DatabaseRegistry { ... } }`. | +| [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `config/agents//evals/`. | +| [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. | | [EndpointConfig](Interface.EndpointConfig.md) | - | +| [EvalConfig](Interface.EvalConfig.md) | Per-directory config from `evals.config.ts`. | +| [EvalDefinition](Interface.EvalDefinition.md) | A single eval, default-exported from a `*.eval.ts` file. | +| [EvalDriver](Interface.EvalDriver.md) | Abstraction over how the agent is driven. The HTTP driver posts to a running app's agents endpoint; future drivers (in-process) implement the same shape. | +| [EvalResult](Interface.EvalResult.md) | The outcome of running one eval. | +| [EvalRunSummary](Interface.EvalRunSummary.md) | - | +| [EvalSummary](Interface.EvalSummary.md) | - | | [FilePolicyUser](Interface.FilePolicyUser.md) | Minimal user identity passed to the policy function. | | [FileResource](Interface.FileResource.md) | Describes the file or directory being acted upon. | | [FunctionTool](Interface.FunctionTool.md) | - | | [GenerateDatabaseCredentialRequest](Interface.GenerateDatabaseCredentialRequest.md) | Request parameters for generating database OAuth credentials | -| [GenerationParams](Interface.GenerationParams.md) | Optional generation parameters forwarded to the OpenAI-compatible serving request body. Names match the serving API wire keys. Only keys that are set are sent — undefined values are omitted so the endpoint applies its own defaults. Ranges are not validated here; the serving endpoint validates. | -| [HostedSupervisorTool](Interface.HostedSupervisorTool.md) | Tagged record returned by every [supervisorTools](Variable.supervisorTools.md) factory. The `__kind` discriminator lets the agents plugin (and standalone `runAgent`) classify these tools without a structural match against the wire format — keeps the SA wire shape free to evolve and avoids namespace collisions with MCP hosted tools (which use `type: "genie-space"` hyphenated, vs SA's `type: "genie_space"` underscored). | -| [IAiSearchConfig](Interface.IAiSearchConfig.md) | Base configuration interface for AppKit plugins | +| [HttpDriverOptions](Interface.HttpDriverOptions.md) | - | | [IJobsConfig](Interface.IJobsConfig.md) | Configuration for the Jobs plugin. | -| [IndexConfig](Interface.IndexConfig.md) | - | | [ITelemetry](Interface.ITelemetry.md) | Plugin-facing interface for OpenTelemetry instrumentation. Provides a thin abstraction over OpenTelemetry APIs for plugins. | | [JobAPI](Interface.JobAPI.md) | User-facing API for a single configured job. | | [JobConfig](Interface.JobConfig.md) | Per-job configuration options. | @@ -62,29 +68,28 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [LakebasePool](Interface.LakebasePool.md) | Subset of `pg.Pool` exposed by the Lakebase plugin. | | [LakebasePoolConfig](Interface.LakebasePoolConfig.md) | Configuration for creating a Lakebase connection pool | | [LakebasePoolManager](Interface.LakebasePoolManager.md) | Manages multiple Lakebase connection pools keyed by an identifier (e.g. userId). | +| [MatchResult](Interface.MatchResult.md) | Result of a deterministic matcher run against a value. | | [McpConnectAllResult](Interface.McpConnectAllResult.md) | Per-endpoint outcome of [AppKitMcpClient.connectAll](Class.AppKitMcpClient.md#connectall). Callers (the agents plugin in particular) use the split to warn at startup when some MCP servers are unreachable without aborting boot for the rest. | | [Message](Interface.Message.md) | - | +| [MlflowReportOptions](Interface.MlflowReportOptions.md) | - | | [PluginManifest](Interface.PluginManifest.md) | Plugin manifest that declares metadata and resource requirements. Attached to plugin classes as a static property. Extends the shared PluginManifest with strict resource types. | | [PluginToolkitProvider](Interface.PluginToolkitProvider.md) | Minimum shape every entry in the [Plugins](TypeAlias.Plugins.md) map must expose. Core plugins (analytics, files, genie, lakebase) implement this directly via their `.toolkit()` method. The agents plugin and standalone `runAgent` synthesize this shape for any registered plugin that doesn't implement `.toolkit()` directly (falling back to `getAgentTools()` walking). | | [PromptContext](Interface.PromptContext.md) | Context passed to `baseSystemPrompt` callbacks. | | [RegisteredAgent](Interface.RegisteredAgent.md) | - | +| [ReportOutcome](Interface.ReportOutcome.md) | - | | [RequestedClaims](Interface.RequestedClaims.md) | Optional claims for fine-grained Unity Catalog table permissions When specified, the returned token will be scoped to only the requested tables | | [RequestedResource](Interface.RequestedResource.md) | Resource to request permissions for in Unity Catalog | -| [RerankerConfig](Interface.RerankerConfig.md) | - | | [ResourceEntry](Interface.ResourceEntry.md) | Internal representation of a resource in the registry. Extends ResourceRequirement with resolution state and plugin ownership. | | [ResourceRequirement](Interface.ResourceRequirement.md) | Declares a resource requirement for a plugin. Can be defined statically in a manifest or dynamically via getResourceRequirements(). | | [RunAgentInput](Interface.RunAgentInput.md) | - | | [RunAgentResult](Interface.RunAgentResult.md) | - | -| [Schema](Interface.Schema.md) | One finalized schema. `TTableName` keeps the declared names in the type, so configuration that addresses a table by name is checked against the schema it was written for. Code that accepts any schema uses the default. | -| [SearchRequest](Interface.SearchRequest.md) | - | -| [SearchResponse](Interface.SearchResponse.md) | - | -| [SearchResult](Interface.SearchResult.md) | - | +| [RunEvalOptions](Interface.RunEvalOptions.md) | - | +| [RunEvalsOptions](Interface.RunEvalsOptions.md) | - | | [ServingEndpointEntry](Interface.ServingEndpointEntry.md) | Shape of a single registry entry. | | [ServingEndpointRegistry](Interface.ServingEndpointRegistry.md) | Registry interface for serving endpoint type generation. Empty by default — augmented by the Vite type generator's `.d.ts` output via module augmentation. When populated, provides autocomplete for alias names and typed request/response/chunk per endpoint. | | [StreamExecutionSettings](Interface.StreamExecutionSettings.md) | Execution settings for streaming endpoints. Extends PluginExecutionSettings with SSE stream configuration. | -| [SupervisorApiAdapterOptions](Interface.SupervisorApiAdapterOptions.md) | - | -| [SupervisorExtension](Interface.SupervisorExtension.md) | Shape of the value at `AgentInput.extensions[SUPERVISOR_EXTENSION_KEY]`. The agents plugin / `runAgent` build this from the tool index; advanced callers invoking `adapter.run(...)` directly populate it themselves. | | [TelemetryConfig](Interface.TelemetryConfig.md) | OpenTelemetry configuration for AppKit applications | +| [TestContext](Interface.TestContext.md) | The `t` context passed to an eval's `test` function. | | [Thread](Interface.Thread.md) | - | | [ThreadStore](Interface.ThreadStore.md) | - | | [ToolAnnotations](Interface.ToolAnnotations.md) | - | @@ -94,36 +99,33 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [ToolkitOptions](Interface.ToolkitOptions.md) | - | | [ToolProvider](Interface.ToolProvider.md) | - | | [ValidationResult](Interface.ValidationResult.md) | Result of validating all registered resources against the environment. | -| [WorkspaceClient](Interface.WorkspaceClient.md) | AppKit's workspace client facade. Mirrors the multi-client shape of the modular Databricks SDK: each service is its own accessor, so services can be migrated one at a time behind this stable interface. | -| [WorkspaceClientLike](Interface.WorkspaceClientLike.md) | Structural shape of a Databricks SDK client used by [fromSupervisorApi](Function.fromSupervisorApi.md). Only what we need: `apiClient.request` for streaming and `config.ensureResolved` to materialise the host/credentials. | -| [WorkspaceClientOptions](Interface.WorkspaceClientOptions.md) | Options used to construct the wrapper. Mirrors the subset of the old SDK's `Config` + `ClientOptions` that AppKit relies on today; we deliberately do NOT re-expose every old-SDK config knob. | ## Type Aliases | Type Alias | Description | | ------ | ------ | | [AgentEvent](TypeAlias.AgentEvent.md) | - | -| [AgentTool](TypeAlias.AgentTool.md) | Any tool an agent can invoke: inline function tools (`tool()`), hosted MCP tools (`mcpServer()` / raw hosted), toolkit references from plugins (`analytics().toolkit()`), or adapter-hosted Supervisor-API tools (`supervisorTools.*`). | +| [AgentTool](TypeAlias.AgentTool.md) | Any tool an agent can invoke: inline function tools (`tool()`), hosted MCP tools (`mcpServer()` / raw hosted), or toolkit references from plugins (`analytics().toolkit()`). | | [AgentTools](TypeAlias.AgentTools.md) | Per-agent tool record. String keys map to inline tools, toolkit entries, hosted tools, etc. | | [AgentToolsFn](TypeAlias.AgentToolsFn.md) | Function form of `AgentDefinition.tools`. Receives the typed [Plugins](TypeAlias.Plugins.md) map and returns a tool record. Invoked exactly once at setup (or once per `runAgent` call in standalone mode); the result is cached as the agent's resolved tool record. | | [BaseSystemPromptOption](TypeAlias.BaseSystemPromptOption.md) | - | | [ConfigSchema](TypeAlias.ConfigSchema.md) | Configuration schema definition for plugin config. Re-exported from the standard JSON Schema Draft 7 types. | -| [DatabaseExports](TypeAlias.DatabaseExports.md) | Typed database API published by the plugin. | +| [EvalProgress](TypeAlias.EvalProgress.md) | - | | [ExecutionResult](TypeAlias.ExecutionResult.md) | Discriminated union for plugin execution results. | | [FileAction](TypeAlias.FileAction.md) | Every action the files plugin can perform. | | [FilePolicy](TypeAlias.FilePolicy.md) | A policy function that decides whether `user` may perform `action` on `resource`. Return `true` to allow, `false` to deny. | | [HostedTool](TypeAlias.HostedTool.md) | - | | [IAppRouter](TypeAlias.IAppRouter.md) | Express router type for plugin route registration | -| [IDatabaseConfig](TypeAlias.IDatabaseConfig.md) | Configuration for one schema-bound DatabasePlugin instance. | +| [JobHandle](TypeAlias.JobHandle.md) | Job handle returned by `appkit.jobs("etl")`. Supports OBO access via `.asUser(req)`. | | [JobsExport](TypeAlias.JobsExport.md) | Public API shape of the jobs plugin. Callable to select a job by key. | +| [Matcher](TypeAlias.Matcher.md) | A deterministic matcher: inspects a string value and returns a result. | | [PluginData](TypeAlias.PluginData.md) | Tuple of plugin class, config, and name. Created by `toPlugin()` and passed to `createApp()`. | | [Plugins](TypeAlias.Plugins.md) | Plugin map passed to the function form of [AgentDefinition.tools](Interface.AgentDefinition.md#tools). Each entry exposes a `.toolkit(opts?)` method that returns a record of [ToolkitEntry](Interface.ToolkitEntry.md) markers ready to be spread into a tool record. | | [ResolvedToolEntry](TypeAlias.ResolvedToolEntry.md) | Internal tool-index entry after a tool record has been resolved to a dispatchable form. | | [ResourceFieldEntry](TypeAlias.ResourceFieldEntry.md) | - | | [ResourcePermission](TypeAlias.ResourcePermission.md) | Union of all possible permission levels across all resource types. | -| [SearchFilters](TypeAlias.SearchFilters.md) | - | | [ServingFactory](TypeAlias.ServingFactory.md) | Factory function returned by `AppKit.serving`. | -| [SupervisorTool](TypeAlias.SupervisorTool.md) | Tools supported by the Databricks AI Gateway Responses API. The shapes match the wire format the endpoint expects, so the adapter passes the array straight into the request body. | +| [Severity](TypeAlias.Severity.md) | Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). | | [ToolRegistry](TypeAlias.ToolRegistry.md) | - | | [ToPlugin](TypeAlias.ToPlugin.md) | Factory function type returned by `toPlugin()`. Accepts optional config and returns a PluginData tuple. | @@ -131,12 +133,9 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | Variable | Description | | ------ | ------ | -| [agents](Variable.agents.md) | Plugin factory for the agents plugin. Discovers agents from `server/agents//agent.{ts,md}` by default (markdown still in `config/agents/` is read as a deprecated fallback), resolves toolkits/tools from registered plugins, exposes the `appkit.agents.*` runtime API and mounts `POST /invocations` and `POST /responses` (aliased non-streaming invoke endpoints) plus `POST /chat` (streaming, HITL-capable). | -| [aiSearch](Variable.aiSearch.md) | - | +| [agents](Variable.agents.md) | Plugin factory for the agents plugin. Reads `config/agents/*.md` by default, resolves toolkits/tools from registered plugins, exposes `appkit.agents.*` runtime API and mounts `POST /invocations` and `POST /responses` (aliased non-streaming invoke endpoints) plus `POST /chat` (streaming, HITL-capable). | | [READ\_ACTIONS](Variable.READ_ACTIONS.md) | Actions that only read data. | | [sql](Variable.sql.md) | SQL helper namespace | -| [SUPERVISOR\_EXTENSION\_KEY](Variable.SUPERVISOR_EXTENSION_KEY.md) | Namespace key under which the adapter reads its hosted-tool payload from [AgentInput.extensions](Interface.AgentInput.md#extensions). Exported so the agents plugin and standalone `runAgent` (the producers) can write under the same key the adapter reads. | -| [supervisorTools](Variable.supervisorTools.md) | Concise factories for declaring Supervisor API tools. | | [WRITE\_ACTIONS](Variable.WRITE_ACTIONS.md) | Actions that mutate data. | ## Functions @@ -146,24 +145,25 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [agentIdFromMarkdownPath](Function.agentIdFromMarkdownPath.md) | Derives the logical agent id from a markdown path. When the file is named `agent.md`, the id is the parent directory name (folder-based layout); otherwise the id is the file stem (e.g. legacy single-file paths). | | [appKitServingTypesPlugin](Function.appKitServingTypesPlugin.md) | Vite plugin to generate TypeScript types for AppKit serving endpoints. Fetches OpenAPI schemas from Databricks and generates a .d.ts with ServingEndpointRegistry module augmentation. | | [appKitTypesPlugin](Function.appKitTypesPlugin.md) | Vite plugin to generate types for AppKit queries. Calls generateFromEntryPoint under the hood. | -| [bigid](Function.bigid.md) | - | -| [bigint](Function.bigint.md) | - | -| [boolean](Function.boolean.md) | - | -| [createAgent](Function.createAgent.md) | Pure factory for agent definitions: cycle-detects the sub-agent graph and returns the same object, stamped with a non-enumerable AGENT\_BRAND so discovery recognizes it. Safe at module top-level; no adapter is built. Don't `Object.freeze` the definition before passing it in — the brand is written onto the argument. | +| [buildAssessment](Function.buildAssessment.md) | Build the single pass/fail Feedback assessment for an eval result. Returns undefined when there's no trace to attach to or the eval was skipped. | +| [createAgent](Function.createAgent.md) | Pure factory for agent definitions. Returns the passed-in definition after cycle-detecting the sub-agent graph. Accepts the full `AgentDefinition` shape and is safe to call at module top-level. | | [createApp](Function.createApp.md) | Bootstraps AppKit with the provided configuration. | +| [createHttpDriver](Function.createHttpDriver.md) | Drives an agent by POSTing to a running app's chat endpoint and parsing the SSE response. Keeps the thread id across `send`s so multi-turn evals share a conversation. Agent/stream errors surface as `succeeded: false` rather than throwing, so `t.succeeded()` can assert on them. | | [createLakebasePool](Function.createLakebasePool.md) | Create a Lakebase pool with appkit's logger integration. Telemetry automatically uses appkit's OpenTelemetry configuration via global registry. | | [createLakebasePoolManager](Function.createLakebasePoolManager.md) | Create a pool manager that maintains per-key Lakebase connection pools. | -| [createWorkspaceClient](Function.createWorkspaceClient.md) | Construct an AppKit workspace client. | -| [database](Function.database.md) | Create a typed database plugin registration for a finalized schema. | -| [defineManifest](Function.defineManifest.md) | Validates a raw manifest (typically a `manifest.json` import) against the canonical Zod schema and returns it as a strict [PluginManifest](Interface.PluginManifest.md). | -| [defineSchema](Function.defineSchema.md) | Compile one declared schema. The returned type keeps the table names the builder returned, so `crudRoutes` and `hooks` can name only real tables. | +| [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `config/agents//evals/*.eval.ts` file. | +| [defineEvalConfig](Function.defineEvalConfig.md) | Define per-directory eval config. Default-export from `evals.config.ts`. | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | -| [enumColumn](Function.enumColumn.md) | - | +| [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/config/agents//evals/`. The agent id is the directory name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Returns a stable, sorted list. | +| [equals](Function.equals.md) | Passes when the value equals `expected` exactly. | +| [evalGlyph](Function.evalGlyph.md) | Status glyph for a single eval result. | | [executeFromRegistry](Function.executeFromRegistry.md) | Validates tool-call arguments against the entry's schema and invokes its handler. On validation failure, returns an LLM-friendly error string (matching the behavior of `tool()`) rather than throwing, so the model can self-correct on its next turn. | | [extractServingEndpoints](Function.extractServingEndpoints.md) | Extract serving endpoint config from a server file by AST-parsing it. Looks for `serving({ endpoints: { alias: { env: "..." }, ... } })` calls and extracts the endpoint alias names and their environment variable mappings. | | [findServerFile](Function.findServerFile.md) | Find the server entry file by checking candidate paths in order. | -| [fk](Function.fk.md) | Declare foreign-key to another column. | -| [fromSupervisorApi](Function.fromSupervisorApi.md) | Creates an [AgentAdapter](Interface.AgentAdapter.md) backed by the Databricks AI Gateway Responses API (`/ai-gateway/mlflow/v1/responses`). | +| [formatEvalDetail](Function.formatEvalDetail.md) | Indented detail lines for a failing eval (error + failing assertions). | +| [formatEvalHeadline](Function.formatEvalHeadline.md) | The one-line header for a single eval result (no failure detail). | +| [formatEvalResults](Function.formatEvalResults.md) | Render all results as a human-readable console report (non-streaming). | +| [formatSummaryLine](Function.formatSummaryLine.md) | The final PASS/FAIL summary line. | | [functionToolToDefinition](Function.functionToolToDefinition.md) | - | | [generateDatabaseCredential](Function.generateDatabaseCredential.md) | Generate OAuth credentials for Postgres database connection using the proper Postgres API. | | [getExecutionContext](Function.getExecutionContext.md) | Get the current execution context. | @@ -173,23 +173,21 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [getResourceRequirements](Function.getResourceRequirements.md) | Gets the resource requirements from a plugin's manifest. | | [getUsernameWithApiLookup](Function.getUsernameWithApiLookup.md) | Resolves the PostgreSQL username for a Lakebase connection. | | [getWorkspaceClient](Function.getWorkspaceClient.md) | Get workspace client from config or SDK default auth chain | -| [id](Function.id.md) | - | -| [integer](Function.integer.md) | - | +| [includes](Function.includes.md) | Passes when the value contains `substring`. | | [isFunctionTool](Function.isFunctionTool.md) | - | | [isHostedTool](Function.isHostedTool.md) | - | | [isSQLTypeMarker](Function.isSQLTypeMarker.md) | Type guard to check if a value is a SQL type marker | -| [isSupervisorTool](Function.isSupervisorTool.md) | Type guard for [HostedSupervisorTool](Interface.HostedSupervisorTool.md). Used by the agents plugin (`buildToolIndex`) and standalone `runAgent` (`classifyTool`) to route supervisor-hosted tools to the extensions payload rather than the adapter's `tools` array. | | [isToolkitEntry](Function.isToolkitEntry.md) | Type guard for `ToolkitEntry` — used by the agents plugin to differentiate toolkit references from inline tools in a mixed `tools` record. | -| [jsonb](Function.jsonb.md) | - | | [loadAgentFromFile](Function.loadAgentFromFile.md) | Loads a single markdown agent file and resolves its frontmatter against registered plugin toolkits + ambient tool library. | | [loadAgentsFromDir](Function.loadAgentsFromDir.md) | Scans a directory for one subdirectory per agent, each containing `agent.md` (frontmatter + body). Produces an `AgentDefinition` record keyed by agent id (folder name). Throws on frontmatter errors or unresolved references. Returns an empty map if the directory does not exist. | +| [matches](Function.matches.md) | Passes when the value matches `pattern`. | | [mcpServer](Function.mcpServer.md) | Factory for declaring a custom MCP server tool. | | [parseTextToolCalls](Function.parseTextToolCalls.md) | Parses text-based tool calls from model output. | +| [reportToMlflow](Function.reportToMlflow.md) | Write one pass/fail assessment per eval result to the Databricks MLflow REST API. Never throws — failures are collected so the run still reports. | | [resolveHostedTools](Function.resolveHostedTools.md) | - | | [runAgent](Function.runAgent.md) | Standalone agent execution without `createApp`. Resolves the adapter, binds inline tools, and drives the adapter's `run()` loop to completion. | -| [text](Function.text.md) | - | -| [timestamp](Function.timestamp.md) | - | +| [runEval](Function.runEval.md) | Run a single eval against a driver. Never throws for assertion or agent failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed eval definition surfaces as `result.error`. | +| [runEvalsInDir](Function.runEvalsInDir.md) | Discover, load, and run every eval under each agent's `evals/` dir, driving the agents on a running app. Never throws for an individual eval — load/run failures become non-passing [EvalResult](Interface.EvalResult.md)s. | +| [summarize](Function.summarize.md) | - | | [tool](Function.tool.md) | Factory for defining function tools with Zod schemas. | | [toolsFromRegistry](Function.toolsFromRegistry.md) | Produces the `AgentToolDefinition[]` a ToolProvider exposes to the LLM, deriving `parameters` JSON Schema from each entry's Zod schema. | -| [uuid](Function.uuid.md) | - | -| [varchar](Function.varchar.md) | - | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index 3999e830c..bac9bf8bf 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -81,11 +81,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Class.ServerError", label: "ServerError" }, - { - type: "doc", - id: "api/appkit/Class.SupervisorApiAdapter", - label: "SupervisorApiAdapter" - }, { type: "doc", id: "api/appkit/Class.TunnelError", @@ -132,6 +127,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.AgentToolDefinition", label: "AgentToolDefinition" }, + { + type: "doc", + id: "api/appkit/Interface.AssertionHandle", + label: "AssertionHandle" + }, + { + type: "doc", + id: "api/appkit/Interface.AssertionResult", + label: "AssertionResult" + }, + { + type: "doc", + id: "api/appkit/Interface.Assessment", + label: "Assessment" + }, { type: "doc", id: "api/appkit/Interface.AutoInheritToolsConfig", @@ -154,14 +164,49 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Interface.DatabaseRegistry", - label: "DatabaseRegistry" + id: "api/appkit/Interface.DiscoveredEval", + label: "DiscoveredEval" + }, + { + type: "doc", + id: "api/appkit/Interface.DriveResult", + label: "DriveResult" }, { type: "doc", id: "api/appkit/Interface.EndpointConfig", label: "EndpointConfig" }, + { + type: "doc", + id: "api/appkit/Interface.EvalConfig", + label: "EvalConfig" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalDefinition", + label: "EvalDefinition" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalDriver", + label: "EvalDriver" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalResult", + label: "EvalResult" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalRunSummary", + label: "EvalRunSummary" + }, + { + type: "doc", + id: "api/appkit/Interface.EvalSummary", + label: "EvalSummary" + }, { type: "doc", id: "api/appkit/Interface.FilePolicyUser", @@ -184,29 +229,14 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Interface.GenerationParams", - label: "GenerationParams" - }, - { - type: "doc", - id: "api/appkit/Interface.HostedSupervisorTool", - label: "HostedSupervisorTool" - }, - { - type: "doc", - id: "api/appkit/Interface.IAiSearchConfig", - label: "IAiSearchConfig" + id: "api/appkit/Interface.HttpDriverOptions", + label: "HttpDriverOptions" }, { type: "doc", id: "api/appkit/Interface.IJobsConfig", label: "IJobsConfig" }, - { - type: "doc", - id: "api/appkit/Interface.IndexConfig", - label: "IndexConfig" - }, { type: "doc", id: "api/appkit/Interface.ITelemetry", @@ -242,6 +272,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.LakebasePoolManager", label: "LakebasePoolManager" }, + { + type: "doc", + id: "api/appkit/Interface.MatchResult", + label: "MatchResult" + }, { type: "doc", id: "api/appkit/Interface.McpConnectAllResult", @@ -252,6 +287,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.Message", label: "Message" }, + { + type: "doc", + id: "api/appkit/Interface.MlflowReportOptions", + label: "MlflowReportOptions" + }, { type: "doc", id: "api/appkit/Interface.PluginManifest", @@ -272,6 +312,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.RegisteredAgent", label: "RegisteredAgent" }, + { + type: "doc", + id: "api/appkit/Interface.ReportOutcome", + label: "ReportOutcome" + }, { type: "doc", id: "api/appkit/Interface.RequestedClaims", @@ -282,11 +327,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.RequestedResource", label: "RequestedResource" }, - { - type: "doc", - id: "api/appkit/Interface.RerankerConfig", - label: "RerankerConfig" - }, { type: "doc", id: "api/appkit/Interface.ResourceEntry", @@ -309,23 +349,13 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Interface.Schema", - label: "Schema" - }, - { - type: "doc", - id: "api/appkit/Interface.SearchRequest", - label: "SearchRequest" + id: "api/appkit/Interface.RunEvalOptions", + label: "RunEvalOptions" }, { type: "doc", - id: "api/appkit/Interface.SearchResponse", - label: "SearchResponse" - }, - { - type: "doc", - id: "api/appkit/Interface.SearchResult", - label: "SearchResult" + id: "api/appkit/Interface.RunEvalsOptions", + label: "RunEvalsOptions" }, { type: "doc", @@ -344,18 +374,13 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Interface.SupervisorApiAdapterOptions", - label: "SupervisorApiAdapterOptions" - }, - { - type: "doc", - id: "api/appkit/Interface.SupervisorExtension", - label: "SupervisorExtension" + id: "api/appkit/Interface.TelemetryConfig", + label: "TelemetryConfig" }, { type: "doc", - id: "api/appkit/Interface.TelemetryConfig", - label: "TelemetryConfig" + id: "api/appkit/Interface.TestContext", + label: "TestContext" }, { type: "doc", @@ -401,21 +426,6 @@ const typedocSidebar: SidebarsConfig = { type: "doc", id: "api/appkit/Interface.ValidationResult", label: "ValidationResult" - }, - { - type: "doc", - id: "api/appkit/Interface.WorkspaceClient", - label: "WorkspaceClient" - }, - { - type: "doc", - id: "api/appkit/Interface.WorkspaceClientLike", - label: "WorkspaceClientLike" - }, - { - type: "doc", - id: "api/appkit/Interface.WorkspaceClientOptions", - label: "WorkspaceClientOptions" } ] }, @@ -455,8 +465,8 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/TypeAlias.DatabaseExports", - label: "DatabaseExports" + id: "api/appkit/TypeAlias.EvalProgress", + label: "EvalProgress" }, { type: "doc", @@ -485,14 +495,19 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/TypeAlias.IDatabaseConfig", - label: "IDatabaseConfig" + id: "api/appkit/TypeAlias.JobHandle", + label: "JobHandle" }, { type: "doc", id: "api/appkit/TypeAlias.JobsExport", label: "JobsExport" }, + { + type: "doc", + id: "api/appkit/TypeAlias.Matcher", + label: "Matcher" + }, { type: "doc", id: "api/appkit/TypeAlias.PluginData", @@ -518,11 +533,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.ResourcePermission", label: "ResourcePermission" }, - { - type: "doc", - id: "api/appkit/TypeAlias.SearchFilters", - label: "SearchFilters" - }, { type: "doc", id: "api/appkit/TypeAlias.ServingFactory", @@ -530,8 +540,8 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/TypeAlias.SupervisorTool", - label: "SupervisorTool" + id: "api/appkit/TypeAlias.Severity", + label: "Severity" }, { type: "doc", @@ -554,11 +564,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Variable.agents", label: "agents" }, - { - type: "doc", - id: "api/appkit/Variable.aiSearch", - label: "aiSearch" - }, { type: "doc", id: "api/appkit/Variable.READ_ACTIONS", @@ -569,16 +574,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Variable.sql", label: "sql" }, - { - type: "doc", - id: "api/appkit/Variable.SUPERVISOR_EXTENSION_KEY", - label: "SUPERVISOR_EXTENSION_KEY" - }, - { - type: "doc", - id: "api/appkit/Variable.supervisorTools", - label: "supervisorTools" - }, { type: "doc", id: "api/appkit/Variable.WRITE_ACTIONS", @@ -607,18 +602,8 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Function.bigid", - label: "bigid" - }, - { - type: "doc", - id: "api/appkit/Function.bigint", - label: "bigint" - }, - { - type: "doc", - id: "api/appkit/Function.boolean", - label: "boolean" + id: "api/appkit/Function.buildAssessment", + label: "buildAssessment" }, { type: "doc", @@ -630,6 +615,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.createApp", label: "createApp" }, + { + type: "doc", + id: "api/appkit/Function.createHttpDriver", + label: "createHttpDriver" + }, { type: "doc", id: "api/appkit/Function.createLakebasePool", @@ -642,13 +632,13 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Function.createWorkspaceClient", - label: "createWorkspaceClient" + id: "api/appkit/Function.defineEval", + label: "defineEval" }, { type: "doc", - id: "api/appkit/Function.database", - label: "database" + id: "api/appkit/Function.defineEvalConfig", + label: "defineEvalConfig" }, { type: "doc", @@ -657,18 +647,23 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Function.defineSchema", - label: "defineSchema" + id: "api/appkit/Function.defineTool", + label: "defineTool" }, { type: "doc", - id: "api/appkit/Function.defineTool", - label: "defineTool" + id: "api/appkit/Function.discoverEvalFiles", + label: "discoverEvalFiles" }, { type: "doc", - id: "api/appkit/Function.enumColumn", - label: "enumColumn" + id: "api/appkit/Function.equals", + label: "equals" + }, + { + type: "doc", + id: "api/appkit/Function.evalGlyph", + label: "evalGlyph" }, { type: "doc", @@ -687,13 +682,23 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Function.fk", - label: "fk" + id: "api/appkit/Function.formatEvalDetail", + label: "formatEvalDetail" }, { type: "doc", - id: "api/appkit/Function.fromSupervisorApi", - label: "fromSupervisorApi" + id: "api/appkit/Function.formatEvalHeadline", + label: "formatEvalHeadline" + }, + { + type: "doc", + id: "api/appkit/Function.formatEvalResults", + label: "formatEvalResults" + }, + { + type: "doc", + id: "api/appkit/Function.formatSummaryLine", + label: "formatSummaryLine" }, { type: "doc", @@ -742,13 +747,8 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Function.id", - label: "id" - }, - { - type: "doc", - id: "api/appkit/Function.integer", - label: "integer" + id: "api/appkit/Function.includes", + label: "includes" }, { type: "doc", @@ -765,21 +765,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.isSQLTypeMarker", label: "isSQLTypeMarker" }, - { - type: "doc", - id: "api/appkit/Function.isSupervisorTool", - label: "isSupervisorTool" - }, { type: "doc", id: "api/appkit/Function.isToolkitEntry", label: "isToolkitEntry" }, - { - type: "doc", - id: "api/appkit/Function.jsonb", - label: "jsonb" - }, { type: "doc", id: "api/appkit/Function.loadAgentFromFile", @@ -790,6 +780,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.loadAgentsFromDir", label: "loadAgentsFromDir" }, + { + type: "doc", + id: "api/appkit/Function.matches", + label: "matches" + }, { type: "doc", id: "api/appkit/Function.mcpServer", @@ -800,6 +795,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.parseTextToolCalls", label: "parseTextToolCalls" }, + { + type: "doc", + id: "api/appkit/Function.reportToMlflow", + label: "reportToMlflow" + }, { type: "doc", id: "api/appkit/Function.resolveHostedTools", @@ -812,13 +812,18 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Function.text", - label: "text" + id: "api/appkit/Function.runEval", + label: "runEval" }, { type: "doc", - id: "api/appkit/Function.timestamp", - label: "timestamp" + id: "api/appkit/Function.runEvalsInDir", + label: "runEvalsInDir" + }, + { + type: "doc", + id: "api/appkit/Function.summarize", + label: "summarize" }, { type: "doc", @@ -829,16 +834,6 @@ const typedocSidebar: SidebarsConfig = { type: "doc", id: "api/appkit/Function.toolsFromRegistry", label: "toolsFromRegistry" - }, - { - type: "doc", - id: "api/appkit/Function.uuid", - label: "uuid" - }, - { - type: "doc", - id: "api/appkit/Function.varchar", - label: "varchar" } ] } diff --git a/packages/appkit/package.json b/packages/appkit/package.json index 8a30fbd9d..951d909a9 100644 --- a/packages/appkit/package.json +++ b/packages/appkit/package.json @@ -88,6 +88,7 @@ "@opentelemetry/semantic-conventions": "1.38.0", "@types/semver": "7.7.1", "apache-arrow": "21.1.0", + "autoevals": "^0.3.0", "dotenv": "16.6.1", "drizzle-orm": "0.45.2", "express": "4.22.2", diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index 7143d993a..bcfecfe02 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -1,10 +1,16 @@ export { defineEval, defineEvalConfig } from "./define-eval"; export { type DiscoveredEval, discoverEvalFiles } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; +export { + configureJudge, + isJudgeConfigured, + type JudgeConfig, + type JudgeScore, +} from "./judge"; export { equals, includes, matches } from "./matchers"; export { type Assessment, - buildAssessment, + buildAssessments, type MlflowReportOptions, type ReportOutcome, reportToMlflow, @@ -28,6 +34,7 @@ export { export type { AssertionHandle, AssertionResult, + CustomJudgeSpec, DriveResult, EvalConfig, EvalDefinition, diff --git a/packages/appkit/src/evals/judge.ts b/packages/appkit/src/evals/judge.ts new file mode 100644 index 000000000..7588d9970 --- /dev/null +++ b/packages/appkit/src/evals/judge.ts @@ -0,0 +1,108 @@ +import { normalizeHost } from "./mlflow-rest"; + +/** + * LLM-as-judge scoring via the `autoevals` library (the same scorers eve uses), + * pointed at a Databricks serving endpoint. autoevals talks to an + * OpenAI-compatible API; Databricks Model Serving exposes one at + * `/serving-endpoints`, so we set `OPENAI_BASE_URL`/`OPENAI_API_KEY` and + * use the judge endpoint name as the model. + * + * There is no public REST to call Databricks' built-in judges directly (they're + * Python/SDK-only and the rubric prompts live in the mlflow package), so we run + * autoevals' equivalent scorers against a Databricks judge model. + */ +type AutoEvals = typeof import("autoevals"); + +let mod: AutoEvals | undefined; +let enabled = false; + +export interface JudgeConfig { + /** Databricks host (scheme optional). */ + host: string; + /** Bearer token for the serving endpoint. */ + token: string; + /** Serving endpoint name used as the judge model. */ + model: string; +} + +/** A normalized judge result. `score` is 0..1. */ +export interface JudgeScore { + score: number; + rationale?: string; +} + +/** + * Configure the judge once. Sets the OpenAI-compatible client env autoevals + * reads and the default judge model. No-op-safe: on failure, judging stays + * disabled and {@link isJudgeConfigured} returns false. + */ +export async function configureJudge(config: JudgeConfig): Promise { + try { + mod = await import("autoevals"); + process.env.OPENAI_BASE_URL = `${normalizeHost(config.host)}/serving-endpoints`; + process.env.OPENAI_API_KEY = config.token; + mod.init({ defaultModel: config.model }); + enabled = true; + } catch { + enabled = false; + } +} + +export function isJudgeConfigured(): boolean { + return enabled; +} + +/** Normalize an autoevals `Score` into a `JudgeScore`. */ +export function toJudgeScore(s: { + score?: number | null; + metadata?: Record; +}): JudgeScore { + const rationale = s.metadata?.rationale; + return { + score: typeof s.score === "number" ? s.score : 0, + rationale: typeof rationale === "string" ? rationale : undefined, + }; +} + +function ensure(): AutoEvals { + if (!enabled || !mod) { + throw new Error( + "LLM judge is not configured. Set --judge-model (and DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*", + ); + } + return mod; +} + +/** Factuality of `output` vs an `expected` reference. */ +export async function judgeFactuality(args: { + input: string; + output: string; + expected: string; +}): Promise { + return toJudgeScore(await ensure().Factuality(args)); +} + +/** Whether `output` answers the question in `input`, optionally constrained by `criteria`. */ +export async function judgeClosedQA(args: { + input: string; + output: string; + criteria: string; +}): Promise { + return toJudgeScore(await ensure().ClosedQA(args)); +} + +/** + * A custom LLM judge defined by a prompt template + choice→score map — the + * TypeScript analog of MLflow's custom `@scorer`. + */ +export async function judgeCustom( + spec: { + name: string; + promptTemplate: string; + choiceScores: Record; + }, + args: { input: string; output: string }, +): Promise { + const scorer = ensure().LLMClassifierFromTemplate(spec); + return toJudgeScore(await scorer(args)); +} diff --git a/packages/appkit/src/evals/mlflow-report.ts b/packages/appkit/src/evals/mlflow-report.ts index f801bbd65..2136782c9 100644 --- a/packages/appkit/src/evals/mlflow-report.ts +++ b/packages/appkit/src/evals/mlflow-report.ts @@ -25,32 +25,58 @@ export interface ReportOutcome { } /** - * Build the single pass/fail Feedback assessment for an eval result. Returns - * undefined when there's no trace to attach to or the eval was skipped. + * MLflow assessment names reject `.` (and we avoid spaces/parens too), so map + * anything outside `[A-Za-z0-9_-]` to `_`. The judge check on the raw label + * (`judge.`-prefixed) is unaffected — it runs before sanitization. */ -export function buildAssessment(result: EvalResult): Assessment | undefined { - if (!result.traceId || result.skipped) return undefined; +function sanitizeName(label: string): string { + return label.replace(/[^A-Za-z0-9_-]/g, "_"); +} - const failed = result.assertions.filter((a) => !a.pass); - const rationale = result.error - ? `error: ${result.error}` - : failed.length - ? failed - .map( - (a) => - `${a.severity}:${a.label}${a.detail ? ` (${a.detail})` : ""}`, - ) - .join("; ") - : "all assertions passed"; +/** + * Build the Feedback assessments for an eval result: one per assertion (judge + * assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they + * render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. + * Returns [] when there's no trace to attach to or the eval was skipped. + */ +export function buildAssessments(result: EvalResult): Assessment[] { + if (!result.traceId || result.skipped) return []; + const traceId = result.traceId; + const out: Assessment[] = []; + const used = new Map(); - return { - trace_id: result.traceId, + for (const a of result.assertions) { + const isJudge = a.label.startsWith("judge."); + const base = sanitizeName(a.label); + const seen = used.get(base) ?? 0; + used.set(base, seen + 1); + out.push({ + trace_id: traceId, + assessment_name: seen === 0 ? base : `${base}_${seen + 1}`, + source: isJudge + ? { source_type: "LLM_JUDGE", source_id: "appkit-judge" } + : { source_type: "CODE", source_id: "appkit-eval" }, + // Judges report a numeric score; deterministic assertions a boolean. + feedback: { value: a.score ?? a.pass }, + rationale: a.detail, + metadata: { eval_id: result.id, severity: a.severity }, + }); + } + + out.push({ + trace_id: traceId, assessment_name: "appkit_eval", source: { source_type: "CODE", source_id: "appkit-eval" }, feedback: { value: result.passed }, - rationale, + rationale: result.error + ? `error: ${result.error}` + : result.passed + ? "all gates passed" + : "one or more gates failed", metadata: { eval_id: result.id }, - }; + }); + + return out; } async function postAssessment( @@ -93,20 +119,22 @@ export async function reportToMlflow( ): Promise { const outcome: ReportOutcome = { written: 0, skipped: 0, failures: [] }; for (const result of results) { - const assessment = buildAssessment(result); - if (!assessment) { + const assessments = buildAssessments(result); + if (assessments.length === 0) { outcome.skipped++; continue; } - const res = await postAssessment(options.host, options.token, assessment); - if (res.ok) { - outcome.written++; - } else { - outcome.failures.push({ - traceId: assessment.trace_id, - status: res.status, - error: res.error, - }); + for (const assessment of assessments) { + const res = await postAssessment(options.host, options.token, assessment); + if (res.ok) { + outcome.written++; + } else { + outcome.failures.push({ + traceId: assessment.trace_id, + status: res.status, + error: res.error, + }); + } } } return outcome; diff --git a/packages/appkit/src/evals/mlflow-run.ts b/packages/appkit/src/evals/mlflow-run.ts index 5806a322a..4253bc83c 100644 --- a/packages/appkit/src/evals/mlflow-run.ts +++ b/packages/appkit/src/evals/mlflow-run.ts @@ -4,6 +4,13 @@ import type { EvalResult } from "./types"; /** Run tag value that makes a run appear under the experiment's "Evaluation runs". */ const GENAI_EVALUATE_RUN_TYPE = "genai_evaluate"; +/** + * Source tags on the eval run. Traces linked via `mlflow.sourceRun` surface the + * run's source in the traces-table "Source" column (mirrors how Python's + * `evaluate()` shows the script name), so tagging the run is what populates it. + */ +const EVAL_SOURCE_NAME = "appkit agent eval"; + interface CreateRunResponse { run: { info: { run_id?: string; run_uuid?: string } }; } @@ -34,17 +41,17 @@ export async function createEvalRun( experiment_id: options.experimentId, start_time: options.startTime, ...(options.runName ? { run_name: options.runName } : {}), + tags: [ + { key: "mlflow.runType", value: GENAI_EVALUATE_RUN_TYPE }, + { key: "mlflow.source.name", value: EVAL_SOURCE_NAME }, + { key: "mlflow.source.type", value: "LOCAL" }, + ], }, ); const runId = created.run?.info?.run_id ?? created.run?.info?.run_uuid; if (!runId) { throw new Error("runs/create returned no run id"); } - await mlflowPost(options, "/api/2.0/mlflow/runs/set-tag", { - run_id: runId, - key: "mlflow.runType", - value: GENAI_EVALUATE_RUN_TYPE, - }); return runId; } diff --git a/packages/appkit/src/evals/run-eval.ts b/packages/appkit/src/evals/run-eval.ts index 637508f20..e227af757 100644 --- a/packages/appkit/src/evals/run-eval.ts +++ b/packages/appkit/src/evals/run-eval.ts @@ -1,3 +1,4 @@ +import { judgeClosedQA, judgeCustom, judgeFactuality } from "./judge"; import type { AssertionHandle, AssertionResult, @@ -8,6 +9,9 @@ import type { TestContext, } from "./types"; +/** Default pass threshold for an LLM-judge score (0..1) before `.atLeast()`. */ +const DEFAULT_JUDGE_THRESHOLD = 0.5; + /** Thrown by `t.skip()` to unwind the test and mark the eval skipped. */ class SkipSignal extends Error { constructor(public reason?: string) { @@ -36,6 +40,7 @@ export async function runEval( ): Promise { const assertions: AssertionResult[] = []; let reply = ""; + let lastInput = ""; let toolCalls: string[] = []; let sessionId: string | undefined; let lastTraceId: string | undefined; @@ -73,8 +78,18 @@ export async function runEval( return handle; }; + // LLM-judge assertions are scored and soft by default; the caller chains + // `.atLeast(n)` to set the pass threshold or `.gate()` to promote. + const recordJudge = ( + label: string, + score: number, + rationale?: string, + ): AssertionHandle => + record(label, score >= DEFAULT_JUDGE_THRESHOLD, score, rationale).soft(); + const t: TestContext = { async send(message) { + lastInput = message; const r = await options.driver.send(message); reply = r.reply; toolCalls = r.toolCalls; @@ -113,6 +128,31 @@ export async function runEval( const m = matcher(value); return record("check", m.pass, m.score, m.detail); }, + judge: { + async factuality(expected) { + const { score, rationale } = await judgeFactuality({ + input: lastInput, + output: reply, + expected, + }); + return recordJudge("judge.factuality", score, rationale); + }, + async closedQA(criteria) { + const { score, rationale } = await judgeClosedQA({ + input: lastInput, + output: reply, + criteria, + }); + return recordJudge("judge.closedQA", score, rationale); + }, + async custom(spec) { + const { score, rationale } = await judgeCustom(spec, { + input: lastInput, + output: reply, + }); + return recordJudge(`judge.${spec.name}`, score, rationale); + }, + }, skip(reason) { throw new SkipSignal(reason); }, diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 4cbd51c01..2a531add2 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -1,6 +1,7 @@ import { pathToFileURL } from "node:url"; import { discoverEvalFiles } from "./discover"; import { createHttpDriver } from "./http-driver"; +import { configureJudge } from "./judge"; import { type ReportOutcome, reportToMlflow } from "./mlflow-report"; import { createEvalRun, type FinishOutcome, finishEvalRun } from "./mlflow-run"; import { runEval } from "./run-eval"; @@ -23,6 +24,11 @@ export interface RunEvalsOptions { * are logged. Requires Databricks creds + the target experiment. */ mlflow?: { host: string; token: string; experimentId: string }; + /** + * When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a + * Databricks serving endpoint (`model`). + */ + judge?: { host: string; token: string; model: string }; /** Wall-clock timestamp (ms) for run create/finish — pass `Date.now()`. */ now?: number; /** Progress callback, invoked as evals are discovered, started, and finished. */ @@ -110,6 +116,10 @@ export async function runEvalsInDir( const total = discovered.length; emit({ type: "discovered", total }); + if (options.judge) { + await configureJudge(options.judge); + } + // Create the MLflow evaluation run up front so each eval's trace can be // linked to it as it runs. let runId: string | undefined; diff --git a/packages/appkit/src/evals/tests/judge.test.ts b/packages/appkit/src/evals/tests/judge.test.ts new file mode 100644 index 000000000..7052d6a95 --- /dev/null +++ b/packages/appkit/src/evals/tests/judge.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "vitest"; +import { isJudgeConfigured, toJudgeScore } from "../judge"; + +describe("judge score mapping", () => { + test("toJudgeScore reads score and rationale from an autoevals Score", () => { + expect( + toJudgeScore({ score: 0.8, metadata: { rationale: "mostly correct" } }), + ).toEqual({ score: 0.8, rationale: "mostly correct" }); + }); + + test("toJudgeScore defaults a missing/null score to 0 and omits rationale", () => { + expect(toJudgeScore({ score: null })).toEqual({ score: 0 }); + expect(toJudgeScore({})).toEqual({ score: 0 }); + expect(toJudgeScore({ score: 1, metadata: { rationale: 42 } })).toEqual({ + score: 1, + }); + }); + + test("judging is disabled until configured", () => { + expect(isJudgeConfigured()).toBe(false); + }); +}); diff --git a/packages/appkit/src/evals/tests/mlflow-report.test.ts b/packages/appkit/src/evals/tests/mlflow-report.test.ts index 141556400..db658c131 100644 --- a/packages/appkit/src/evals/tests/mlflow-report.test.ts +++ b/packages/appkit/src/evals/tests/mlflow-report.test.ts @@ -1,57 +1,73 @@ import { describe, expect, test } from "vitest"; -import { buildAssessment } from "../mlflow-report"; +import { buildAssessments } from "../mlflow-report"; import type { EvalResult } from "../types"; -describe("buildAssessment", () => { - test("builds a pass assessment with trace id and source", () => { +describe("buildAssessments", () => { + test("emits one feedback per assertion plus an overall appkit_eval", () => { const result: EvalResult = { id: "support/weather", traceId: "tr-123", - assertions: [{ label: "succeeded", severity: "gate", pass: true }], + assertions: [ + { label: "succeeded", severity: "gate", pass: true }, + { + label: "judge.closedQA", + severity: "soft", + pass: true, + score: 0.9, + detail: "clearly relevant", + }, + ], passed: true, }; - const a = buildAssessment(result); - expect(a).toEqual({ - trace_id: "tr-123", - assessment_name: "appkit_eval", - source: { source_type: "CODE", source_id: "appkit-eval" }, - feedback: { value: true }, - rationale: "all assertions passed", - metadata: { eval_id: "support/weather" }, - }); + const out = buildAssessments(result); + expect(out.map((a) => a.assessment_name)).toEqual([ + "succeeded", + "judge_closedQA", + "appkit_eval", + ]); + + const judge = out.find((a) => a.assessment_name === "judge_closedQA"); + expect(judge?.source.source_type).toBe("LLM_JUDGE"); + expect(judge?.feedback.value).toBe(0.9); // numeric score, not boolean + expect(judge?.rationale).toBe("clearly relevant"); + + const succeeded = out.find((a) => a.assessment_name === "succeeded"); + expect(succeeded?.source.source_type).toBe("CODE"); + expect(succeeded?.feedback.value).toBe(true); + + const overall = out.find((a) => a.assessment_name === "appkit_eval"); + expect(overall?.feedback.value).toBe(true); }); - test("fail assessment summarizes failing assertions in the rationale", () => { - const a = buildAssessment({ - id: "support/x", - traceId: "tr-9", + test("sanitizes and de-duplicates assertion names", () => { + const out = buildAssessments({ + id: "x", + traceId: "tr-1", assertions: [ - { label: "succeeded", severity: "gate", pass: true }, - { - label: "calledTool(get_weather)", - severity: "gate", - pass: false, - detail: "not called", - }, + { label: "calledTool(get_weather)", severity: "gate", pass: true }, + { label: "check", severity: "gate", pass: true }, + { label: "check", severity: "gate", pass: true }, ], - passed: false, + passed: true, }); - expect(a?.feedback.value).toBe(false); - expect(a?.rationale).toContain("gate:calledTool(get_weather) (not called)"); + const names = out.map((a) => a.assessment_name); + expect(names).toContain("calledTool_get_weather_"); + expect(names).toContain("check"); + expect(names).toContain("check_2"); }); - test("returns undefined without a trace id or when skipped", () => { - expect( - buildAssessment({ id: "x", assertions: [], passed: true }), - ).toBeUndefined(); + test("returns [] without a trace id or when skipped", () => { + expect(buildAssessments({ id: "x", assertions: [], passed: true })).toEqual( + [], + ); expect( - buildAssessment({ + buildAssessments({ id: "x", traceId: "tr-1", assertions: [], passed: true, skipped: { reason: "no data" }, }), - ).toBeUndefined(); + ).toEqual([]); }); }); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts index c6ef02e1a..16a7d3bed 100644 --- a/packages/appkit/src/evals/types.ts +++ b/packages/appkit/src/evals/types.ts @@ -84,10 +84,31 @@ export interface TestContext { calledTool(name: string): AssertionHandle; /** Assert a value against a matcher, e.g. `t.check(t.reply, includes("Sunny"))`. */ check(value: string, matcher: Matcher): AssertionHandle; + /** + * LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge + * model). Each returns a scored, soft-by-default assertion; chain `.atLeast(n)` + * to set the pass threshold or `.gate()` to make it a hard gate. Requires the + * judge to be configured (`--judge-model`). + */ + judge: { + /** Score factuality of the reply against an expected reference. */ + factuality(expected: string): Promise; + /** Score whether the reply answers the question, per optional `criteria`. */ + closedQA(criteria: string): Promise; + /** A custom prompt-template judge (the TS analog of MLflow's `@scorer`). */ + custom(spec: CustomJudgeSpec): Promise; + }; /** Skip this eval with an optional reason. */ skip(reason?: string): never; } +/** A custom LLM-judge definition: a prompt template and choice→score mapping. */ +export interface CustomJudgeSpec { + name: string; + promptTemplate: string; + choiceScores: Record; +} + /** A single eval, default-exported from a `*.eval.ts` file. */ export interface EvalDefinition { /** Short human description, shown in reports. */ diff --git a/packages/appkit/src/plugins/agents/agents.ts b/packages/appkit/src/plugins/agents/agents.ts index ea4e2ed51..17f2371ee 100644 --- a/packages/appkit/src/plugins/agents/agents.ts +++ b/packages/appkit/src/plugins/agents/agents.ts @@ -65,6 +65,8 @@ import { initAgentTracing, linkTraceToRun, traceAgent, + traceTool, + updateTracePreview, } from "./mlflow"; import { composePromptForAgent } from "./prompt"; import { @@ -1314,6 +1316,18 @@ export class AgentsPlugin extends Plugin implements ToolProvider { }); } + // Populate the trace-level Request/Response columns and the trace + // name in the MLflow traces table (span inputs/outputs and the span + // name don't fill these). + const lastUserMessage = [...thread.messages] + .reverse() + .find((m) => m.role === "user")?.content; + updateTracePreview({ + request: lastUserMessage, + response: fullContent || undefined, + name: registered.name || "agent", + }); + // Surface the MLflow trace id so eval runs can attach assessments // to this turn's trace. No-op when tracing is disabled. const mlflowTraceId = currentTraceId(); diff --git a/packages/appkit/src/plugins/agents/mlflow.ts b/packages/appkit/src/plugins/agents/mlflow.ts index cd48b8190..d990b903d 100644 --- a/packages/appkit/src/plugins/agents/mlflow.ts +++ b/packages/appkit/src/plugins/agents/mlflow.ts @@ -155,3 +155,30 @@ export function linkTraceToRun(runId: string): void { logger.warn("Failed to link trace to run %s: %O", runId, err); } } + +/** + * Populate the trace-level Request/Response columns and the trace name shown in + * the MLflow traces table (span-level inputs/outputs don't fill these, and the + * Trace-name column reads the `mlflow.traceName` tag, not the root span name). + * The Source column is run-derived (via `mlflow.sourceRun`), so a live chat + * turn with no run leaves it empty — it's only set for eval runs, where the run + * itself carries the source tags. No-op when tracing is off. + */ +export function updateTracePreview(opts: { + request?: string; + response?: string; + name?: string; +}): void { + if (!enabled || !mlflow) return; + try { + mlflow.updateCurrentTrace({ + ...(opts.request !== undefined ? { requestPreview: opts.request } : {}), + ...(opts.response !== undefined + ? { responsePreview: opts.response } + : {}), + ...(opts.name ? { tags: { "mlflow.traceName": opts.name } } : {}), + }); + } catch (err) { + logger.warn("Failed to update trace preview: %O", err); + } +} diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index ee6924230..fa3275677 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -28,6 +28,7 @@ interface EvalRunner { strict?: boolean; headers?: Record; mlflow?: { host: string; token: string; experimentId: string }; + judge?: { host: string; token: string; model: string }; onEvent?: (event: EvalProgress) => void; }): Promise; evalGlyph(result: unknown): string; @@ -72,6 +73,7 @@ interface EvalOptions { databricksHost?: string; databricksToken?: string; experiment?: string; + judgeModel?: string; } async function runAgentEval( @@ -89,6 +91,13 @@ async function runAgentEval( const mlflow = host && token && experimentId ? { host, token, experimentId } : undefined; + // LLM-as-judge: reuse the Databricks creds + a judge serving endpoint. + const judgeModel = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL; + const judge = + judgeModel && host && token + ? { host, token, model: judgeModel } + : undefined; + // Stream progress as evals run, instead of going silent until the end. const onEvent = (event: EvalProgress): void => { switch (event.type) { @@ -122,6 +131,7 @@ async function runAgentEval( strict: opts.strict, headers: opts.header ? parseHeaders(opts.header) : undefined, mlflow, + judge, onEvent, }); console.log(`\n${runner.formatSummaryLine(summary.results)}`); @@ -188,4 +198,8 @@ export const agentEvalCommand = new Command("eval") "--experiment ", "MLflow experiment id for the evaluation run (default: MLFLOW_EXPERIMENT_ID)", ) + .option( + "--judge-model ", + "Databricks serving endpoint to use as the LLM judge for t.judge.* (default: APPKIT_JUDGE_MODEL)", + ) .action(runAgentEval); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1657de0f8..4257d145b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -309,6 +309,9 @@ importers: apache-arrow: specifier: 21.1.0 version: 21.1.0 + autoevals: + specifier: ^0.3.0 + version: 0.3.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.3.6) dotenv: specifier: 16.6.1 version: 16.6.1 @@ -5736,9 +5739,6 @@ packages: ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} - ajv@8.17.1: - resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} - ajv@8.18.0: resolution: {integrity: sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==} @@ -5862,6 +5862,12 @@ packages: autocomplete.js@0.37.1: resolution: {integrity: sha512-PgSe9fHYhZEsm/9jggbjtVsGXJkPLvd+9mC7gZJ662vVL5CRWEtm/mIrrzCx0MrNxHVwxD5d00UOn6NsmL2LUQ==} + autoevals@0.3.0: + resolution: {integrity: sha512-4CEzBVhjVBHvk46s+DBcgmEfOdM+zXEoCkvvDqYO2IWdVpZKUJANfX8BvfE5vfcGy24on9zW21Zf7V9cUJdbfg==} + engines: {npm: please-use-pnpm, pnpm: '>=10.27.0', yarn: please-use-pnpm} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + autoprefixer@10.4.21: resolution: {integrity: sha512-O+A6LWV5LDHSJD3LjHYoNi4VLsj/Whi7k6zG12xTYaU4cQ8oxQGckXNX8cRHK5yOZ/ppVHe0ZBXGzSV9jXdVbQ==} engines: {node: ^10 || ^12 || >=14} @@ -5963,6 +5969,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + binary-search@1.3.6: + resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==} + birpc@4.0.0: resolution: {integrity: sha512-LShSxJP0KTmd101b6DRyGBj57LZxSDYWKitQNW/mi8GRMvZb078Uf9+pveax1DrVL89vm7mWe+TovdI/UDOuPw==} @@ -6164,6 +6173,9 @@ packages: resolution: {integrity: sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==} engines: {node: '>=20.18.1'} + cheminfo-types@1.15.0: + resolution: {integrity: sha512-shv45WN2u0yN9EHH1bisNrv+fy4Cw+eLM5lOoriP67mePrwbHZ1kJqg90C8GEU7K1A8gJsicEoVZHcuBbuul/w==} + chevrotain-allstar@0.3.1: resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} peerDependencies: @@ -6356,6 +6368,15 @@ packages: resolution: {integrity: sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==} engines: {node: '>= 0.8.0'} + compute-cosine-similarity@1.1.0: + resolution: {integrity: sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==} + + compute-dot@1.1.0: + resolution: {integrity: sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==} + + compute-l2norm@1.1.0: + resolution: {integrity: sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -7523,6 +7544,9 @@ packages: fflate@0.8.3: resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} + fft.js@4.0.4: + resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==} + figures@3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -8270,6 +8294,9 @@ packages: is-alphanumerical@2.0.1: resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + is-any-array@3.0.0: + resolution: {integrity: sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww==} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} @@ -8505,6 +8532,10 @@ packages: joi@17.13.3: resolution: {integrity: sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA==} + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -8725,6 +8756,9 @@ packages: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} + linear-sum-assignment@1.0.9: + resolution: {integrity: sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==} + lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} @@ -9268,6 +9302,24 @@ packages: engines: {node: '>=10'} hasBin: true + ml-array-max@2.0.0: + resolution: {integrity: sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A==} + + ml-array-min@2.0.0: + resolution: {integrity: sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg==} + + ml-array-rescale@2.0.0: + resolution: {integrity: sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg==} + + ml-matrix@6.12.2: + resolution: {integrity: sha512-GC+BnW+pBh8Auap8goAxY0senAmF0IEoc3HNVSfnfbvGw0buuDIYb9kAKMS1l+GiwJ1rfK2bzJ8IHhwjzATSFA==} + + ml-spectra-processing@14.29.0: + resolution: {integrity: sha512-825CS864krbjMv7OB0mbjgAmyOL5ymj1OGa0gAzz1h1Dcd3Eeol2DaOimSiPYmRhW+iYhpeQnb7cSU0mlSK6+g==} + + ml-xsadd@3.0.1: + resolution: {integrity: sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==} + mlflow-tracing@0.1.3: resolution: {integrity: sha512-Koqkwaid5ubGHuLprBP6J7Su70WddlD11f2vgzgxbFFHYKsAsJatMGvjIck5CkyhT/gMUyBqpA3Lkl+zC3W3uQ==} engines: {node: '>=18'} @@ -9302,6 +9354,10 @@ packages: resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} hasBin: true + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mute-stream@2.0.0: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} @@ -9526,6 +9582,17 @@ packages: resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} engines: {node: '>=12'} + openai@6.44.0: + resolution: {integrity: sha512-09/gH+8jH0RgUwsgWHAaxsKGRT5zVZ95IaJUnqAWj6XejIBmnFRwq2WUIF37VtDEsmGrtPmvCs5+yBSeZGWvkA==} + peerDependencies: + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + opener@1.5.2: resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} hasBin: true @@ -11883,6 +11950,12 @@ packages: resolution: {integrity: sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A==} engines: {node: ^20.17.0 || >=22.9.0} + validate.io-array@1.0.6: + resolution: {integrity: sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==} + + validate.io-function@1.0.2: + resolution: {integrity: sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==} + validator@13.15.26: resolution: {integrity: sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA==} engines: {node: '>= 0.10'} @@ -12298,6 +12371,11 @@ packages: resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} engines: {node: '>=18'} + zod-to-json-schema@3.25.0: + resolution: {integrity: sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==} + peerDependencies: + zod: ^3.25 || ^4 + zod-validation-error@4.0.2: resolution: {integrity: sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==} engines: {node: '>=18.0.0'} @@ -13495,7 +13573,7 @@ snapshots: '@commitlint/config-validator@19.8.1': dependencies: '@commitlint/types': 19.8.1 - ajv: 8.17.1 + ajv: 8.18.0 '@commitlint/ensure@19.8.1': dependencies: @@ -18481,9 +18559,9 @@ snapshots: '@opentelemetry/api': 1.9.0 zod: 4.3.6 - ajv-formats@2.1.1(ajv@8.17.1): + ajv-formats@2.1.1(ajv@8.18.0): optionalDependencies: - ajv: 8.17.1 + ajv: 8.18.0 ajv-formats@3.0.1(ajv@8.18.0): optionalDependencies: @@ -18493,9 +18571,9 @@ snapshots: dependencies: ajv: 6.12.6 - ajv-keywords@5.1.0(ajv@8.17.1): + ajv-keywords@5.1.0(ajv@8.18.0): dependencies: - ajv: 8.17.1 + ajv: 8.18.0 fast-deep-equal: 3.1.3 ajv@6.12.6: @@ -18505,13 +18583,6 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 - ajv@8.17.1: - dependencies: - fast-deep-equal: 3.1.3 - fast-uri: 3.1.0 - json-schema-traverse: 1.0.0 - require-from-string: 2.0.2 - ajv@8.18.0: dependencies: fast-deep-equal: 3.1.3 @@ -18640,6 +18711,20 @@ snapshots: dependencies: immediate: 3.3.0 + autoevals@0.3.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.3.6): + dependencies: + ajv: 8.18.0 + compute-cosine-similarity: 1.1.0 + js-levenshtein: 1.1.6 + js-yaml: 4.2.0 + linear-sum-assignment: 1.0.9 + mustache: 4.2.0 + openai: 6.44.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.3.6) + zod: 4.3.6 + zod-to-json-schema: 3.25.0(zod@4.3.6) + transitivePeerDependencies: + - ws + autoprefixer@10.4.21(postcss@8.5.6): dependencies: browserslist: 4.28.1 @@ -18740,6 +18825,8 @@ snapshots: binary-extensions@2.3.0: {} + binary-search@1.3.6: {} + birpc@4.0.0: {} bl@4.1.0: @@ -19035,6 +19122,8 @@ snapshots: undici: 7.24.5 whatwg-mimetype: 4.0.0 + cheminfo-types@1.15.0: {} + chevrotain-allstar@0.3.1(chevrotain@11.0.3): dependencies: chevrotain: 11.0.3 @@ -19222,6 +19311,23 @@ snapshots: transitivePeerDependencies: - supports-color + compute-cosine-similarity@1.1.0: + dependencies: + compute-dot: 1.1.0 + compute-l2norm: 1.1.0 + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + + compute-dot@1.1.0: + dependencies: + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + + compute-l2norm@1.1.0: + dependencies: + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + concat-map@0.0.1: {} concat-stream@2.0.0: @@ -20404,6 +20510,8 @@ snapshots: fflate@0.8.3: {} + fft.js@4.0.4: {} + figures@3.2.0: dependencies: escape-string-regexp: 1.0.5 @@ -21367,6 +21475,8 @@ snapshots: is-alphabetical: 2.0.1 is-decimal: 2.0.1 + is-any-array@3.0.0: {} + is-arrayish@0.2.1: {} is-binary-path@2.1.0: @@ -21569,6 +21679,8 @@ snapshots: '@sideway/formula': 3.0.1 '@sideway/pinpoint': 2.0.0 + js-levenshtein@1.1.6: {} + js-tokens@4.0.0: {} js-tokens@9.0.1: {} @@ -21777,6 +21889,12 @@ snapshots: lilconfig@3.1.3: {} + linear-sum-assignment@1.0.9: + dependencies: + cheminfo-types: 1.15.0 + ml-matrix: 6.12.2 + ml-spectra-processing: 14.29.0 + lines-and-columns@1.2.4: {} linkify-it@5.0.0: @@ -22615,6 +22733,36 @@ snapshots: mkdirp@3.0.1: {} + ml-array-max@2.0.0: + dependencies: + is-any-array: 3.0.0 + + ml-array-min@2.0.0: + dependencies: + is-any-array: 3.0.0 + + ml-array-rescale@2.0.0: + dependencies: + is-any-array: 3.0.0 + ml-array-max: 2.0.0 + ml-array-min: 2.0.0 + + ml-matrix@6.12.2: + dependencies: + is-any-array: 3.0.0 + ml-array-rescale: 2.0.0 + + ml-spectra-processing@14.29.0: + dependencies: + binary-search: 1.3.6 + cheminfo-types: 1.15.0 + fft.js: 4.0.4 + is-any-array: 3.0.0 + ml-matrix: 6.12.2 + ml-xsadd: 3.0.1 + + ml-xsadd@3.0.1: {} + mlflow-tracing@0.1.3: dependencies: '@databricks/sdk-experimental': 0.15.0 @@ -22654,6 +22802,8 @@ snapshots: dns-packet: 5.6.1 thunky: 1.1.0 + mustache@4.2.0: {} + mute-stream@2.0.0: {} nanoid@3.3.11: {} @@ -22873,6 +23023,11 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 + openai@6.44.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.3.6): + optionalDependencies: + ws: 8.21.0(bufferutil@4.0.9) + zod: 4.3.6 + opener@1.5.2: {} optionator@0.9.4: @@ -24451,9 +24606,9 @@ snapshots: schema-utils@4.3.3: dependencies: '@types/json-schema': 7.0.15 - ajv: 8.17.1 - ajv-formats: 2.1.1(ajv@8.17.1) - ajv-keywords: 5.1.0(ajv@8.17.1) + ajv: 8.18.0 + ajv-formats: 2.1.1(ajv@8.18.0) + ajv-keywords: 5.1.0(ajv@8.18.0) search-insights@2.17.3: {} @@ -24930,7 +25085,7 @@ snapshots: table@6.9.0: dependencies: - ajv: 8.17.1 + ajv: 8.18.0 lodash.truncate: 4.4.2 slice-ansi: 4.0.0 string-width: 4.2.3 @@ -25472,6 +25627,10 @@ snapshots: validate-npm-package-name@7.0.2: {} + validate.io-array@1.0.6: {} + + validate.io-function@1.0.2: {} + validator@13.15.26: {} value-equal@1.0.1: {} @@ -26043,6 +26202,10 @@ snapshots: yoctocolors@2.1.2: {} + zod-to-json-schema@3.25.0(zod@4.3.6): + dependencies: + zod: 4.3.6 + zod-validation-error@4.0.2(zod@4.1.13): dependencies: zod: 4.1.13 From 0437a493bb99ae6ee6da3af7202d9adb8e43189f Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Fri, 10 Jul 2026 11:21:49 +0200 Subject: [PATCH 03/14] refactor(appkit): add mlflow connector with profile-based oauth Introduce connectors/mlflow as the shared REST + auth layer for MLflow, so the eval runner (and future callers) stop threading host/token and hand-rolling fetch/URL logic: - MlflowClient owns {host, token}: normalizes the host once, exposes post() (throws) for runs/* and postResult() (structured failure) for best-effort assessment writes, plus servingEndpointsUrl() for the judge. - resolveDatabricksAuth() mints an OAuth bearer from a CLI profile via the SDK WorkspaceClient (the AppKit-native path), so `agent eval` no longer requires a hand-set DATABRICKS_TOKEN. Adds an `--profile` flag. - Eval run create/finish, assessment reporting, and the judge take the client; the agents plugin's host normalization now delegates to the connector's normalizeHost. The mlflow-tracing SDK wrapper stays in the agents plugin: it manages a process-global provider (like TelemetryManager) and has an agent-shaped API, so it isn't a connector. Signed-off-by: MarioCadenas --- packages/appkit/src/connectors/index.ts | 1 + packages/appkit/src/connectors/mlflow/auth.ts | 55 ++++++++++++ .../appkit/src/connectors/mlflow/client.ts | 90 +++++++++++++++++++ .../appkit/src/connectors/mlflow/index.ts | 6 ++ .../connectors/mlflow/tests/client.test.ts | 82 +++++++++++++++++ packages/appkit/src/evals/index.ts | 9 +- packages/appkit/src/evals/judge.ts | 10 +-- packages/appkit/src/evals/mlflow-report.ts | 46 ++-------- packages/appkit/src/evals/mlflow-rest.ts | 39 -------- packages/appkit/src/evals/mlflow-run.ts | 15 ++-- packages/appkit/src/evals/run-evals.ts | 23 +++-- packages/appkit/src/plugins/agents/mlflow.ts | 3 +- .../shared/src/cli/commands/agent/eval.ts | 32 +++++-- 13 files changed, 305 insertions(+), 106 deletions(-) create mode 100644 packages/appkit/src/connectors/mlflow/auth.ts create mode 100644 packages/appkit/src/connectors/mlflow/client.ts create mode 100644 packages/appkit/src/connectors/mlflow/index.ts create mode 100644 packages/appkit/src/connectors/mlflow/tests/client.test.ts delete mode 100644 packages/appkit/src/evals/mlflow-rest.ts diff --git a/packages/appkit/src/connectors/index.ts b/packages/appkit/src/connectors/index.ts index 438d334af..714871e67 100644 --- a/packages/appkit/src/connectors/index.ts +++ b/packages/appkit/src/connectors/index.ts @@ -4,4 +4,5 @@ export * from "./genie"; export * from "./jobs"; export * from "./lakebase"; export * from "./mcp"; +export * from "./mlflow"; export * from "./sql-warehouse"; diff --git a/packages/appkit/src/connectors/mlflow/auth.ts b/packages/appkit/src/connectors/mlflow/auth.ts new file mode 100644 index 000000000..adbe6d63d --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/auth.ts @@ -0,0 +1,55 @@ +import { WorkspaceClient } from "@databricks/sdk-experimental"; + +/** Resolved Databricks host + bearer token for the eval runner's REST calls. */ +export interface DatabricksAuth { + host: string; + token: string; +} + +export interface ResolveDatabricksAuthOptions { + /** `~/.databrickscfg` profile to authenticate with (e.g. `dogfood`). */ + profile?: string; + /** Explicit host; wins over the profile/SDK-resolved host when set. */ + host?: string; + /** Explicit bearer token; when set, no OAuth is minted (PAT/CI path). */ + token?: string; +} + +/** + * Resolve `{host, token}` for the eval runner the same way the rest of AppKit + * authenticates: construct a Databricks `WorkspaceClient` and let its config + * mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set + * PAT required. An explicit host/token still wins (PAT or CI env), so the SDK + * is only consulted for whatever isn't supplied. + * + * Returns `undefined` when neither an explicit token nor a resolvable profile + * yields a bearer, so the caller can treat auth as simply unavailable. + */ +export async function resolveDatabricksAuth( + options: ResolveDatabricksAuthOptions = {}, +): Promise { + // Fully explicit — no need to touch the SDK. + if (options.host && options.token) { + return { host: options.host, token: options.token }; + } + + try { + const client = new WorkspaceClient( + options.profile ? { profile: options.profile } : {}, + ); + const headers = new Headers(); + // Mints the OAuth access token (or reuses a PAT from the profile) and adds + // an `Authorization: Bearer ` header — the same call the connectors + // use before each request. + await client.config.authenticate(headers); + const token = + options.token ?? headers.get("authorization")?.replace(/^Bearer\s+/i, ""); + const host = + options.host ?? + (await client.config.getHost()).toString().replace(/\/+$/, ""); + if (!token || !host) return undefined; + return { host, token }; + } catch { + return undefined; + } +} diff --git a/packages/appkit/src/connectors/mlflow/client.ts b/packages/appkit/src/connectors/mlflow/client.ts new file mode 100644 index 000000000..b764c9fbb --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/client.ts @@ -0,0 +1,90 @@ +/** Shared client for talking to the Databricks/MLflow REST API. */ + +/** Ensure the host has a scheme (Databricks env often lacks `https://`). */ +export function normalizeHost(raw: string): string { + const h = raw.trim().replace(/\/+$/, ""); + return /^https?:\/\//i.test(h) ? h : `https://${h}`; +} + +/** Structured result for a best-effort POST that must not throw. */ +export interface PostResult { + ok: boolean; + status?: number; + error?: string; +} + +/** + * A thin client over the Databricks workspace REST API, owning the host + bearer + * token so callers (eval-run creation, assessment writes, the judge's serving + * endpoint) don't each re-derive URLs or re-attach auth. The host is normalized + * once at construction. + */ +export class MlflowClient { + /** Normalized workspace base URL (scheme guaranteed, no trailing slash). */ + readonly baseUrl: string; + private readonly token: string; + + constructor(host: string, token: string) { + this.baseUrl = normalizeHost(host); + this.token = token; + } + + private headers(): Record { + return { + "content-type": "application/json", + authorization: `Bearer ${this.token}`, + }; + } + + /** + * POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or + * throws with the status + response text so callers can surface a precise + * error. Use for calls whose failure should abort (e.g. `runs/create`). + */ + async post(path: string, body: unknown): Promise { + const res = await fetch(`${this.baseUrl}${path}`, { + method: "POST", + headers: this.headers(), + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`${path} -> ${res.status} ${text.slice(0, 500)}`); + } + const text = await res.text(); + return (text ? JSON.parse(text) : {}) as T; + } + + /** + * POST JSON without throwing: returns `{ ok, status, error }` so best-effort + * writes (e.g. per-trace assessments) can be collected and reported without + * aborting the run. + */ + async postResult(path: string, body: unknown): Promise { + try { + const res = await fetch(`${this.baseUrl}${path}`, { + method: "POST", + headers: this.headers(), + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { ok: false, status: res.status, error: text.slice(0, 500) }; + } + return { ok: true }; + } catch (err) { + return { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } + } + + /** + * OpenAI-compatible base URL for Databricks Model Serving, used as the judge's + * `OPENAI_BASE_URL`. Same workspace host + token as the MLflow REST calls. + */ + servingEndpointsUrl(): string { + return `${this.baseUrl}/serving-endpoints`; + } +} diff --git a/packages/appkit/src/connectors/mlflow/index.ts b/packages/appkit/src/connectors/mlflow/index.ts new file mode 100644 index 000000000..62506288b --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/index.ts @@ -0,0 +1,6 @@ +export { + type DatabricksAuth, + type ResolveDatabricksAuthOptions, + resolveDatabricksAuth, +} from "./auth"; +export { MlflowClient, normalizeHost, type PostResult } from "./client"; diff --git a/packages/appkit/src/connectors/mlflow/tests/client.test.ts b/packages/appkit/src/connectors/mlflow/tests/client.test.ts new file mode 100644 index 000000000..32c396095 --- /dev/null +++ b/packages/appkit/src/connectors/mlflow/tests/client.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { MlflowClient, normalizeHost } from "../client"; + +describe("normalizeHost", () => { + test("adds https:// when missing and strips trailing slashes", () => { + expect(normalizeHost("workspace.cloud.databricks.com")).toBe( + "https://workspace.cloud.databricks.com", + ); + expect(normalizeHost("https://host.com/")).toBe("https://host.com"); + expect(normalizeHost("http://host.com//")).toBe("http://host.com"); + }); +}); + +describe("MlflowClient", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + test("normalizes host once and derives the serving-endpoints URL", () => { + const client = new MlflowClient("host.databricks.com", "tok"); + expect(client.baseUrl).toBe("https://host.databricks.com"); + expect(client.servingEndpointsUrl()).toBe( + "https://host.databricks.com/serving-endpoints", + ); + }); + + test("post() sends bearer auth to the normalized URL and parses JSON", async () => { + const fetchMock = vi.fn( + async (_url: string, _init: RequestInit) => + new Response(JSON.stringify({ ok: 1 })), + ); + vi.stubGlobal("fetch", fetchMock); + + const client = new MlflowClient("host.com", "secret"); + const body = await client.post("/api/2.0/mlflow/runs/create", { a: 1 }); + + expect(body).toEqual({ ok: 1 }); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("https://host.com/api/2.0/mlflow/runs/create"); + expect(init.method).toBe("POST"); + expect((init.headers as Record).authorization).toBe( + "Bearer secret", + ); + }); + + test("post() throws with status + body on a non-2xx response", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("boom", { status: 400 })), + ); + const client = new MlflowClient("host.com", "t"); + await expect(client.post("/p", {})).rejects.toThrow(/400 boom/); + }); + + test("postResult() returns a structured failure instead of throwing", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("nope", { status: 403 })), + ); + const client = new MlflowClient("host.com", "t"); + expect(await client.postResult("/p", {})).toEqual({ + ok: false, + status: 403, + error: "nope", + }); + }); + + test("postResult() reports ok on success and network errors without throwing", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockRejectedValueOnce(new Error("ECONNREFUSED")); + vi.stubGlobal("fetch", fetchMock); + + const client = new MlflowClient("host.com", "t"); + expect(await client.postResult("/p", {})).toEqual({ ok: true }); + expect(await client.postResult("/p", {})).toEqual({ + ok: false, + error: "ECONNREFUSED", + }); + }); +}); diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index bcfecfe02..506d05cb5 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -1,3 +1,11 @@ +export { + type DatabricksAuth, + MlflowClient, + normalizeHost, + type PostResult, + type ResolveDatabricksAuthOptions, + resolveDatabricksAuth, +} from "../connectors/mlflow"; export { defineEval, defineEvalConfig } from "./define-eval"; export { type DiscoveredEval, discoverEvalFiles } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; @@ -11,7 +19,6 @@ export { equals, includes, matches } from "./matchers"; export { type Assessment, buildAssessments, - type MlflowReportOptions, type ReportOutcome, reportToMlflow, } from "./mlflow-report"; diff --git a/packages/appkit/src/evals/judge.ts b/packages/appkit/src/evals/judge.ts index 7588d9970..1f640af68 100644 --- a/packages/appkit/src/evals/judge.ts +++ b/packages/appkit/src/evals/judge.ts @@ -1,4 +1,4 @@ -import { normalizeHost } from "./mlflow-rest"; +import type { MlflowClient } from "../connectors/mlflow"; /** * LLM-as-judge scoring via the `autoevals` library (the same scorers eve uses), @@ -17,8 +17,8 @@ let mod: AutoEvals | undefined; let enabled = false; export interface JudgeConfig { - /** Databricks host (scheme optional). */ - host: string; + /** Client for the workspace hosting the judge serving endpoint. */ + client: MlflowClient; /** Bearer token for the serving endpoint. */ token: string; /** Serving endpoint name used as the judge model. */ @@ -39,7 +39,7 @@ export interface JudgeScore { export async function configureJudge(config: JudgeConfig): Promise { try { mod = await import("autoevals"); - process.env.OPENAI_BASE_URL = `${normalizeHost(config.host)}/serving-endpoints`; + process.env.OPENAI_BASE_URL = config.client.servingEndpointsUrl(); process.env.OPENAI_API_KEY = config.token; mod.init({ defaultModel: config.model }); enabled = true; @@ -67,7 +67,7 @@ export function toJudgeScore(s: { function ensure(): AutoEvals { if (!enabled || !mod) { throw new Error( - "LLM judge is not configured. Set --judge-model (and DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*", + "LLM judge is not configured. Pass --judge-model and authenticate via --profile (or DATABRICKS_HOST/DATABRICKS_TOKEN) to use t.judge.*", ); } return mod; diff --git a/packages/appkit/src/evals/mlflow-report.ts b/packages/appkit/src/evals/mlflow-report.ts index 2136782c9..859ffaf9d 100644 --- a/packages/appkit/src/evals/mlflow-report.ts +++ b/packages/appkit/src/evals/mlflow-report.ts @@ -1,4 +1,4 @@ -import { normalizeHost } from "./mlflow-rest"; +import type { MlflowClient } from "../connectors/mlflow"; import type { EvalResult } from "./types"; /** A Feedback assessment in the MLflow REST proto-JSON shape. */ @@ -11,13 +11,6 @@ export interface Assessment { metadata?: Record; } -export interface MlflowReportOptions { - /** Databricks workspace host (scheme optional — normalized). */ - host: string; - /** Bearer token for the MLflow REST API. */ - token: string; -} - export interface ReportOutcome { written: number; skipped: number; @@ -79,34 +72,9 @@ export function buildAssessments(result: EvalResult): Assessment[] { return out; } -async function postAssessment( - host: string, - token: string, - assessment: Assessment, -): Promise<{ ok: boolean; status?: number; error?: string }> { - const url = `${normalizeHost(host)}/api/3.0/mlflow/traces/${encodeURIComponent( - assessment.trace_id, - )}/assessments`; - try { - const res = await fetch(url, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ assessment }), - }); - if (!res.ok) { - const text = await res.text().catch(() => ""); - return { ok: false, status: res.status, error: text.slice(0, 500) }; - } - return { ok: true }; - } catch (err) { - return { - ok: false, - error: err instanceof Error ? err.message : String(err), - }; - } +/** REST path for writing a Feedback assessment onto a trace. */ +function assessmentPath(traceId: string): string { + return `/api/3.0/mlflow/traces/${encodeURIComponent(traceId)}/assessments`; } /** @@ -114,8 +82,8 @@ async function postAssessment( * API. Never throws — failures are collected so the run still reports. */ export async function reportToMlflow( + client: MlflowClient, results: EvalResult[], - options: MlflowReportOptions, ): Promise { const outcome: ReportOutcome = { written: 0, skipped: 0, failures: [] }; for (const result of results) { @@ -125,7 +93,9 @@ export async function reportToMlflow( continue; } for (const assessment of assessments) { - const res = await postAssessment(options.host, options.token, assessment); + const res = await client.postResult(assessmentPath(assessment.trace_id), { + assessment, + }); if (res.ok) { outcome.written++; } else { diff --git a/packages/appkit/src/evals/mlflow-rest.ts b/packages/appkit/src/evals/mlflow-rest.ts deleted file mode 100644 index a89912d24..000000000 --- a/packages/appkit/src/evals/mlflow-rest.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** Shared helpers for talking to the Databricks/MLflow REST API. */ - -export interface MlflowRestOptions { - /** Databricks workspace host (scheme optional — normalized). */ - host: string; - /** Bearer token for the MLflow REST API. */ - token: string; -} - -/** Ensure the host has a scheme (Databricks env often lacks `https://`). */ -export function normalizeHost(raw: string): string { - const h = raw.trim().replace(/\/+$/, ""); - return /^https?:\/\//i.test(h) ? h : `https://${h}`; -} - -/** - * POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or throws - * with the status + response text so callers can surface a precise error. - */ -export async function mlflowPost( - options: MlflowRestOptions, - path: string, - body: unknown, -): Promise { - const res = await fetch(`${normalizeHost(options.host)}${path}`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${options.token}`, - }, - body: JSON.stringify(body), - }); - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new Error(`${path} -> ${res.status} ${text.slice(0, 500)}`); - } - const text = await res.text(); - return (text ? JSON.parse(text) : {}) as T; -} diff --git a/packages/appkit/src/evals/mlflow-run.ts b/packages/appkit/src/evals/mlflow-run.ts index 4253bc83c..dadcfc384 100644 --- a/packages/appkit/src/evals/mlflow-run.ts +++ b/packages/appkit/src/evals/mlflow-run.ts @@ -1,4 +1,4 @@ -import { type MlflowRestOptions, mlflowPost } from "./mlflow-rest"; +import type { MlflowClient } from "../connectors/mlflow"; import type { EvalResult } from "./types"; /** Run tag value that makes a run appear under the experiment's "Evaluation runs". */ @@ -28,14 +28,14 @@ interface MlflowMetric { * `mlflow.sourceRun` trace metadata and log results before finishing. */ export async function createEvalRun( - options: MlflowRestOptions & { + client: MlflowClient, + options: { experimentId: string; runName?: string; startTime: number; }, ): Promise { - const created = await mlflowPost( - options, + const created = await client.post( "/api/2.0/mlflow/runs/create", { experiment_id: options.experimentId, @@ -89,7 +89,8 @@ export interface FinishOutcome { * or it would be left stuck in RUNNING forever. */ export async function finishEvalRun( - options: MlflowRestOptions & { + client: MlflowClient, + options: { runId: string; results: EvalResult[]; endTime: number; @@ -100,7 +101,7 @@ export async function finishEvalRun( const metrics = aggregateMetrics(options.results, options.endTime); if (metrics.length) { try { - await mlflowPost(options, "/api/2.0/mlflow/runs/log-batch", { + await client.post("/api/2.0/mlflow/runs/log-batch", { run_id: options.runId, metrics, }); @@ -110,7 +111,7 @@ export async function finishEvalRun( } try { - await mlflowPost(options, "/api/2.0/mlflow/runs/update", { + await client.post("/api/2.0/mlflow/runs/update", { run_id: options.runId, status: "FINISHED", end_time: options.endTime, diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 2a531add2..f31dc38ad 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -1,4 +1,5 @@ import { pathToFileURL } from "node:url"; +import { MlflowClient } from "../connectors/mlflow"; import { discoverEvalFiles } from "./discover"; import { createHttpDriver } from "./http-driver"; import { configureJudge } from "./judge"; @@ -117,15 +118,22 @@ export async function runEvalsInDir( emit({ type: "discovered", total }); if (options.judge) { - await configureJudge(options.judge); + await configureJudge({ + client: new MlflowClient(options.judge.host, options.judge.token), + token: options.judge.token, + model: options.judge.model, + }); } // Create the MLflow evaluation run up front so each eval's trace can be - // linked to it as it runs. + // linked to it as it runs. One client is shared by run create/finish and the + // per-trace assessment writes. let runId: string | undefined; + let mlflowClient: MlflowClient | undefined; if (options.mlflow) { - runId = await createEvalRun({ - ...options.mlflow, + mlflowClient = new MlflowClient(options.mlflow.host, options.mlflow.token); + runId = await createEvalRun(mlflowClient, { + experimentId: options.mlflow.experimentId, runName: `appkit-eval ${new Date(now).toISOString()}`, startTime: now, }); @@ -161,10 +169,9 @@ export async function runEvalsInDir( emit({ type: "result", result, index, total }); } - if (options.mlflow && runId) { - const report = await reportToMlflow(results, options.mlflow); - const finish = await finishEvalRun({ - ...options.mlflow, + if (mlflowClient && runId) { + const report = await reportToMlflow(mlflowClient, results); + const finish = await finishEvalRun(mlflowClient, { runId, results, endTime: options.now ?? Date.now(), diff --git a/packages/appkit/src/plugins/agents/mlflow.ts b/packages/appkit/src/plugins/agents/mlflow.ts index d990b903d..de6d81a23 100644 --- a/packages/appkit/src/plugins/agents/mlflow.ts +++ b/packages/appkit/src/plugins/agents/mlflow.ts @@ -1,3 +1,4 @@ +import { normalizeHost } from "../../connectors/mlflow"; import { createLogger } from "../../logging/logger"; const logger = createLogger("agents"); @@ -24,7 +25,7 @@ function experimentId(): string | undefined { function normalizedDatabricksHost(): string | undefined { const raw = process.env.DATABRICKS_HOST?.trim(); if (!raw) return undefined; - return /^https?:\/\//i.test(raw) ? raw : `https://${raw}`; + return normalizeHost(raw); } /** diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index fa3275677..41390890a 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -31,6 +31,11 @@ interface EvalRunner { judge?: { host: string; token: string; model: string }; onEvent?: (event: EvalProgress) => void; }): Promise; + resolveDatabricksAuth(opts: { + profile?: string; + host?: string; + token?: string; + }): Promise<{ host: string; token: string } | undefined>; evalGlyph(result: unknown): string; formatEvalDetail(result: unknown): string[]; formatSummaryLine(results: unknown[]): string; @@ -70,6 +75,7 @@ interface EvalOptions { strict?: boolean; root?: string; header?: string[]; + profile?: string; databricksHost?: string; databricksToken?: string; experiment?: string; @@ -82,11 +88,19 @@ async function runAgentEval( ): Promise { const runner = await loadRunner(); - // Create a native MLflow "Evaluation run" when Databricks creds + an - // experiment are available (traces live in the app; the run + scores are - // driven from here, so this side needs creds). - const host = opts.databricksHost ?? process.env.DATABRICKS_HOST; - const token = opts.databricksToken ?? process.env.DATABRICKS_TOKEN; + // Resolve Databricks host + bearer the AppKit-native way: an explicit + // host/token (or DATABRICKS_* env) wins; otherwise the SDK mints an OAuth + // token from the CLI profile — so no hand-set PAT is required. + const auth = await runner.resolveDatabricksAuth({ + profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE, + host: opts.databricksHost ?? process.env.DATABRICKS_HOST, + token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN, + }); + const host = auth?.host; + const token = auth?.token; + + // Create a native MLflow "Evaluation run" when creds + an experiment are + // available (traces live in the app; the run + scores are driven from here). const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID; const mlflow = host && token && experimentId ? { host, token, experimentId } : undefined; @@ -158,8 +172,8 @@ async function runAgentEval( } } else { console.log( - "\nMLflow evaluation run skipped — set DATABRICKS_HOST + DATABRICKS_TOKEN" + - " + MLFLOW_EXPERIMENT_ID (or the matching flags) to create one.", + "\nMLflow evaluation run skipped — pass --experiment (or set" + + " MLFLOW_EXPERIMENT_ID) plus --profile/--databricks-host to create one.", ); } @@ -186,6 +200,10 @@ export const agentEvalCommand = new Command("eval") "--header ", "Extra request header as 'Key: value' (repeatable)", ) + .option( + "--profile ", + "Databricks CLI profile to authenticate with via OAuth (default: DATABRICKS_CONFIG_PROFILE)", + ) .option( "--databricks-host ", "Databricks host for writing MLflow assessments (default: DATABRICKS_HOST)", From 3b786cff75df60817f4b8025952694f5bb0ebbcd Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Thu, 13 Aug 2026 11:46:17 +0200 Subject: [PATCH 04/14] docs(appkit): regenerate API reference after rebase onto main Signed-off-by: MarioCadenas --- docs/docs/api/appkit/Class.MlflowClient.md | 102 ++++++++++ .../api/appkit/Function.buildAssessment.md | 18 -- .../api/appkit/Function.buildAssessments.md | 20 ++ .../api/appkit/Function.configureJudge.md | 19 ++ .../api/appkit/Function.isJudgeConfigured.md | 9 + .../docs/api/appkit/Function.normalizeHost.md | 17 ++ .../api/appkit/Function.reportToMlflow.md | 4 +- .../appkit/Function.resolveDatabricksAuth.md | 24 +++ .../api/appkit/Interface.CustomJudgeSpec.md | 27 +++ .../api/appkit/Interface.DatabricksAuth.md | 19 ++ docs/docs/api/appkit/Interface.JudgeConfig.md | 31 +++ docs/docs/api/appkit/Interface.JudgeScore.md | 19 ++ .../appkit/Interface.MlflowReportOptions.md | 21 -- docs/docs/api/appkit/Interface.PostResult.md | 27 +++ .../Interface.ResolveDatabricksAuthOptions.md | 31 +++ .../api/appkit/Interface.RunEvalsOptions.md | 33 ++++ docs/docs/api/appkit/Interface.TestContext.md | 71 +++++++ docs/docs/api/appkit/index.md | 39 +++- docs/docs/api/appkit/typedoc-sidebar.ts | 179 ++++++++++++++++-- 19 files changed, 653 insertions(+), 57 deletions(-) create mode 100644 docs/docs/api/appkit/Class.MlflowClient.md delete mode 100644 docs/docs/api/appkit/Function.buildAssessment.md create mode 100644 docs/docs/api/appkit/Function.buildAssessments.md create mode 100644 docs/docs/api/appkit/Function.configureJudge.md create mode 100644 docs/docs/api/appkit/Function.isJudgeConfigured.md create mode 100644 docs/docs/api/appkit/Function.normalizeHost.md create mode 100644 docs/docs/api/appkit/Function.resolveDatabricksAuth.md create mode 100644 docs/docs/api/appkit/Interface.CustomJudgeSpec.md create mode 100644 docs/docs/api/appkit/Interface.DatabricksAuth.md create mode 100644 docs/docs/api/appkit/Interface.JudgeConfig.md create mode 100644 docs/docs/api/appkit/Interface.JudgeScore.md delete mode 100644 docs/docs/api/appkit/Interface.MlflowReportOptions.md create mode 100644 docs/docs/api/appkit/Interface.PostResult.md create mode 100644 docs/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md diff --git a/docs/docs/api/appkit/Class.MlflowClient.md b/docs/docs/api/appkit/Class.MlflowClient.md new file mode 100644 index 000000000..4b186e397 --- /dev/null +++ b/docs/docs/api/appkit/Class.MlflowClient.md @@ -0,0 +1,102 @@ +# Class: MlflowClient + +A thin client over the Databricks workspace REST API, owning the host + bearer +token so callers (eval-run creation, assessment writes, the judge's serving +endpoint) don't each re-derive URLs or re-attach auth. The host is normalized +once at construction. + +## Constructors + +### Constructor + +```ts +new MlflowClient(host: string, token: string): MlflowClient; +``` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `host` | `string` | +| `token` | `string` | + +#### Returns + +`MlflowClient` + +## Properties + +### baseUrl + +```ts +readonly baseUrl: string; +``` + +Normalized workspace base URL (scheme guaranteed, no trailing slash). + +## Methods + +### post() + +```ts +post(path: string, body: unknown): Promise; +``` + +POST JSON to an MLflow REST endpoint. Returns the parsed JSON body, or +throws with the status + response text so callers can surface a precise +error. Use for calls whose failure should abort (e.g. `runs/create`). + +#### Type Parameters + +| Type Parameter | Default type | +| ------ | ------ | +| `T` | `unknown` | + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `path` | `string` | +| `body` | `unknown` | + +#### Returns + +`Promise`\<`T`\> + +*** + +### postResult() + +```ts +postResult(path: string, body: unknown): Promise; +``` + +POST JSON without throwing: returns `{ ok, status, error }` so best-effort +writes (e.g. per-trace assessments) can be collected and reported without +aborting the run. + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `path` | `string` | +| `body` | `unknown` | + +#### Returns + +`Promise`\<[`PostResult`](Interface.PostResult.md)\> + +*** + +### servingEndpointsUrl() + +```ts +servingEndpointsUrl(): string; +``` + +OpenAI-compatible base URL for Databricks Model Serving, used as the judge's +`OPENAI_BASE_URL`. Same workspace host + token as the MLflow REST calls. + +#### Returns + +`string` diff --git a/docs/docs/api/appkit/Function.buildAssessment.md b/docs/docs/api/appkit/Function.buildAssessment.md deleted file mode 100644 index 62c08b2bd..000000000 --- a/docs/docs/api/appkit/Function.buildAssessment.md +++ /dev/null @@ -1,18 +0,0 @@ -# Function: buildAssessment() - -```ts -function buildAssessment(result: EvalResult): Assessment | undefined; -``` - -Build the single pass/fail Feedback assessment for an eval result. Returns -undefined when there's no trace to attach to or the eval was skipped. - -## Parameters - -| Parameter | Type | -| ------ | ------ | -| `result` | [`EvalResult`](Interface.EvalResult.md) | - -## Returns - -[`Assessment`](Interface.Assessment.md) \| `undefined` diff --git a/docs/docs/api/appkit/Function.buildAssessments.md b/docs/docs/api/appkit/Function.buildAssessments.md new file mode 100644 index 000000000..eda73df95 --- /dev/null +++ b/docs/docs/api/appkit/Function.buildAssessments.md @@ -0,0 +1,20 @@ +# Function: buildAssessments() + +```ts +function buildAssessments(result: EvalResult): Assessment[]; +``` + +Build the Feedback assessments for an eval result: one per assertion (judge +assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they +render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. +Returns [] when there's no trace to attach to or the eval was skipped. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `result` | [`EvalResult`](Interface.EvalResult.md) | + +## Returns + +[`Assessment`](Interface.Assessment.md)[] diff --git a/docs/docs/api/appkit/Function.configureJudge.md b/docs/docs/api/appkit/Function.configureJudge.md new file mode 100644 index 000000000..140042b14 --- /dev/null +++ b/docs/docs/api/appkit/Function.configureJudge.md @@ -0,0 +1,19 @@ +# Function: configureJudge() + +```ts +function configureJudge(config: JudgeConfig): Promise; +``` + +Configure the judge once. Sets the OpenAI-compatible client env autoevals +reads and the default judge model. No-op-safe: on failure, judging stays +disabled and [isJudgeConfigured](Function.isJudgeConfigured.md) returns false. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `config` | [`JudgeConfig`](Interface.JudgeConfig.md) | + +## Returns + +`Promise`\<`void`\> diff --git a/docs/docs/api/appkit/Function.isJudgeConfigured.md b/docs/docs/api/appkit/Function.isJudgeConfigured.md new file mode 100644 index 000000000..04d8f89f0 --- /dev/null +++ b/docs/docs/api/appkit/Function.isJudgeConfigured.md @@ -0,0 +1,9 @@ +# Function: isJudgeConfigured() + +```ts +function isJudgeConfigured(): boolean; +``` + +## Returns + +`boolean` diff --git a/docs/docs/api/appkit/Function.normalizeHost.md b/docs/docs/api/appkit/Function.normalizeHost.md new file mode 100644 index 000000000..2f4606cc6 --- /dev/null +++ b/docs/docs/api/appkit/Function.normalizeHost.md @@ -0,0 +1,17 @@ +# Function: normalizeHost() + +```ts +function normalizeHost(raw: string): string; +``` + +Ensure the host has a scheme (Databricks env often lacks `https://`). + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `raw` | `string` | + +## Returns + +`string` diff --git a/docs/docs/api/appkit/Function.reportToMlflow.md b/docs/docs/api/appkit/Function.reportToMlflow.md index 5d3c3b7b9..1b5803d24 100644 --- a/docs/docs/api/appkit/Function.reportToMlflow.md +++ b/docs/docs/api/appkit/Function.reportToMlflow.md @@ -1,7 +1,7 @@ # Function: reportToMlflow() ```ts -function reportToMlflow(results: EvalResult[], options: MlflowReportOptions): Promise; +function reportToMlflow(client: MlflowClient, results: EvalResult[]): Promise; ``` Write one pass/fail assessment per eval result to the Databricks MLflow REST @@ -11,8 +11,8 @@ API. Never throws — failures are collected so the run still reports. | Parameter | Type | | ------ | ------ | +| `client` | [`MlflowClient`](Class.MlflowClient.md) | | `results` | [`EvalResult`](Interface.EvalResult.md)[] | -| `options` | [`MlflowReportOptions`](Interface.MlflowReportOptions.md) | ## Returns diff --git a/docs/docs/api/appkit/Function.resolveDatabricksAuth.md b/docs/docs/api/appkit/Function.resolveDatabricksAuth.md new file mode 100644 index 000000000..0fc2f1030 --- /dev/null +++ b/docs/docs/api/appkit/Function.resolveDatabricksAuth.md @@ -0,0 +1,24 @@ +# Function: resolveDatabricksAuth() + +```ts +function resolveDatabricksAuth(options: ResolveDatabricksAuthOptions): Promise; +``` + +Resolve `{host, token}` for the eval runner the same way the rest of AppKit +authenticates: construct a Databricks `WorkspaceClient` and let its config +mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set +PAT required. An explicit host/token still wins (PAT or CI env), so the SDK +is only consulted for whatever isn't supplied. + +Returns `undefined` when neither an explicit token nor a resolvable profile +yields a bearer, so the caller can treat auth as simply unavailable. + +## Parameters + +| Parameter | Type | +| ------ | ------ | +| `options` | [`ResolveDatabricksAuthOptions`](Interface.ResolveDatabricksAuthOptions.md) | + +## Returns + +`Promise`\<[`DatabricksAuth`](Interface.DatabricksAuth.md) \| `undefined`\> diff --git a/docs/docs/api/appkit/Interface.CustomJudgeSpec.md b/docs/docs/api/appkit/Interface.CustomJudgeSpec.md new file mode 100644 index 000000000..5bfd881e9 --- /dev/null +++ b/docs/docs/api/appkit/Interface.CustomJudgeSpec.md @@ -0,0 +1,27 @@ +# Interface: CustomJudgeSpec + +A custom LLM-judge definition: a prompt template and choice→score mapping. + +## Properties + +### choiceScores + +```ts +choiceScores: Record; +``` + +*** + +### name + +```ts +name: string; +``` + +*** + +### promptTemplate + +```ts +promptTemplate: string; +``` diff --git a/docs/docs/api/appkit/Interface.DatabricksAuth.md b/docs/docs/api/appkit/Interface.DatabricksAuth.md new file mode 100644 index 000000000..b1d31b35b --- /dev/null +++ b/docs/docs/api/appkit/Interface.DatabricksAuth.md @@ -0,0 +1,19 @@ +# Interface: DatabricksAuth + +Resolved Databricks host + bearer token for the eval runner's REST calls. + +## Properties + +### host + +```ts +host: string; +``` + +*** + +### token + +```ts +token: string; +``` diff --git a/docs/docs/api/appkit/Interface.JudgeConfig.md b/docs/docs/api/appkit/Interface.JudgeConfig.md new file mode 100644 index 000000000..e20780ce0 --- /dev/null +++ b/docs/docs/api/appkit/Interface.JudgeConfig.md @@ -0,0 +1,31 @@ +# Interface: JudgeConfig + +## Properties + +### client + +```ts +client: MlflowClient; +``` + +Client for the workspace hosting the judge serving endpoint. + +*** + +### model + +```ts +model: string; +``` + +Serving endpoint name used as the judge model. + +*** + +### token + +```ts +token: string; +``` + +Bearer token for the serving endpoint. diff --git a/docs/docs/api/appkit/Interface.JudgeScore.md b/docs/docs/api/appkit/Interface.JudgeScore.md new file mode 100644 index 000000000..d4bf0519f --- /dev/null +++ b/docs/docs/api/appkit/Interface.JudgeScore.md @@ -0,0 +1,19 @@ +# Interface: JudgeScore + +A normalized judge result. `score` is 0..1. + +## Properties + +### rationale? + +```ts +optional rationale: string; +``` + +*** + +### score + +```ts +score: number; +``` diff --git a/docs/docs/api/appkit/Interface.MlflowReportOptions.md b/docs/docs/api/appkit/Interface.MlflowReportOptions.md deleted file mode 100644 index 22733c8be..000000000 --- a/docs/docs/api/appkit/Interface.MlflowReportOptions.md +++ /dev/null @@ -1,21 +0,0 @@ -# Interface: MlflowReportOptions - -## Properties - -### host - -```ts -host: string; -``` - -Databricks workspace host (scheme optional — normalized). - -*** - -### token - -```ts -token: string; -``` - -Bearer token for the MLflow REST API. diff --git a/docs/docs/api/appkit/Interface.PostResult.md b/docs/docs/api/appkit/Interface.PostResult.md new file mode 100644 index 000000000..35bcf684a --- /dev/null +++ b/docs/docs/api/appkit/Interface.PostResult.md @@ -0,0 +1,27 @@ +# Interface: PostResult + +Structured result for a best-effort POST that must not throw. + +## Properties + +### error? + +```ts +optional error: string; +``` + +*** + +### ok + +```ts +ok: boolean; +``` + +*** + +### status? + +```ts +optional status: number; +``` diff --git a/docs/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md b/docs/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md new file mode 100644 index 000000000..d13080407 --- /dev/null +++ b/docs/docs/api/appkit/Interface.ResolveDatabricksAuthOptions.md @@ -0,0 +1,31 @@ +# Interface: ResolveDatabricksAuthOptions + +## Properties + +### host? + +```ts +optional host: string; +``` + +Explicit host; wins over the profile/SDK-resolved host when set. + +*** + +### profile? + +```ts +optional profile: string; +``` + +`~/.databrickscfg` profile to authenticate with (e.g. `dogfood`). + +*** + +### token? + +```ts +optional token: string; +``` + +Explicit bearer token; when set, no OAuth is minted (PAT/CI path). diff --git a/docs/docs/api/appkit/Interface.RunEvalsOptions.md b/docs/docs/api/appkit/Interface.RunEvalsOptions.md index 5fe5bbd9c..949d521e2 100644 --- a/docs/docs/api/appkit/Interface.RunEvalsOptions.md +++ b/docs/docs/api/appkit/Interface.RunEvalsOptions.md @@ -32,6 +32,39 @@ Extra request headers for the driver (e.g. auth for a deployed app). *** +### judge? + +```ts +optional judge: { + host: string; + model: string; + token: string; +}; +``` + +When set, enable `t.judge.*` LLM-as-judge scoring via autoevals against a +Databricks serving endpoint (`model`). + +#### host + +```ts +host: string; +``` + +#### model + +```ts +model: string; +``` + +#### token + +```ts +token: string; +``` + +*** + ### mlflow? ```ts diff --git a/docs/docs/api/appkit/Interface.TestContext.md b/docs/docs/api/appkit/Interface.TestContext.md index 5f249952b..e21156235 100644 --- a/docs/docs/api/appkit/Interface.TestContext.md +++ b/docs/docs/api/appkit/Interface.TestContext.md @@ -4,6 +4,77 @@ The `t` context passed to an eval's `test` function. ## Properties +### judge + +```ts +judge: { + closedQA: Promise; + custom: Promise; + factuality: Promise; +}; +``` + +LLM-as-judge scoring of the last reply (via autoevals → a Databricks judge +model). Each returns a scored, soft-by-default assertion; chain `.atLeast(n)` +to set the pass threshold or `.gate()` to make it a hard gate. Requires the +judge to be configured (`--judge-model`). + +#### closedQA() + +```ts +closedQA(criteria: string): Promise; +``` + +Score whether the reply answers the question, per optional `criteria`. + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `criteria` | `string` | + +##### Returns + +`Promise`\<[`AssertionHandle`](Interface.AssertionHandle.md)\> + +#### custom() + +```ts +custom(spec: CustomJudgeSpec): Promise; +``` + +A custom prompt-template judge (the TS analog of MLflow's `@scorer`). + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `spec` | [`CustomJudgeSpec`](Interface.CustomJudgeSpec.md) | + +##### Returns + +`Promise`\<[`AssertionHandle`](Interface.AssertionHandle.md)\> + +#### factuality() + +```ts +factuality(expected: string): Promise; +``` + +Score factuality of the reply against an expected reference. + +##### Parameters + +| Parameter | Type | +| ------ | ------ | +| `expected` | `string` | + +##### Returns + +`Promise`\<[`AssertionHandle`](Interface.AssertionHandle.md)\> + +*** + ### reply ```ts diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index a21025585..d6ec7415d 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -22,10 +22,12 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [DatabricksAdapter](Class.DatabricksAdapter.md) | Adapter that talks directly to Databricks Model Serving `/invocations` endpoint. | | [ExecutionError](Class.ExecutionError.md) | Error thrown when an operation execution fails. Use for statement failures, canceled operations, or unexpected states. | | [InitializationError](Class.InitializationError.md) | Error thrown when a service or component is not properly initialized. Use when accessing services before they are ready. | +| [MlflowClient](Class.MlflowClient.md) | A thin client over the Databricks workspace REST API, owning the host + bearer token so callers (eval-run creation, assessment writes, the judge's serving endpoint) don't each re-derive URLs or re-attach auth. The host is normalized once at construction. | | [Plugin](Class.Plugin.md) | Base abstract class for creating AppKit plugins. | | [PolicyDeniedError](Class.PolicyDeniedError.md) | Thrown when a policy denies an action. | | [ResourceRegistry](Class.ResourceRegistry.md) | Central registry for tracking plugin resource requirements. Deduplication uses type + resourceKey (machine-stable); alias is for display only. | | [ServerError](Class.ServerError.md) | Error thrown when server lifecycle operations fail. Use for server start/stop issues, configuration conflicts, etc. | +| [SupervisorApiAdapter](Class.SupervisorApiAdapter.md) | Adapter that calls the Databricks AI Gateway Responses API (`/ai-gateway/mlflow/v1/responses`). | | [TunnelError](Class.TunnelError.md) | Error thrown when remote tunnel operations fail. Use for tunnel connection issues, message parsing failures, etc. | | [ValidationError](Class.ValidationError.md) | Error thrown when input validation fails. Use for invalid parameters, missing required fields, or type mismatches. | @@ -45,7 +47,9 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [AutoInheritToolsConfig](Interface.AutoInheritToolsConfig.md) | Auto-inherit configuration. When enabled for a given agent origin, agents with no explicit `tools:` declaration receive every registered ToolProvider plugin tool whose author marked `autoInheritable: true`. Tools without that flag — destructive, state-mutating, or privilege-sensitive — never spread automatically and must be wired via `tools:` (object or function form in code, `plugin:NAME` entries in markdown frontmatter). | | [BasePluginConfig](Interface.BasePluginConfig.md) | Base configuration interface for AppKit plugins | | [CacheConfig](Interface.CacheConfig.md) | Configuration for the CacheInterceptor. Controls TTL, size limits, storage backend, and probabilistic cleanup. | +| [CustomJudgeSpec](Interface.CustomJudgeSpec.md) | A custom LLM-judge definition: a prompt template and choice→score mapping. | | [DatabaseCredential](Interface.DatabaseCredential.md) | Database credentials with OAuth token for Postgres connection | +| [DatabricksAuth](Interface.DatabricksAuth.md) | Resolved Databricks host + bearer token for the eval runner's REST calls. | | [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `config/agents//evals/`. | | [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. | | [EndpointConfig](Interface.EndpointConfig.md) | - | @@ -59,35 +63,48 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [FileResource](Interface.FileResource.md) | Describes the file or directory being acted upon. | | [FunctionTool](Interface.FunctionTool.md) | - | | [GenerateDatabaseCredentialRequest](Interface.GenerateDatabaseCredentialRequest.md) | Request parameters for generating database OAuth credentials | +| [GenerationParams](Interface.GenerationParams.md) | Optional generation parameters forwarded to the OpenAI-compatible serving request body. Names match the serving API wire keys. Only keys that are set are sent — undefined values are omitted so the endpoint applies its own defaults. Ranges are not validated here; the serving endpoint validates. | +| [HostedSupervisorTool](Interface.HostedSupervisorTool.md) | Tagged record returned by every [supervisorTools](Variable.supervisorTools.md) factory. The `__kind` discriminator lets the agents plugin (and standalone `runAgent`) classify these tools without a structural match against the wire format — keeps the SA wire shape free to evolve and avoids namespace collisions with MCP hosted tools (which use `type: "genie-space"` hyphenated, vs SA's `type: "genie_space"` underscored). | | [HttpDriverOptions](Interface.HttpDriverOptions.md) | - | +| [IAiSearchConfig](Interface.IAiSearchConfig.md) | Base configuration interface for AppKit plugins | | [IJobsConfig](Interface.IJobsConfig.md) | Configuration for the Jobs plugin. | +| [IndexConfig](Interface.IndexConfig.md) | - | | [ITelemetry](Interface.ITelemetry.md) | Plugin-facing interface for OpenTelemetry instrumentation. Provides a thin abstraction over OpenTelemetry APIs for plugins. | | [JobAPI](Interface.JobAPI.md) | User-facing API for a single configured job. | | [JobConfig](Interface.JobConfig.md) | Per-job configuration options. | | [JobsConnectorConfig](Interface.JobsConnectorConfig.md) | - | +| [JudgeConfig](Interface.JudgeConfig.md) | - | +| [JudgeScore](Interface.JudgeScore.md) | A normalized judge result. `score` is 0..1. | | [LakebasePool](Interface.LakebasePool.md) | Subset of `pg.Pool` exposed by the Lakebase plugin. | | [LakebasePoolConfig](Interface.LakebasePoolConfig.md) | Configuration for creating a Lakebase connection pool | | [LakebasePoolManager](Interface.LakebasePoolManager.md) | Manages multiple Lakebase connection pools keyed by an identifier (e.g. userId). | | [MatchResult](Interface.MatchResult.md) | Result of a deterministic matcher run against a value. | | [McpConnectAllResult](Interface.McpConnectAllResult.md) | Per-endpoint outcome of [AppKitMcpClient.connectAll](Class.AppKitMcpClient.md#connectall). Callers (the agents plugin in particular) use the split to warn at startup when some MCP servers are unreachable without aborting boot for the rest. | | [Message](Interface.Message.md) | - | -| [MlflowReportOptions](Interface.MlflowReportOptions.md) | - | | [PluginManifest](Interface.PluginManifest.md) | Plugin manifest that declares metadata and resource requirements. Attached to plugin classes as a static property. Extends the shared PluginManifest with strict resource types. | | [PluginToolkitProvider](Interface.PluginToolkitProvider.md) | Minimum shape every entry in the [Plugins](TypeAlias.Plugins.md) map must expose. Core plugins (analytics, files, genie, lakebase) implement this directly via their `.toolkit()` method. The agents plugin and standalone `runAgent` synthesize this shape for any registered plugin that doesn't implement `.toolkit()` directly (falling back to `getAgentTools()` walking). | +| [PostResult](Interface.PostResult.md) | Structured result for a best-effort POST that must not throw. | | [PromptContext](Interface.PromptContext.md) | Context passed to `baseSystemPrompt` callbacks. | | [RegisteredAgent](Interface.RegisteredAgent.md) | - | | [ReportOutcome](Interface.ReportOutcome.md) | - | | [RequestedClaims](Interface.RequestedClaims.md) | Optional claims for fine-grained Unity Catalog table permissions When specified, the returned token will be scoped to only the requested tables | | [RequestedResource](Interface.RequestedResource.md) | Resource to request permissions for in Unity Catalog | +| [RerankerConfig](Interface.RerankerConfig.md) | - | +| [ResolveDatabricksAuthOptions](Interface.ResolveDatabricksAuthOptions.md) | - | | [ResourceEntry](Interface.ResourceEntry.md) | Internal representation of a resource in the registry. Extends ResourceRequirement with resolution state and plugin ownership. | | [ResourceRequirement](Interface.ResourceRequirement.md) | Declares a resource requirement for a plugin. Can be defined statically in a manifest or dynamically via getResourceRequirements(). | | [RunAgentInput](Interface.RunAgentInput.md) | - | | [RunAgentResult](Interface.RunAgentResult.md) | - | | [RunEvalOptions](Interface.RunEvalOptions.md) | - | | [RunEvalsOptions](Interface.RunEvalsOptions.md) | - | +| [SearchRequest](Interface.SearchRequest.md) | - | +| [SearchResponse](Interface.SearchResponse.md) | - | +| [SearchResult](Interface.SearchResult.md) | - | | [ServingEndpointEntry](Interface.ServingEndpointEntry.md) | Shape of a single registry entry. | | [ServingEndpointRegistry](Interface.ServingEndpointRegistry.md) | Registry interface for serving endpoint type generation. Empty by default — augmented by the Vite type generator's `.d.ts` output via module augmentation. When populated, provides autocomplete for alias names and typed request/response/chunk per endpoint. | | [StreamExecutionSettings](Interface.StreamExecutionSettings.md) | Execution settings for streaming endpoints. Extends PluginExecutionSettings with SSE stream configuration. | +| [SupervisorApiAdapterOptions](Interface.SupervisorApiAdapterOptions.md) | - | +| [SupervisorExtension](Interface.SupervisorExtension.md) | Shape of the value at `AgentInput.extensions[SUPERVISOR_EXTENSION_KEY]`. The agents plugin / `runAgent` build this from the tool index; advanced callers invoking `adapter.run(...)` directly populate it themselves. | | [TelemetryConfig](Interface.TelemetryConfig.md) | OpenTelemetry configuration for AppKit applications | | [TestContext](Interface.TestContext.md) | The `t` context passed to an eval's `test` function. | | [Thread](Interface.Thread.md) | - | @@ -99,13 +116,16 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [ToolkitOptions](Interface.ToolkitOptions.md) | - | | [ToolProvider](Interface.ToolProvider.md) | - | | [ValidationResult](Interface.ValidationResult.md) | Result of validating all registered resources against the environment. | +| [WorkspaceClient](Interface.WorkspaceClient.md) | AppKit's workspace client facade. Mirrors the multi-client shape of the modular Databricks SDK: each service is its own accessor, so services can be migrated one at a time behind this stable interface. | +| [WorkspaceClientLike](Interface.WorkspaceClientLike.md) | Structural shape of a Databricks SDK client used by [fromSupervisorApi](Function.fromSupervisorApi.md). Only what we need: `apiClient.request` for streaming and `config.ensureResolved` to materialise the host/credentials. | +| [WorkspaceClientOptions](Interface.WorkspaceClientOptions.md) | Options used to construct the wrapper. Mirrors the subset of the old SDK's `Config` + `ClientOptions` that AppKit relies on today; we deliberately do NOT re-expose every old-SDK config knob. | ## Type Aliases | Type Alias | Description | | ------ | ------ | | [AgentEvent](TypeAlias.AgentEvent.md) | - | -| [AgentTool](TypeAlias.AgentTool.md) | Any tool an agent can invoke: inline function tools (`tool()`), hosted MCP tools (`mcpServer()` / raw hosted), or toolkit references from plugins (`analytics().toolkit()`). | +| [AgentTool](TypeAlias.AgentTool.md) | Any tool an agent can invoke: inline function tools (`tool()`), hosted MCP tools (`mcpServer()` / raw hosted), toolkit references from plugins (`analytics().toolkit()`), or adapter-hosted Supervisor-API tools (`supervisorTools.*`). | | [AgentTools](TypeAlias.AgentTools.md) | Per-agent tool record. String keys map to inline tools, toolkit entries, hosted tools, etc. | | [AgentToolsFn](TypeAlias.AgentToolsFn.md) | Function form of `AgentDefinition.tools`. Receives the typed [Plugins](TypeAlias.Plugins.md) map and returns a tool record. Invoked exactly once at setup (or once per `runAgent` call in standalone mode); the result is cached as the agent's resolved tool record. | | [BaseSystemPromptOption](TypeAlias.BaseSystemPromptOption.md) | - | @@ -116,7 +136,6 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [FilePolicy](TypeAlias.FilePolicy.md) | A policy function that decides whether `user` may perform `action` on `resource`. Return `true` to allow, `false` to deny. | | [HostedTool](TypeAlias.HostedTool.md) | - | | [IAppRouter](TypeAlias.IAppRouter.md) | Express router type for plugin route registration | -| [JobHandle](TypeAlias.JobHandle.md) | Job handle returned by `appkit.jobs("etl")`. Supports OBO access via `.asUser(req)`. | | [JobsExport](TypeAlias.JobsExport.md) | Public API shape of the jobs plugin. Callable to select a job by key. | | [Matcher](TypeAlias.Matcher.md) | A deterministic matcher: inspects a string value and returns a result. | | [PluginData](TypeAlias.PluginData.md) | Tuple of plugin class, config, and name. Created by `toPlugin()` and passed to `createApp()`. | @@ -124,8 +143,10 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [ResolvedToolEntry](TypeAlias.ResolvedToolEntry.md) | Internal tool-index entry after a tool record has been resolved to a dispatchable form. | | [ResourceFieldEntry](TypeAlias.ResourceFieldEntry.md) | - | | [ResourcePermission](TypeAlias.ResourcePermission.md) | Union of all possible permission levels across all resource types. | +| [SearchFilters](TypeAlias.SearchFilters.md) | - | | [ServingFactory](TypeAlias.ServingFactory.md) | Factory function returned by `AppKit.serving`. | | [Severity](TypeAlias.Severity.md) | Whether an assertion fails the eval (`gate`) or is tracked only (`soft`). | +| [SupervisorTool](TypeAlias.SupervisorTool.md) | Tools supported by the Databricks AI Gateway Responses API. The shapes match the wire format the endpoint expects, so the adapter passes the array straight into the request body. | | [ToolRegistry](TypeAlias.ToolRegistry.md) | - | | [ToPlugin](TypeAlias.ToPlugin.md) | Factory function type returned by `toPlugin()`. Accepts optional config and returns a PluginData tuple. | @@ -134,8 +155,11 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | Variable | Description | | ------ | ------ | | [agents](Variable.agents.md) | Plugin factory for the agents plugin. Reads `config/agents/*.md` by default, resolves toolkits/tools from registered plugins, exposes `appkit.agents.*` runtime API and mounts `POST /invocations` and `POST /responses` (aliased non-streaming invoke endpoints) plus `POST /chat` (streaming, HITL-capable). | +| [aiSearch](Variable.aiSearch.md) | - | | [READ\_ACTIONS](Variable.READ_ACTIONS.md) | Actions that only read data. | | [sql](Variable.sql.md) | SQL helper namespace | +| [SUPERVISOR\_EXTENSION\_KEY](Variable.SUPERVISOR_EXTENSION_KEY.md) | Namespace key under which the adapter reads its hosted-tool payload from [AgentInput.extensions](Interface.AgentInput.md#extensions). Exported so the agents plugin and standalone `runAgent` (the producers) can write under the same key the adapter reads. | +| [supervisorTools](Variable.supervisorTools.md) | Concise factories for declaring Supervisor API tools. | | [WRITE\_ACTIONS](Variable.WRITE_ACTIONS.md) | Actions that mutate data. | ## Functions @@ -145,12 +169,14 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [agentIdFromMarkdownPath](Function.agentIdFromMarkdownPath.md) | Derives the logical agent id from a markdown path. When the file is named `agent.md`, the id is the parent directory name (folder-based layout); otherwise the id is the file stem (e.g. legacy single-file paths). | | [appKitServingTypesPlugin](Function.appKitServingTypesPlugin.md) | Vite plugin to generate TypeScript types for AppKit serving endpoints. Fetches OpenAPI schemas from Databricks and generates a .d.ts with ServingEndpointRegistry module augmentation. | | [appKitTypesPlugin](Function.appKitTypesPlugin.md) | Vite plugin to generate types for AppKit queries. Calls generateFromEntryPoint under the hood. | -| [buildAssessment](Function.buildAssessment.md) | Build the single pass/fail Feedback assessment for an eval result. Returns undefined when there's no trace to attach to or the eval was skipped. | +| [buildAssessments](Function.buildAssessments.md) | Build the Feedback assessments for an eval result: one per assertion (judge assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. Returns [] when there's no trace to attach to or the eval was skipped. | +| [configureJudge](Function.configureJudge.md) | Configure the judge once. Sets the OpenAI-compatible client env autoevals reads and the default judge model. No-op-safe: on failure, judging stays disabled and [isJudgeConfigured](Function.isJudgeConfigured.md) returns false. | | [createAgent](Function.createAgent.md) | Pure factory for agent definitions. Returns the passed-in definition after cycle-detecting the sub-agent graph. Accepts the full `AgentDefinition` shape and is safe to call at module top-level. | | [createApp](Function.createApp.md) | Bootstraps AppKit with the provided configuration. | | [createHttpDriver](Function.createHttpDriver.md) | Drives an agent by POSTing to a running app's chat endpoint and parsing the SSE response. Keeps the thread id across `send`s so multi-turn evals share a conversation. Agent/stream errors surface as `succeeded: false` rather than throwing, so `t.succeeded()` can assert on them. | | [createLakebasePool](Function.createLakebasePool.md) | Create a Lakebase pool with appkit's logger integration. Telemetry automatically uses appkit's OpenTelemetry configuration via global registry. | | [createLakebasePoolManager](Function.createLakebasePoolManager.md) | Create a pool manager that maintains per-key Lakebase connection pools. | +| [createWorkspaceClient](Function.createWorkspaceClient.md) | Construct an AppKit workspace client. | | [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `config/agents//evals/*.eval.ts` file. | | [defineEvalConfig](Function.defineEvalConfig.md) | Define per-directory eval config. Default-export from `evals.config.ts`. | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | @@ -164,6 +190,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [formatEvalHeadline](Function.formatEvalHeadline.md) | The one-line header for a single eval result (no failure detail). | | [formatEvalResults](Function.formatEvalResults.md) | Render all results as a human-readable console report (non-streaming). | | [formatSummaryLine](Function.formatSummaryLine.md) | The final PASS/FAIL summary line. | +| [fromSupervisorApi](Function.fromSupervisorApi.md) | Creates an [AgentAdapter](Interface.AgentAdapter.md) backed by the Databricks AI Gateway Responses API (`/ai-gateway/mlflow/v1/responses`). | | [functionToolToDefinition](Function.functionToolToDefinition.md) | - | | [generateDatabaseCredential](Function.generateDatabaseCredential.md) | Generate OAuth credentials for Postgres database connection using the proper Postgres API. | | [getExecutionContext](Function.getExecutionContext.md) | Get the current execution context. | @@ -176,14 +203,18 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [includes](Function.includes.md) | Passes when the value contains `substring`. | | [isFunctionTool](Function.isFunctionTool.md) | - | | [isHostedTool](Function.isHostedTool.md) | - | +| [isJudgeConfigured](Function.isJudgeConfigured.md) | - | | [isSQLTypeMarker](Function.isSQLTypeMarker.md) | Type guard to check if a value is a SQL type marker | +| [isSupervisorTool](Function.isSupervisorTool.md) | Type guard for [HostedSupervisorTool](Interface.HostedSupervisorTool.md). Used by the agents plugin (`buildToolIndex`) and standalone `runAgent` (`classifyTool`) to route supervisor-hosted tools to the extensions payload rather than the adapter's `tools` array. | | [isToolkitEntry](Function.isToolkitEntry.md) | Type guard for `ToolkitEntry` — used by the agents plugin to differentiate toolkit references from inline tools in a mixed `tools` record. | | [loadAgentFromFile](Function.loadAgentFromFile.md) | Loads a single markdown agent file and resolves its frontmatter against registered plugin toolkits + ambient tool library. | | [loadAgentsFromDir](Function.loadAgentsFromDir.md) | Scans a directory for one subdirectory per agent, each containing `agent.md` (frontmatter + body). Produces an `AgentDefinition` record keyed by agent id (folder name). Throws on frontmatter errors or unresolved references. Returns an empty map if the directory does not exist. | | [matches](Function.matches.md) | Passes when the value matches `pattern`. | | [mcpServer](Function.mcpServer.md) | Factory for declaring a custom MCP server tool. | +| [normalizeHost](Function.normalizeHost.md) | Ensure the host has a scheme (Databricks env often lacks `https://`). | | [parseTextToolCalls](Function.parseTextToolCalls.md) | Parses text-based tool calls from model output. | | [reportToMlflow](Function.reportToMlflow.md) | Write one pass/fail assessment per eval result to the Databricks MLflow REST API. Never throws — failures are collected so the run still reports. | +| [resolveDatabricksAuth](Function.resolveDatabricksAuth.md) | Resolve `{host, token}` for the eval runner the same way the rest of AppKit authenticates: construct a Databricks `WorkspaceClient` and let its config mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set PAT required. An explicit host/token still wins (PAT or CI env), so the SDK is only consulted for whatever isn't supplied. | | [resolveHostedTools](Function.resolveHostedTools.md) | - | | [runAgent](Function.runAgent.md) | Standalone agent execution without `createApp`. Resolves the adapter, binds inline tools, and drives the adapter's `run()` loop to completion. | | [runEval](Function.runEval.md) | Run a single eval against a driver. Never throws for assertion or agent failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed eval definition surfaces as `result.error`. | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index bac9bf8bf..04267b777 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -61,6 +61,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Class.InitializationError", label: "InitializationError" }, + { + type: "doc", + id: "api/appkit/Class.MlflowClient", + label: "MlflowClient" + }, { type: "doc", id: "api/appkit/Class.Plugin", @@ -81,6 +86,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Class.ServerError", label: "ServerError" }, + { + type: "doc", + id: "api/appkit/Class.SupervisorApiAdapter", + label: "SupervisorApiAdapter" + }, { type: "doc", id: "api/appkit/Class.TunnelError", @@ -157,11 +167,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.CacheConfig", label: "CacheConfig" }, + { + type: "doc", + id: "api/appkit/Interface.CustomJudgeSpec", + label: "CustomJudgeSpec" + }, { type: "doc", id: "api/appkit/Interface.DatabaseCredential", label: "DatabaseCredential" }, + { + type: "doc", + id: "api/appkit/Interface.DatabricksAuth", + label: "DatabricksAuth" + }, { type: "doc", id: "api/appkit/Interface.DiscoveredEval", @@ -227,16 +247,36 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.GenerateDatabaseCredentialRequest", label: "GenerateDatabaseCredentialRequest" }, + { + type: "doc", + id: "api/appkit/Interface.GenerationParams", + label: "GenerationParams" + }, + { + type: "doc", + id: "api/appkit/Interface.HostedSupervisorTool", + label: "HostedSupervisorTool" + }, { type: "doc", id: "api/appkit/Interface.HttpDriverOptions", label: "HttpDriverOptions" }, + { + type: "doc", + id: "api/appkit/Interface.IAiSearchConfig", + label: "IAiSearchConfig" + }, { type: "doc", id: "api/appkit/Interface.IJobsConfig", label: "IJobsConfig" }, + { + type: "doc", + id: "api/appkit/Interface.IndexConfig", + label: "IndexConfig" + }, { type: "doc", id: "api/appkit/Interface.ITelemetry", @@ -257,6 +297,16 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.JobsConnectorConfig", label: "JobsConnectorConfig" }, + { + type: "doc", + id: "api/appkit/Interface.JudgeConfig", + label: "JudgeConfig" + }, + { + type: "doc", + id: "api/appkit/Interface.JudgeScore", + label: "JudgeScore" + }, { type: "doc", id: "api/appkit/Interface.LakebasePool", @@ -287,11 +337,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.Message", label: "Message" }, - { - type: "doc", - id: "api/appkit/Interface.MlflowReportOptions", - label: "MlflowReportOptions" - }, { type: "doc", id: "api/appkit/Interface.PluginManifest", @@ -302,6 +347,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.PluginToolkitProvider", label: "PluginToolkitProvider" }, + { + type: "doc", + id: "api/appkit/Interface.PostResult", + label: "PostResult" + }, { type: "doc", id: "api/appkit/Interface.PromptContext", @@ -327,6 +377,16 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.RequestedResource", label: "RequestedResource" }, + { + type: "doc", + id: "api/appkit/Interface.RerankerConfig", + label: "RerankerConfig" + }, + { + type: "doc", + id: "api/appkit/Interface.ResolveDatabricksAuthOptions", + label: "ResolveDatabricksAuthOptions" + }, { type: "doc", id: "api/appkit/Interface.ResourceEntry", @@ -357,6 +417,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.RunEvalsOptions", label: "RunEvalsOptions" }, + { + type: "doc", + id: "api/appkit/Interface.SearchRequest", + label: "SearchRequest" + }, + { + type: "doc", + id: "api/appkit/Interface.SearchResponse", + label: "SearchResponse" + }, + { + type: "doc", + id: "api/appkit/Interface.SearchResult", + label: "SearchResult" + }, { type: "doc", id: "api/appkit/Interface.ServingEndpointEntry", @@ -372,6 +447,16 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.StreamExecutionSettings", label: "StreamExecutionSettings" }, + { + type: "doc", + id: "api/appkit/Interface.SupervisorApiAdapterOptions", + label: "SupervisorApiAdapterOptions" + }, + { + type: "doc", + id: "api/appkit/Interface.SupervisorExtension", + label: "SupervisorExtension" + }, { type: "doc", id: "api/appkit/Interface.TelemetryConfig", @@ -426,6 +511,21 @@ const typedocSidebar: SidebarsConfig = { type: "doc", id: "api/appkit/Interface.ValidationResult", label: "ValidationResult" + }, + { + type: "doc", + id: "api/appkit/Interface.WorkspaceClient", + label: "WorkspaceClient" + }, + { + type: "doc", + id: "api/appkit/Interface.WorkspaceClientLike", + label: "WorkspaceClientLike" + }, + { + type: "doc", + id: "api/appkit/Interface.WorkspaceClientOptions", + label: "WorkspaceClientOptions" } ] }, @@ -493,11 +593,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.IAppRouter", label: "IAppRouter" }, - { - type: "doc", - id: "api/appkit/TypeAlias.JobHandle", - label: "JobHandle" - }, { type: "doc", id: "api/appkit/TypeAlias.JobsExport", @@ -533,6 +628,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.ResourcePermission", label: "ResourcePermission" }, + { + type: "doc", + id: "api/appkit/TypeAlias.SearchFilters", + label: "SearchFilters" + }, { type: "doc", id: "api/appkit/TypeAlias.ServingFactory", @@ -543,6 +643,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.Severity", label: "Severity" }, + { + type: "doc", + id: "api/appkit/TypeAlias.SupervisorTool", + label: "SupervisorTool" + }, { type: "doc", id: "api/appkit/TypeAlias.ToolRegistry", @@ -564,6 +669,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Variable.agents", label: "agents" }, + { + type: "doc", + id: "api/appkit/Variable.aiSearch", + label: "aiSearch" + }, { type: "doc", id: "api/appkit/Variable.READ_ACTIONS", @@ -574,6 +684,16 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Variable.sql", label: "sql" }, + { + type: "doc", + id: "api/appkit/Variable.SUPERVISOR_EXTENSION_KEY", + label: "SUPERVISOR_EXTENSION_KEY" + }, + { + type: "doc", + id: "api/appkit/Variable.supervisorTools", + label: "supervisorTools" + }, { type: "doc", id: "api/appkit/Variable.WRITE_ACTIONS", @@ -602,8 +722,13 @@ const typedocSidebar: SidebarsConfig = { }, { type: "doc", - id: "api/appkit/Function.buildAssessment", - label: "buildAssessment" + id: "api/appkit/Function.buildAssessments", + label: "buildAssessments" + }, + { + type: "doc", + id: "api/appkit/Function.configureJudge", + label: "configureJudge" }, { type: "doc", @@ -630,6 +755,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.createLakebasePoolManager", label: "createLakebasePoolManager" }, + { + type: "doc", + id: "api/appkit/Function.createWorkspaceClient", + label: "createWorkspaceClient" + }, { type: "doc", id: "api/appkit/Function.defineEval", @@ -700,6 +830,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.formatSummaryLine", label: "formatSummaryLine" }, + { + type: "doc", + id: "api/appkit/Function.fromSupervisorApi", + label: "fromSupervisorApi" + }, { type: "doc", id: "api/appkit/Function.functionToolToDefinition", @@ -760,11 +895,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.isHostedTool", label: "isHostedTool" }, + { + type: "doc", + id: "api/appkit/Function.isJudgeConfigured", + label: "isJudgeConfigured" + }, { type: "doc", id: "api/appkit/Function.isSQLTypeMarker", label: "isSQLTypeMarker" }, + { + type: "doc", + id: "api/appkit/Function.isSupervisorTool", + label: "isSupervisorTool" + }, { type: "doc", id: "api/appkit/Function.isToolkitEntry", @@ -790,6 +935,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.mcpServer", label: "mcpServer" }, + { + type: "doc", + id: "api/appkit/Function.normalizeHost", + label: "normalizeHost" + }, { type: "doc", id: "api/appkit/Function.parseTextToolCalls", @@ -800,6 +950,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.reportToMlflow", label: "reportToMlflow" }, + { + type: "doc", + id: "api/appkit/Function.resolveDatabricksAuth", + label: "resolveDatabricksAuth" + }, { type: "doc", id: "api/appkit/Function.resolveHostedTools", From 71c348745838e9a6ab3b4bce252a002a69d2dbe4 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Tue, 18 Aug 2026 14:55:38 +0200 Subject: [PATCH 05/14] chore(appkit): conform evals + mlflow connector to oxc toolchain Post-rebase integration with main's biome->oxc migration (#538) and the SDK-facade boundary rule (#534): - Route the mlflow connector's auth through createWorkspaceClient instead of importing @databricks/sdk-experimental directly (oxlint no-restricted-imports); behaviour is unchanged. - Apply oxfmt import grouping to the evals + connector files authored before the migration. Signed-off-by: MarioCadenas --- packages/appkit/src/connectors/mlflow/auth.ts | 4 ++-- packages/appkit/src/connectors/mlflow/tests/client.test.ts | 1 + packages/appkit/src/evals/run-evals.ts | 1 + packages/appkit/src/evals/tests/discover.test.ts | 2 ++ packages/appkit/src/evals/tests/judge.test.ts | 1 + packages/appkit/src/evals/tests/matchers.test.ts | 1 + packages/appkit/src/evals/tests/mlflow-report.test.ts | 1 + packages/appkit/src/evals/tests/mlflow-run.test.ts | 1 + packages/appkit/src/evals/tests/report.test.ts | 1 + packages/appkit/src/evals/tests/resolve-default.test.ts | 1 + packages/appkit/src/evals/tests/run-eval.test.ts | 1 + packages/shared/src/cli/commands/agent/index.ts | 1 + 12 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/appkit/src/connectors/mlflow/auth.ts b/packages/appkit/src/connectors/mlflow/auth.ts index adbe6d63d..6b01b43ba 100644 --- a/packages/appkit/src/connectors/mlflow/auth.ts +++ b/packages/appkit/src/connectors/mlflow/auth.ts @@ -1,4 +1,4 @@ -import { WorkspaceClient } from "@databricks/sdk-experimental"; +import { createWorkspaceClient } from "../../workspace-client"; /** Resolved Databricks host + bearer token for the eval runner's REST calls. */ export interface DatabricksAuth { @@ -34,7 +34,7 @@ export async function resolveDatabricksAuth( } try { - const client = new WorkspaceClient( + const client = createWorkspaceClient( options.profile ? { profile: options.profile } : {}, ); const headers = new Headers(); diff --git a/packages/appkit/src/connectors/mlflow/tests/client.test.ts b/packages/appkit/src/connectors/mlflow/tests/client.test.ts index 32c396095..2e44ea132 100644 --- a/packages/appkit/src/connectors/mlflow/tests/client.test.ts +++ b/packages/appkit/src/connectors/mlflow/tests/client.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, test, vi } from "vitest"; + import { MlflowClient, normalizeHost } from "../client"; describe("normalizeHost", () => { diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index f31dc38ad..92c911aa3 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -1,4 +1,5 @@ import { pathToFileURL } from "node:url"; + import { MlflowClient } from "../connectors/mlflow"; import { discoverEvalFiles } from "./discover"; import { createHttpDriver } from "./http-driver"; diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts index 6eb7c2208..29ecf0ae6 100644 --- a/packages/appkit/src/evals/tests/discover.test.ts +++ b/packages/appkit/src/evals/tests/discover.test.ts @@ -1,7 +1,9 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; + import { afterEach, beforeEach, describe, expect, test } from "vitest"; + import { discoverEvalFiles } from "../discover"; let root: string; diff --git a/packages/appkit/src/evals/tests/judge.test.ts b/packages/appkit/src/evals/tests/judge.test.ts index 7052d6a95..45b92ea9a 100644 --- a/packages/appkit/src/evals/tests/judge.test.ts +++ b/packages/appkit/src/evals/tests/judge.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "vitest"; + import { isJudgeConfigured, toJudgeScore } from "../judge"; describe("judge score mapping", () => { diff --git a/packages/appkit/src/evals/tests/matchers.test.ts b/packages/appkit/src/evals/tests/matchers.test.ts index 03653b520..4836a88f6 100644 --- a/packages/appkit/src/evals/tests/matchers.test.ts +++ b/packages/appkit/src/evals/tests/matchers.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "vitest"; + import { equals, includes, matches } from "../matchers"; describe("eval matchers", () => { diff --git a/packages/appkit/src/evals/tests/mlflow-report.test.ts b/packages/appkit/src/evals/tests/mlflow-report.test.ts index db658c131..38f959132 100644 --- a/packages/appkit/src/evals/tests/mlflow-report.test.ts +++ b/packages/appkit/src/evals/tests/mlflow-report.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "vitest"; + import { buildAssessments } from "../mlflow-report"; import type { EvalResult } from "../types"; diff --git a/packages/appkit/src/evals/tests/mlflow-run.test.ts b/packages/appkit/src/evals/tests/mlflow-run.test.ts index 8c4318cd6..872e0b11c 100644 --- a/packages/appkit/src/evals/tests/mlflow-run.test.ts +++ b/packages/appkit/src/evals/tests/mlflow-run.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "vitest"; + import { aggregateMetrics } from "../mlflow-run"; import type { EvalResult } from "../types"; diff --git a/packages/appkit/src/evals/tests/report.test.ts b/packages/appkit/src/evals/tests/report.test.ts index c02c68843..49b80fe89 100644 --- a/packages/appkit/src/evals/tests/report.test.ts +++ b/packages/appkit/src/evals/tests/report.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "vitest"; + import { formatEvalResults, summarize } from "../report"; import type { EvalResult } from "../types"; diff --git a/packages/appkit/src/evals/tests/resolve-default.test.ts b/packages/appkit/src/evals/tests/resolve-default.test.ts index 40079226c..2dcc8a449 100644 --- a/packages/appkit/src/evals/tests/resolve-default.test.ts +++ b/packages/appkit/src/evals/tests/resolve-default.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "vitest"; + import { resolveEvalDefault } from "../run-evals"; const def = { description: "x", test: async () => {} }; diff --git a/packages/appkit/src/evals/tests/run-eval.test.ts b/packages/appkit/src/evals/tests/run-eval.test.ts index abc780302..828daea97 100644 --- a/packages/appkit/src/evals/tests/run-eval.test.ts +++ b/packages/appkit/src/evals/tests/run-eval.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "vitest"; + import { defineEval } from "../define-eval"; import { includes } from "../matchers"; import { runEval } from "../run-eval"; diff --git a/packages/shared/src/cli/commands/agent/index.ts b/packages/shared/src/cli/commands/agent/index.ts index 3f31a92d8..9ef8b6cf5 100644 --- a/packages/shared/src/cli/commands/agent/index.ts +++ b/packages/shared/src/cli/commands/agent/index.ts @@ -1,4 +1,5 @@ import { Command } from "commander"; + import { agentEvalCommand } from "./eval"; /** From 420d141be43bf9da42e91207ba41b20e02615d93 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Thu, 20 Aug 2026 12:03:35 +0200 Subject: [PATCH 06/14] refactor(appkit): drop unwired eval config and use recursive readdir Remove dead surface flagged in review: - EvalConfig + defineEvalConfig: nothing loads evals.config.ts (the runner never reads it), so the type and helper configured nothing. - EvalDefinition.tags / .timeoutMs: never read by the runner. - discover: replace hand-rolled statSync/recursion with readdirSync({ recursive, withFileTypes }). - resolveEvalDefault: drop the seen-Set, redundant with the loop bound. Signed-off-by: MarioCadenas --- .../api/appkit/Function.defineEvalConfig.md | 17 ------- docs/docs/api/appkit/Interface.EvalConfig.md | 41 ----------------- .../api/appkit/Interface.EvalDefinition.md | 20 --------- docs/docs/api/appkit/index.md | 2 - docs/docs/api/appkit/typedoc-sidebar.ts | 15 ------- packages/appkit/src/evals/define-eval.ts | 7 +-- packages/appkit/src/evals/discover.ts | 45 ++++++------------- packages/appkit/src/evals/index.ts | 3 +- packages/appkit/src/evals/run-evals.ts | 4 +- packages/appkit/src/evals/types.ts | 14 ------ 10 files changed, 16 insertions(+), 152 deletions(-) delete mode 100644 docs/docs/api/appkit/Function.defineEvalConfig.md delete mode 100644 docs/docs/api/appkit/Interface.EvalConfig.md diff --git a/docs/docs/api/appkit/Function.defineEvalConfig.md b/docs/docs/api/appkit/Function.defineEvalConfig.md deleted file mode 100644 index 2710a598c..000000000 --- a/docs/docs/api/appkit/Function.defineEvalConfig.md +++ /dev/null @@ -1,17 +0,0 @@ -# Function: defineEvalConfig() - -```ts -function defineEvalConfig(config: EvalConfig): EvalConfig; -``` - -Define per-directory eval config. Default-export from `evals.config.ts`. - -## Parameters - -| Parameter | Type | -| ------ | ------ | -| `config` | [`EvalConfig`](Interface.EvalConfig.md) | - -## Returns - -[`EvalConfig`](Interface.EvalConfig.md) diff --git a/docs/docs/api/appkit/Interface.EvalConfig.md b/docs/docs/api/appkit/Interface.EvalConfig.md deleted file mode 100644 index eb8227fcd..000000000 --- a/docs/docs/api/appkit/Interface.EvalConfig.md +++ /dev/null @@ -1,41 +0,0 @@ -# Interface: EvalConfig - -Per-directory config from `evals.config.ts`. - -## Properties - -### judge? - -```ts -optional judge: { - model?: string; -}; -``` - -LLM judge config. Defaults to the agent's own serving endpoint. - -#### model? - -```ts -optional model: string; -``` - -*** - -### maxConcurrency? - -```ts -optional maxConcurrency: number; -``` - -Max evals to run concurrently. - -*** - -### timeoutMs? - -```ts -optional timeoutMs: number; -``` - -Default per-eval timeout. diff --git a/docs/docs/api/appkit/Interface.EvalDefinition.md b/docs/docs/api/appkit/Interface.EvalDefinition.md index e579ce345..708cbec56 100644 --- a/docs/docs/api/appkit/Interface.EvalDefinition.md +++ b/docs/docs/api/appkit/Interface.EvalDefinition.md @@ -22,26 +22,6 @@ optional description: string; Short human description, shown in reports. -*** - -### tags? - -```ts -optional tags: string[]; -``` - -Free-form tags for filtering. - -*** - -### timeoutMs? - -```ts -optional timeoutMs: number; -``` - -Per-eval timeout. - ## Methods ### test() diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index d6ec7415d..b4f67e550 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -53,7 +53,6 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `config/agents//evals/`. | | [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. | | [EndpointConfig](Interface.EndpointConfig.md) | - | -| [EvalConfig](Interface.EvalConfig.md) | Per-directory config from `evals.config.ts`. | | [EvalDefinition](Interface.EvalDefinition.md) | A single eval, default-exported from a `*.eval.ts` file. | | [EvalDriver](Interface.EvalDriver.md) | Abstraction over how the agent is driven. The HTTP driver posts to a running app's agents endpoint; future drivers (in-process) implement the same shape. | | [EvalResult](Interface.EvalResult.md) | The outcome of running one eval. | @@ -178,7 +177,6 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [createLakebasePoolManager](Function.createLakebasePoolManager.md) | Create a pool manager that maintains per-key Lakebase connection pools. | | [createWorkspaceClient](Function.createWorkspaceClient.md) | Construct an AppKit workspace client. | | [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `config/agents//evals/*.eval.ts` file. | -| [defineEvalConfig](Function.defineEvalConfig.md) | Define per-directory eval config. Default-export from `evals.config.ts`. | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | | [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/config/agents//evals/`. The agent id is the directory name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Returns a stable, sorted list. | | [equals](Function.equals.md) | Passes when the value equals `expected` exactly. | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index 04267b777..f58223c54 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -197,11 +197,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.EndpointConfig", label: "EndpointConfig" }, - { - type: "doc", - id: "api/appkit/Interface.EvalConfig", - label: "EvalConfig" - }, { type: "doc", id: "api/appkit/Interface.EvalDefinition", @@ -765,16 +760,6 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.defineEval", label: "defineEval" }, - { - type: "doc", - id: "api/appkit/Function.defineEvalConfig", - label: "defineEvalConfig" - }, - { - type: "doc", - id: "api/appkit/Function.defineManifest", - label: "defineManifest" - }, { type: "doc", id: "api/appkit/Function.defineTool", diff --git a/packages/appkit/src/evals/define-eval.ts b/packages/appkit/src/evals/define-eval.ts index 9b8ff9072..1ca9ca7ea 100644 --- a/packages/appkit/src/evals/define-eval.ts +++ b/packages/appkit/src/evals/define-eval.ts @@ -1,4 +1,4 @@ -import type { EvalConfig, EvalDefinition } from "./types"; +import type { EvalDefinition } from "./types"; /** * Define an agent eval. Default-export the result from a @@ -25,8 +25,3 @@ export function defineEval(def: EvalDefinition): EvalDefinition { } return def; } - -/** Define per-directory eval config. Default-export from `evals.config.ts`. */ -export function defineEvalConfig(config: EvalConfig): EvalConfig { - return config; -} diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts index e60ae4030..ec1e28dce 100644 --- a/packages/appkit/src/evals/discover.ts +++ b/packages/appkit/src/evals/discover.ts @@ -1,4 +1,4 @@ -import { readdirSync, statSync } from "node:fs"; +import { type Dirent, readdirSync } from "node:fs"; import path from "node:path"; /** An eval file found under `config/agents//evals/`. */ @@ -11,34 +11,17 @@ export interface DiscoveredEval { agent: string; } -function isDir(p: string): boolean { +/** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */ +function evalFilesIn(dir: string): string[] { try { - return statSync(p).isDirectory(); + return readdirSync(dir, { recursive: true, withFileTypes: true }) + .filter((e) => e.isFile() && e.name.endsWith(".eval.ts")) + .map((e) => path.join(e.parentPath, e.name)); } catch { - return false; + return []; } } -/** Recursively collect `*.eval.ts` files (skips `evals.config.ts`). */ -function walkEvalFiles(dir: string): string[] { - const out: string[] = []; - let entries: string[]; - try { - entries = readdirSync(dir); - } catch { - return out; - } - for (const entry of entries) { - const full = path.join(dir, entry); - if (isDir(full)) { - out.push(...walkEvalFiles(full)); - } else if (entry.endsWith(".eval.ts")) { - out.push(full); - } - } - return out; -} - /** * Discover evals under `/config/agents//evals/`. The agent id * is the directory name; the eval id is the file path relative to that evals @@ -48,25 +31,23 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { const agentsDir = path.join(rootDir, "config", "agents"); const out: DiscoveredEval[] = []; - let agents: string[]; + let agents: Dirent[]; try { - agents = readdirSync(agentsDir).filter((n) => - isDir(path.join(agentsDir, n)), - ); + agents = readdirSync(agentsDir, { withFileTypes: true }); } catch { return out; } for (const agent of agents) { - const evalsDir = path.join(agentsDir, agent, "evals"); - if (!isDir(evalsDir)) continue; - for (const file of walkEvalFiles(evalsDir)) { + if (!agent.isDirectory()) continue; + const evalsDir = path.join(agentsDir, agent.name, "evals"); + for (const file of evalFilesIn(evalsDir)) { const id = path .relative(evalsDir, file) .replace(/\.eval\.ts$/, "") .split(path.sep) .join("/"); - out.push({ file, id, agent }); + out.push({ file, id, agent: agent.name }); } } diff --git a/packages/appkit/src/evals/index.ts b/packages/appkit/src/evals/index.ts index 506d05cb5..92b1df702 100644 --- a/packages/appkit/src/evals/index.ts +++ b/packages/appkit/src/evals/index.ts @@ -6,7 +6,7 @@ export { type ResolveDatabricksAuthOptions, resolveDatabricksAuth, } from "../connectors/mlflow"; -export { defineEval, defineEvalConfig } from "./define-eval"; +export { defineEval } from "./define-eval"; export { type DiscoveredEval, discoverEvalFiles } from "./discover"; export { createHttpDriver, type HttpDriverOptions } from "./http-driver"; export { @@ -43,7 +43,6 @@ export type { AssertionResult, CustomJudgeSpec, DriveResult, - EvalConfig, EvalDefinition, EvalDriver, EvalResult, diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 92c911aa3..5a0622915 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -83,13 +83,11 @@ async function loadEval(file: string): Promise { * `mod` itself. Returns the first candidate that looks like an eval. */ export function resolveEvalDefault(mod: unknown): EvalDefinition | undefined { - const seen = new Set(); let candidate: unknown = mod; - for (let i = 0; i < 4 && candidate && !seen.has(candidate); i++) { + for (let i = 0; i < 4 && candidate; i++) { if (typeof (candidate as EvalDefinition).test === "function") { return candidate as EvalDefinition; } - seen.add(candidate); candidate = (candidate as { default?: unknown }).default; } return undefined; diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts index 16a7d3bed..9cd7cb819 100644 --- a/packages/appkit/src/evals/types.ts +++ b/packages/appkit/src/evals/types.ts @@ -115,24 +115,10 @@ export interface EvalDefinition { description?: string; /** Target agent id. Defaults to the eval's parent `config/agents/` dir. */ agent?: string; - /** Free-form tags for filtering. */ - tags?: string[]; - /** Per-eval timeout. */ - timeoutMs?: number; /** The eval body: drive the agent and assert on its behavior. */ test(t: TestContext): Promise | void; } -/** Per-directory config from `evals.config.ts`. */ -export interface EvalConfig { - /** LLM judge config. Defaults to the agent's own serving endpoint. */ - judge?: { model?: string }; - /** Max evals to run concurrently. */ - maxConcurrency?: number; - /** Default per-eval timeout. */ - timeoutMs?: number; -} - /** The outcome of running one eval. */ export interface EvalResult { id: string; From dce77ad850bc08d7398d08ccd456ca6a025a9593 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Thu, 20 Aug 2026 12:44:05 +0200 Subject: [PATCH 07/14] refactor(appkit): discover evals from server/agents to match folder-per-agent layout Adapts the eval framework to #533's unified agent discovery: agents now live in server/agents//agent.{md,ts} (config/agents is a deprecated fallback), so evals move next to them. - discover.ts scans server/agents//evals/, reusing CODE_AGENTS_SOURCE_DIR and agentDirNames so eval discovery follows the same folder-selection policy (symlinked agent folders included) as the agents plugin. - Relocate the dev-playground example evals config/agents/query/evals -> server/agents/query/evals. - Update discover.test fixtures + doc strings. Signed-off-by: MarioCadenas --- .../agents/query/evals/judge.eval.ts | 0 .../agents/query/evals/smoke.eval.ts | 0 .../agents/query/evals/sum.eval.ts | 0 .../agents/query/evals/tool-call.eval.ts | 0 docs/docs/api/appkit/Function.defineEval.md | 2 +- .../api/appkit/Function.discoverEvalFiles.md | 7 ++--- .../api/appkit/Interface.DiscoveredEval.md | 4 +-- .../api/appkit/Interface.EvalDefinition.md | 2 +- .../api/appkit/Interface.RunEvalsOptions.md | 2 +- docs/docs/api/appkit/index.md | 10 +++---- packages/appkit/src/evals/define-eval.ts | 2 +- packages/appkit/src/evals/discover.ts | 27 ++++++++++--------- packages/appkit/src/evals/run-evals.ts | 2 +- .../appkit/src/evals/tests/discover.test.ts | 14 +++++----- packages/appkit/src/evals/types.ts | 4 +-- .../shared/src/cli/commands/agent/eval.ts | 4 +-- 16 files changed, 42 insertions(+), 38 deletions(-) rename apps/dev-playground/{config => server}/agents/query/evals/judge.eval.ts (100%) rename apps/dev-playground/{config => server}/agents/query/evals/smoke.eval.ts (100%) rename apps/dev-playground/{config => server}/agents/query/evals/sum.eval.ts (100%) rename apps/dev-playground/{config => server}/agents/query/evals/tool-call.eval.ts (100%) diff --git a/apps/dev-playground/config/agents/query/evals/judge.eval.ts b/apps/dev-playground/server/agents/query/evals/judge.eval.ts similarity index 100% rename from apps/dev-playground/config/agents/query/evals/judge.eval.ts rename to apps/dev-playground/server/agents/query/evals/judge.eval.ts diff --git a/apps/dev-playground/config/agents/query/evals/smoke.eval.ts b/apps/dev-playground/server/agents/query/evals/smoke.eval.ts similarity index 100% rename from apps/dev-playground/config/agents/query/evals/smoke.eval.ts rename to apps/dev-playground/server/agents/query/evals/smoke.eval.ts diff --git a/apps/dev-playground/config/agents/query/evals/sum.eval.ts b/apps/dev-playground/server/agents/query/evals/sum.eval.ts similarity index 100% rename from apps/dev-playground/config/agents/query/evals/sum.eval.ts rename to apps/dev-playground/server/agents/query/evals/sum.eval.ts diff --git a/apps/dev-playground/config/agents/query/evals/tool-call.eval.ts b/apps/dev-playground/server/agents/query/evals/tool-call.eval.ts similarity index 100% rename from apps/dev-playground/config/agents/query/evals/tool-call.eval.ts rename to apps/dev-playground/server/agents/query/evals/tool-call.eval.ts diff --git a/docs/docs/api/appkit/Function.defineEval.md b/docs/docs/api/appkit/Function.defineEval.md index a47e86629..d69656eeb 100644 --- a/docs/docs/api/appkit/Function.defineEval.md +++ b/docs/docs/api/appkit/Function.defineEval.md @@ -5,7 +5,7 @@ function defineEval(def: EvalDefinition): EvalDefinition; ``` Define an agent eval. Default-export the result from a -`config/agents//evals/*.eval.ts` file. +`server/agents//evals/*.eval.ts` file. ## Parameters diff --git a/docs/docs/api/appkit/Function.discoverEvalFiles.md b/docs/docs/api/appkit/Function.discoverEvalFiles.md index 746b45e5f..57347de8d 100644 --- a/docs/docs/api/appkit/Function.discoverEvalFiles.md +++ b/docs/docs/api/appkit/Function.discoverEvalFiles.md @@ -4,9 +4,10 @@ function discoverEvalFiles(rootDir: string): DiscoveredEval[]; ``` -Discover evals under `/config/agents//evals/`. The agent id -is the directory name; the eval id is the file path relative to that evals -dir with `.eval.ts` stripped. Returns a stable, sorted list. +Discover evals under `/server/agents//evals/` — co-located +with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents +plugin discovers). The agent id is the folder name; the eval id is the file +path relative to that evals dir with `.eval.ts` stripped. Sorted + stable. ## Parameters diff --git a/docs/docs/api/appkit/Interface.DiscoveredEval.md b/docs/docs/api/appkit/Interface.DiscoveredEval.md index 376aa4c4a..c64ec7796 100644 --- a/docs/docs/api/appkit/Interface.DiscoveredEval.md +++ b/docs/docs/api/appkit/Interface.DiscoveredEval.md @@ -1,6 +1,6 @@ # Interface: DiscoveredEval -An eval file found under `config/agents//evals/`. +An eval file found under `server/agents//evals/`. ## Properties @@ -10,7 +10,7 @@ An eval file found under `config/agents//evals/`. agent: string; ``` -The agent id (the `config/agents/` directory name). +The agent id (the `server/agents/` directory name). *** diff --git a/docs/docs/api/appkit/Interface.EvalDefinition.md b/docs/docs/api/appkit/Interface.EvalDefinition.md index 708cbec56..3bf035507 100644 --- a/docs/docs/api/appkit/Interface.EvalDefinition.md +++ b/docs/docs/api/appkit/Interface.EvalDefinition.md @@ -10,7 +10,7 @@ A single eval, default-exported from a `*.eval.ts` file. optional agent: string; ``` -Target agent id. Defaults to the eval's parent `config/agents/` dir. +Target agent id. Defaults to the eval's parent `server/agents/` dir. *** diff --git a/docs/docs/api/appkit/Interface.RunEvalsOptions.md b/docs/docs/api/appkit/Interface.RunEvalsOptions.md index 949d521e2..174d0e772 100644 --- a/docs/docs/api/appkit/Interface.RunEvalsOptions.md +++ b/docs/docs/api/appkit/Interface.RunEvalsOptions.md @@ -135,7 +135,7 @@ Progress callback, invoked as evals are discovered, started, and finished. optional rootDir: string; ``` -Project root containing `config/agents/`. Defaults to `process.cwd()`. +Project root containing `server/agents/`. Defaults to `process.cwd()`. *** diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index b4f67e550..58ed44b05 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -50,7 +50,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [CustomJudgeSpec](Interface.CustomJudgeSpec.md) | A custom LLM-judge definition: a prompt template and choice→score mapping. | | [DatabaseCredential](Interface.DatabaseCredential.md) | Database credentials with OAuth token for Postgres connection | | [DatabricksAuth](Interface.DatabricksAuth.md) | Resolved Databricks host + bearer token for the eval runner's REST calls. | -| [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `config/agents//evals/`. | +| [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `server/agents//evals/`. | | [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. | | [EndpointConfig](Interface.EndpointConfig.md) | - | | [EvalDefinition](Interface.EvalDefinition.md) | A single eval, default-exported from a `*.eval.ts` file. | @@ -153,7 +153,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | Variable | Description | | ------ | ------ | -| [agents](Variable.agents.md) | Plugin factory for the agents plugin. Reads `config/agents/*.md` by default, resolves toolkits/tools from registered plugins, exposes `appkit.agents.*` runtime API and mounts `POST /invocations` and `POST /responses` (aliased non-streaming invoke endpoints) plus `POST /chat` (streaming, HITL-capable). | +| [agents](Variable.agents.md) | Plugin factory for the agents plugin. Discovers agents from `server/agents//agent.{ts,md}` by default (markdown still in `config/agents/` is read as a deprecated fallback), resolves toolkits/tools from registered plugins, exposes the `appkit.agents.*` runtime API and mounts `POST /invocations` and `POST /responses` (aliased non-streaming invoke endpoints) plus `POST /chat` (streaming, HITL-capable). | | [aiSearch](Variable.aiSearch.md) | - | | [READ\_ACTIONS](Variable.READ_ACTIONS.md) | Actions that only read data. | | [sql](Variable.sql.md) | SQL helper namespace | @@ -170,15 +170,15 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [appKitTypesPlugin](Function.appKitTypesPlugin.md) | Vite plugin to generate types for AppKit queries. Calls generateFromEntryPoint under the hood. | | [buildAssessments](Function.buildAssessments.md) | Build the Feedback assessments for an eval result: one per assertion (judge assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. Returns [] when there's no trace to attach to or the eval was skipped. | | [configureJudge](Function.configureJudge.md) | Configure the judge once. Sets the OpenAI-compatible client env autoevals reads and the default judge model. No-op-safe: on failure, judging stays disabled and [isJudgeConfigured](Function.isJudgeConfigured.md) returns false. | -| [createAgent](Function.createAgent.md) | Pure factory for agent definitions. Returns the passed-in definition after cycle-detecting the sub-agent graph. Accepts the full `AgentDefinition` shape and is safe to call at module top-level. | +| [createAgent](Function.createAgent.md) | Pure factory for agent definitions: cycle-detects the sub-agent graph and returns the same object, stamped with a non-enumerable AGENT\_BRAND so discovery recognizes it. Safe at module top-level; no adapter is built. Don't `Object.freeze` the definition before passing it in — the brand is written onto the argument. | | [createApp](Function.createApp.md) | Bootstraps AppKit with the provided configuration. | | [createHttpDriver](Function.createHttpDriver.md) | Drives an agent by POSTing to a running app's chat endpoint and parsing the SSE response. Keeps the thread id across `send`s so multi-turn evals share a conversation. Agent/stream errors surface as `succeeded: false` rather than throwing, so `t.succeeded()` can assert on them. | | [createLakebasePool](Function.createLakebasePool.md) | Create a Lakebase pool with appkit's logger integration. Telemetry automatically uses appkit's OpenTelemetry configuration via global registry. | | [createLakebasePoolManager](Function.createLakebasePoolManager.md) | Create a pool manager that maintains per-key Lakebase connection pools. | | [createWorkspaceClient](Function.createWorkspaceClient.md) | Construct an AppKit workspace client. | -| [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `config/agents//evals/*.eval.ts` file. | +| [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `server/agents//evals/*.eval.ts` file. | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | -| [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/config/agents//evals/`. The agent id is the directory name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Returns a stable, sorted list. | +| [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/server/agents//evals/` — co-located with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents plugin discovers). The agent id is the folder name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Sorted + stable. | | [equals](Function.equals.md) | Passes when the value equals `expected` exactly. | | [evalGlyph](Function.evalGlyph.md) | Status glyph for a single eval result. | | [executeFromRegistry](Function.executeFromRegistry.md) | Validates tool-call arguments against the entry's schema and invokes its handler. On validation failure, returns an LLM-friendly error string (matching the behavior of `tool()`) rather than throwing, so the model can self-correct on its next turn. | diff --git a/packages/appkit/src/evals/define-eval.ts b/packages/appkit/src/evals/define-eval.ts index 1ca9ca7ea..3e31cb191 100644 --- a/packages/appkit/src/evals/define-eval.ts +++ b/packages/appkit/src/evals/define-eval.ts @@ -2,7 +2,7 @@ import type { EvalDefinition } from "./types"; /** * Define an agent eval. Default-export the result from a - * `config/agents//evals/*.eval.ts` file. + * `server/agents//evals/*.eval.ts` file. * * @example * ```ts diff --git a/packages/appkit/src/evals/discover.ts b/packages/appkit/src/evals/discover.ts index ec1e28dce..47029c3ad 100644 --- a/packages/appkit/src/evals/discover.ts +++ b/packages/appkit/src/evals/discover.ts @@ -1,13 +1,16 @@ import { type Dirent, readdirSync } from "node:fs"; import path from "node:path"; -/** An eval file found under `config/agents//evals/`. */ +import { agentDirNames } from "../core/agent/agent-dirs"; +import { CODE_AGENTS_SOURCE_DIR } from "../core/agent/load-code-agents"; + +/** An eval file found under `server/agents//evals/`. */ export interface DiscoveredEval { /** Absolute path to the `*.eval.ts` file. */ file: string; /** Id relative to the agent's evals dir, without `.eval.ts` (e.g. `weather/basic`). */ id: string; - /** The agent id (the `config/agents/` directory name). */ + /** The agent id (the `server/agents/` directory name). */ agent: string; } @@ -23,31 +26,31 @@ function evalFilesIn(dir: string): string[] { } /** - * Discover evals under `/config/agents//evals/`. The agent id - * is the directory name; the eval id is the file path relative to that evals - * dir with `.eval.ts` stripped. Returns a stable, sorted list. + * Discover evals under `/server/agents//evals/` — co-located + * with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents + * plugin discovers). The agent id is the folder name; the eval id is the file + * path relative to that evals dir with `.eval.ts` stripped. Sorted + stable. */ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] { - const agentsDir = path.join(rootDir, "config", "agents"); + const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR); const out: DiscoveredEval[] = []; - let agents: Dirent[]; + let entries: Dirent[]; try { - agents = readdirSync(agentsDir, { withFileTypes: true }); + entries = readdirSync(agentsDir, { withFileTypes: true }); } catch { return out; } - for (const agent of agents) { - if (!agent.isDirectory()) continue; - const evalsDir = path.join(agentsDir, agent.name, "evals"); + for (const agent of agentDirNames(entries)) { + const evalsDir = path.join(agentsDir, agent, "evals"); for (const file of evalFilesIn(evalsDir)) { const id = path .relative(evalsDir, file) .replace(/\.eval\.ts$/, "") .split(path.sep) .join("/"); - out.push({ file, id, agent: agent.name }); + out.push({ file, id, agent }); } } diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 5a0622915..3a53f9404 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -10,7 +10,7 @@ import { runEval } from "./run-eval"; import type { EvalDefinition, EvalResult } from "./types"; export interface RunEvalsOptions { - /** Project root containing `config/agents/`. Defaults to `process.cwd()`. */ + /** Project root containing `server/agents/`. Defaults to `process.cwd()`. */ rootDir?: string; /** Base URL of the running app to drive, e.g. `http://localhost:3000`. */ baseUrl: string; diff --git a/packages/appkit/src/evals/tests/discover.test.ts b/packages/appkit/src/evals/tests/discover.test.ts index 29ecf0ae6..e93124d70 100644 --- a/packages/appkit/src/evals/tests/discover.test.ts +++ b/packages/appkit/src/evals/tests/discover.test.ts @@ -22,12 +22,12 @@ afterEach(() => { }); describe("discoverEvalFiles", () => { - test("finds *.eval.ts per agent, derives id + agent, ignores config", () => { - write("config/agents/support/evals/basic.eval.ts"); - write("config/agents/support/evals/nested/deep.eval.ts"); - write("config/agents/support/evals/evals.config.ts"); - write("config/agents/analyst/evals/sql.eval.ts"); - write("config/agents/no-evals/agent.md", "# agent"); + test("finds *.eval.ts per agent, derives id + agent, ignores non-evals", () => { + write("server/agents/support/evals/basic.eval.ts"); + write("server/agents/support/evals/nested/deep.eval.ts"); + write("server/agents/support/evals/evals.config.ts"); + write("server/agents/analyst/evals/sql.eval.ts"); + write("server/agents/no-evals/agent.md", "# agent"); const found = discoverEvalFiles(root); @@ -38,7 +38,7 @@ describe("discoverEvalFiles", () => { ]); }); - test("returns empty when there is no config/agents dir", () => { + test("returns empty when there is no server/agents dir", () => { expect(discoverEvalFiles(root)).toEqual([]); }); }); diff --git a/packages/appkit/src/evals/types.ts b/packages/appkit/src/evals/types.ts index 9cd7cb819..b4e7df9b0 100644 --- a/packages/appkit/src/evals/types.ts +++ b/packages/appkit/src/evals/types.ts @@ -2,7 +2,7 @@ * Agent evaluation primitives — an eve-style authoring API that runs against * AppKit agents and reports to Databricks MLflow. * - * Evals live in `config/agents//evals/*.eval.ts`, each default-exporting a + * Evals live in `server/agents//evals/*.eval.ts`, each default-exporting a * {@link EvalDefinition} via {@link defineEval}. A runner drives the agent * (today over HTTP against a running app), and the `test` function asserts on * the reply and tool usage with deterministic matchers (and, later, LLM judges). @@ -113,7 +113,7 @@ export interface CustomJudgeSpec { export interface EvalDefinition { /** Short human description, shown in reports. */ description?: string; - /** Target agent id. Defaults to the eval's parent `config/agents/` dir. */ + /** Target agent id. Defaults to the eval's parent `server/agents/` dir. */ agent?: string; /** The eval body: drive the agent and assert on its behavior. */ test(t: TestContext): Promise | void; diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index 41390890a..003e43217 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -184,7 +184,7 @@ async function runAgentEval( export const agentEvalCommand = new Command("eval") .description( - "Run agent evals (config/agents//evals/*.eval.ts) against a running app", + "Run agent evals (server/agents//evals/*.eval.ts) against a running app", ) .argument( "[filter]", @@ -194,7 +194,7 @@ export const agentEvalCommand = new Command("eval") .option("--strict", "Fail on soft-assertion misses too", false) .option( "--root ", - "Project root containing config/agents/ (default: cwd)", + "Project root containing server/agents/ (default: cwd)", ) .option( "--header ", From a1d34d8d3195cc9f5e238240e183f1cfba6b31ae Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Fri, 21 Aug 2026 12:35:28 +0200 Subject: [PATCH 08/14] docs(appkit): reconcile API reference after rebase onto main Regenerated index + sidebar pick up main's defineManifest (#485) alongside the eval-framework entries. Signed-off-by: MarioCadenas --- docs/docs/api/appkit/index.md | 1 + docs/docs/api/appkit/typedoc-sidebar.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index 58ed44b05..08ffdace6 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -177,6 +177,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [createLakebasePoolManager](Function.createLakebasePoolManager.md) | Create a pool manager that maintains per-key Lakebase connection pools. | | [createWorkspaceClient](Function.createWorkspaceClient.md) | Construct an AppKit workspace client. | | [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `server/agents//evals/*.eval.ts` file. | +| [defineManifest](Function.defineManifest.md) | Validates a raw manifest (typically a `manifest.json` import) against the canonical Zod schema and returns it as a strict [PluginManifest](Interface.PluginManifest.md). | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | | [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/server/agents//evals/` — co-located with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents plugin discovers). The agent id is the folder name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Sorted + stable. | | [equals](Function.equals.md) | Passes when the value equals `expected` exactly. | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index f58223c54..c26271827 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -760,6 +760,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.defineEval", label: "defineEval" }, + { + type: "doc", + id: "api/appkit/Function.defineManifest", + label: "defineManifest" + }, { type: "doc", id: "api/appkit/Function.defineTool", From 274ec9d312fb7ce2e8fd96d2fd149b9cb5556c7c Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Mon, 31 Aug 2026 15:37:23 +0200 Subject: [PATCH 09/14] fix(appkit): stop eval http driver from wedging on stalls and false-passing on errors The eval HTTP driver had two failure modes: - A hung agent (blocked tool / stalled model) never ends the SSE stream; heartbeats keep reader.read() producing bytes, so the read loop spins forever and the sequential suite wedges. Add an AbortSignal.timeout (default 120s, configurable via RunEvalsOptions.timeoutMs) and fail the turn on abort. - A thrown exception is framed by SSEWriter.writeError as `event: error` with a payload that has no `type` field, so applyEvent never sees it and the turn reports succeeded:true (false PASS). Track the SSE `event:` line and treat `error` as a failed turn. Add http-driver.test.ts covering both plus a happy-path guard. Signed-off-by: MarioCadenas --- packages/appkit/src/evals/http-driver.ts | 33 +++++++ packages/appkit/src/evals/run-evals.ts | 3 + .../src/evals/tests/http-driver.test.ts | 87 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 packages/appkit/src/evals/tests/http-driver.test.ts diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts index 0803a6e37..af366429b 100644 --- a/packages/appkit/src/evals/http-driver.ts +++ b/packages/appkit/src/evals/http-driver.ts @@ -11,6 +11,15 @@ export interface HttpDriverOptions { path?: string; /** MLflow run id to link each turn's trace to (for evaluation runs). */ mlflowRunId?: string; + /** + * Max wall-clock time for a single turn before it is abandoned as a failed + * turn (`succeeded: false`). Without this a hung agent — a blocked tool, a + * stalled model — never ends the SSE stream (heartbeats keep it alive), so + * the read loop spins forever and wedges the whole sequential suite. + * Defaults to 120s. + */ + // ponytail: total per-turn cap; switch to an idle timeout if long legit turns get killed. + timeoutMs?: number; } /** Parse a single Responses-API SSE `data:` payload into the running totals. */ @@ -70,6 +79,7 @@ function applyEvent( */ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { const chatPath = options.path ?? "/api/agents/chat"; + const timeoutMs = options.timeoutMs ?? 120_000; let threadId: string | undefined; return { @@ -87,6 +97,9 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { ? { mlflowRunId: options.mlflowRunId } : {}), }), + // bounds connect + the entire read below; on expiry the pending + // reader.read() rejects and we mark the turn failed. + signal: AbortSignal.timeout(timeoutMs), }); } catch { return { reply: "", toolCalls: [], succeeded: false }; @@ -111,6 +124,7 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { const reader = res.body.getReader(); const decoder = new TextDecoder(); let buffer = ""; + let eventName = ""; try { while (true) { @@ -120,9 +134,24 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { const lines = buffer.split("\n"); buffer = lines.pop() ?? ""; for (const line of lines) { + // A blank line terminates an SSE event block — reset the pending + // type so it doesn't bleed into the next event. + if (line === "") { + eventName = ""; + continue; + } + // A stream-level error (a thrown exception in the generator) is + // written as `event: error` with a payload that carries no `type` + // field, so applyEvent can't see it. The SSE event name is the + // real contract, so track it and treat `error` as a failed turn. + if (line.startsWith("event:")) { + eventName = line.slice(6).trim(); + continue; + } if (!line.startsWith("data: ")) continue; const data = line.slice(6).trim(); if (!data || data === "[DONE]") continue; + if (eventName === "error") state.ok = false; try { applyEvent(JSON.parse(data), state, (id) => { threadId = id; @@ -132,6 +161,10 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { } } } + } catch { + // timeout abort or a mid-stream transport error: a hung/broken turn, + // not a passing one. + state.ok = false; } finally { reader.releaseLock(); } diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 3a53f9404..7ee95b0ef 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -20,6 +20,8 @@ export interface RunEvalsOptions { strict?: boolean; /** Extra request headers for the driver (e.g. auth for a deployed app). */ headers?: Record; + /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */ + timeoutMs?: number; /** * When set, create a native MLflow "Evaluation run": each eval's trace is * linked to the run, pass/fail is written as feedback, and aggregate metrics @@ -153,6 +155,7 @@ export async function runEvalsInDir( agent: def.agent ?? d.agent, headers: options.headers, mlflowRunId: runId, + timeoutMs: options.timeoutMs, }); result = await runEval(def, { id, driver, strict: options.strict }); } catch (err) { diff --git a/packages/appkit/src/evals/tests/http-driver.test.ts b/packages/appkit/src/evals/tests/http-driver.test.ts new file mode 100644 index 000000000..011716fb4 --- /dev/null +++ b/packages/appkit/src/evals/tests/http-driver.test.ts @@ -0,0 +1,87 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { createHttpDriver } from "../http-driver"; + +/** + * A real SSE server, one behavior per path, so the driver's actual read loop + * (heartbeat skipping, event-line parsing, timeout abort) is exercised end to + * end rather than mocked. + */ +const server: Server = createServer((req, res) => { + res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8" }); + + switch (req.url) { + // Happy path: a text delta then a normal end. + case "/ok": + res.write( + `id: 1\nevent: response.output_text.delta\ndata: ${JSON.stringify({ + type: "response.output_text.delta", + delta: "Hello", + })}\n\n`, + ); + res.write(`data: [DONE]\n\n`); + res.end(); + return; + + // A thrown exception in the generator is framed by SSEWriter.writeError: + // `event: error` with a payload that has NO `type` field. + case "/thrown-error": + res.write( + `id: 1\nevent: error\ndata: ${JSON.stringify({ + error: "Internal server error", + code: "INTERNAL_ERROR", + })}\n\n`, + ); + res.end(); + return; + + // A hung agent: heartbeat comments keep the socket alive but the stream + // never ends. Deliberately never call res.end(). + default: + res.write(`: heartbeat\n\n`); + return; + } +}); + +let baseUrl = ""; + +beforeAll(async () => { + await new Promise((resolve) => server.listen(0, resolve)); + const { port } = server.address() as AddressInfo; + baseUrl = `http://127.0.0.1:${port}`; +}); + +afterAll(async () => { + server.closeAllConnections?.(); + await new Promise((resolve) => server.close(() => resolve())); +}); + +describe("createHttpDriver", () => { + test("captures the reply and succeeds on a normal stream", async () => { + const driver = createHttpDriver({ baseUrl, path: "/ok" }); + const result = await driver.send("hi"); + expect(result.succeeded).toBe(true); + expect(result.reply).toBe("Hello"); + }); + + test("reports succeeded:false on a thrown-error frame (no false PASS)", async () => { + const driver = createHttpDriver({ baseUrl, path: "/thrown-error" }); + const result = await driver.send("hi"); + expect(result.succeeded).toBe(false); + }); + + test("times out a hung stream instead of hanging forever", async () => { + const driver = createHttpDriver({ + baseUrl, + path: "/stall", + timeoutMs: 150, + }); + const started = Date.now(); + const result = await driver.send("hi"); + expect(result.succeeded).toBe(false); + expect(Date.now() - started).toBeLessThan(2000); + }); +}); From dffd8413ffc44c83ac9f86ec10513546267f01e5 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Tue, 1 Sep 2026 15:59:35 +0200 Subject: [PATCH 10/14] chore(appkit): repair pnpm-lock after rebase (dedup autoevals js-yaml to 4.3.1) The rebase's git auto-merge left autoevals's transitive js-yaml pinned at 4.2.0 while the top-level entries unified to 4.3.1, leaving 4.2.0 with no package entry. pnpm install --frozen-lockfile (CI) fails this as ERR_PNPM_LOCKFILE_MISSING_DEPENDENCY, blocking every job at install. autoevals declares js-yaml ^4.1.0, which 4.3.1 satisfies. Signed-off-by: MarioCadenas --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4257d145b..b809372ac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18716,7 +18716,7 @@ snapshots: ajv: 8.18.0 compute-cosine-similarity: 1.1.0 js-levenshtein: 1.1.6 - js-yaml: 4.2.0 + js-yaml: 4.3.1 linear-sum-assignment: 1.0.9 mustache: 4.2.0 openai: 6.44.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.3.6) From 8e4f9e435b6ac54dc276f45fe81392d6fdc68f02 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Tue, 1 Sep 2026 17:03:26 +0200 Subject: [PATCH 11/14] refactor(appkit): cut cyclomatic complexity in eval framework Behavior-preserving extractions bringing six functions from 10-25 down to <=8 (oxlint complexity, max 8): - http-driver: split send into buildRequestBody + drainSse + handleSseLine; applyEvent delegates to recordToolCall + applyMetadata. The SSE pipeline is now unit-testable end to end instead of only via a live server. - run-evals: extract runOne, maybeConfigureJudge, finalizeMlflow. - eval CLI: extract resolveMlflow, resolveJudge, makeProgressReporter, formatFailureLine, printMlflowOutcome. - mlflow-report: extract assertionAssessment + overallAssessment. - mlflow/auth: extract extractBearer + resolveViaSdk. No behavior change; 35 eval/mlflow tests pass, typecheck + lint clean. Signed-off-by: MarioCadenas --- packages/appkit/src/connectors/mlflow/auth.ts | 33 ++- packages/appkit/src/evals/http-driver.ts | 203 ++++++++++-------- packages/appkit/src/evals/mlflow-report.ts | 67 +++--- packages/appkit/src/evals/run-evals.ts | 107 +++++---- .../shared/src/cli/commands/agent/eval.ts | 132 +++++++----- 5 files changed, 329 insertions(+), 213 deletions(-) diff --git a/packages/appkit/src/connectors/mlflow/auth.ts b/packages/appkit/src/connectors/mlflow/auth.ts index 6b01b43ba..4726c5235 100644 --- a/packages/appkit/src/connectors/mlflow/auth.ts +++ b/packages/appkit/src/connectors/mlflow/auth.ts @@ -25,14 +25,20 @@ export interface ResolveDatabricksAuthOptions { * Returns `undefined` when neither an explicit token nor a resolvable profile * yields a bearer, so the caller can treat auth as simply unavailable. */ -export async function resolveDatabricksAuth( - options: ResolveDatabricksAuthOptions = {}, -): Promise { - // Fully explicit — no need to touch the SDK. - if (options.host && options.token) { - return { host: options.host, token: options.token }; - } +/** Pull the bearer out of an `Authorization: Bearer ` header. */ +function extractBearer(headers: Headers): string | undefined { + return headers.get("authorization")?.replace(/^Bearer\s+/i, ""); +} +/** + * Resolve `{host, token}` via the SDK: construct a `WorkspaceClient`, let it + * mint/refresh an OAuth bearer from the profile (or reuse a PAT), and fall back + * to any explicit host/token the caller supplied. Returns `undefined` when + * either is missing or the SDK can't resolve credentials. + */ +async function resolveViaSdk( + options: ResolveDatabricksAuthOptions, +): Promise { try { const client = createWorkspaceClient( options.profile ? { profile: options.profile } : {}, @@ -42,8 +48,7 @@ export async function resolveDatabricksAuth( // an `Authorization: Bearer ` header — the same call the connectors // use before each request. await client.config.authenticate(headers); - const token = - options.token ?? headers.get("authorization")?.replace(/^Bearer\s+/i, ""); + const token = options.token ?? extractBearer(headers); const host = options.host ?? (await client.config.getHost()).toString().replace(/\/+$/, ""); @@ -53,3 +58,13 @@ export async function resolveDatabricksAuth( return undefined; } } + +export async function resolveDatabricksAuth( + options: ResolveDatabricksAuthOptions = {}, +): Promise { + // Fully explicit — no need to touch the SDK. + if (options.host && options.token) { + return { host: options.host, token: options.token }; + } + return resolveViaSdk(options); +} diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts index af366429b..14c571e9d 100644 --- a/packages/appkit/src/evals/http-driver.ts +++ b/packages/appkit/src/evals/http-driver.ts @@ -22,40 +22,56 @@ export interface HttpDriverOptions { timeoutMs?: number; } +/** Mutable running totals accumulated while draining one turn's SSE stream. */ +interface DriveState { + reply: string; + toolCalls: string[]; + seen: Set; + ok: boolean; + traceId?: string; +} + +/** Record a `function_call` output item once per call id (deduped). */ +function recordToolCall( + item: { type?: string; name?: string; call_id?: string } | undefined, + state: DriveState, +): void { + if (item?.type !== "function_call" || !item.name) return; + const key = item.call_id ?? item.name; + if (state.seen.has(key)) return; + state.seen.add(key); + state.toolCalls.push(item.name); +} + +/** Apply an `appkit.metadata` event's thread/trace ids. */ +function applyMetadata( + data: { threadId?: string; mlflowTraceId?: string } | undefined, + state: DriveState, + setThread: (id: string) => void, +): void { + if (data?.threadId) setThread(data.threadId); + if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId; +} + /** Parse a single Responses-API SSE `data:` payload into the running totals. */ function applyEvent( event: Record, - state: { - reply: string; - toolCalls: string[]; - seen: Set; - ok: boolean; - traceId?: string; - }, + state: DriveState, setThread: (id: string) => void, ): void { const type = event.type; - if ( - type === "response.output_text.delta" && - typeof event.delta === "string" - ) { - state.reply += event.delta; + if (type === "response.output_text.delta") { + if (typeof event.delta === "string") state.reply += event.delta; return; } if ( type === "response.output_item.added" || type === "response.output_item.done" ) { - const item = event.item as - | { type?: string; name?: string; call_id?: string } - | undefined; - if (item?.type === "function_call" && item.name) { - const key = item.call_id ?? item.name; - if (!state.seen.has(key)) { - state.seen.add(key); - state.toolCalls.push(item.name); - } - } + recordToolCall( + event.item as { type?: string; name?: string; call_id?: string }, + state, + ); return; } if (type === "error" || type === "response.failed") { @@ -63,14 +79,83 @@ function applyEvent( return; } if (type === "appkit.metadata") { - const data = event.data as - | { threadId?: string; mlflowTraceId?: string } - | undefined; - if (data?.threadId) setThread(data.threadId); - if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId; + applyMetadata( + event.data as { threadId?: string; mlflowTraceId?: string }, + state, + setThread, + ); + } +} + +/** + * Apply one raw SSE line to `state`, returning the (possibly updated) current + * event name. The event name is tracked because a stream-level error (a thrown + * exception in the generator) is framed as `event: error` with a payload that + * carries no `type` field — applyEvent can't see it, so the SSE event name is + * the real contract, and `error` marks the turn failed. + */ +function handleSseLine( + line: string, + eventName: string, + state: DriveState, + setThread: (id: string) => void, +): string { + if (line === "") return ""; // blank line terminates an SSE event block + if (line.startsWith("event:")) return line.slice(6).trim(); + if (!line.startsWith("data: ")) return eventName; + const data = line.slice(6).trim(); + if (!data || data === "[DONE]") return eventName; + if (eventName === "error") state.ok = false; + try { + applyEvent(JSON.parse(data), state, setThread); + } catch { + // skip malformed event lines + } + return eventName; +} + +/** Read the SSE stream to completion, applying each line into `state`. */ +async function drainSse( + reader: ReadableStreamDefaultReader, + state: DriveState, + setThread: (id: string) => void, +): Promise { + const decoder = new TextDecoder(); + let buffer = ""; + let eventName = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + for (const line of lines) { + eventName = handleSseLine(line, eventName, state, setThread); + } + } + } catch { + // timeout abort or a mid-stream transport error: a hung/broken turn. + state.ok = false; + } finally { + reader.releaseLock(); } } +/** Build the chat request payload, including only the optional fields that are set. */ +function buildRequestBody( + message: string, + options: HttpDriverOptions, + threadId: string | undefined, +): string { + return JSON.stringify({ + message, + ...(options.agent ? { agent: options.agent } : {}), + ...(threadId ? { threadId } : {}), + ...(options.mlflowRunId ? { mlflowRunId: options.mlflowRunId } : {}), + }); +} + /** * Drives an agent by POSTing to a running app's chat endpoint and parsing the * SSE response. Keeps the thread id across `send`s so multi-turn evals share a @@ -89,14 +174,7 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { res = await fetch(`${options.baseUrl}${chatPath}`, { method: "POST", headers: { "content-type": "application/json", ...options.headers }, - body: JSON.stringify({ - message, - ...(options.agent ? { agent: options.agent } : {}), - ...(threadId ? { threadId } : {}), - ...(options.mlflowRunId - ? { mlflowRunId: options.mlflowRunId } - : {}), - }), + body: buildRequestBody(message, options, threadId), // bounds connect + the entire read below; on expiry the pending // reader.read() rejects and we mark the turn failed. signal: AbortSignal.timeout(timeoutMs), @@ -114,60 +192,15 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { }; } - const state = { + const state: DriveState = { reply: "", - toolCalls: [] as string[], + toolCalls: [], seen: new Set(), ok: true, - traceId: undefined as string | undefined, }; - const reader = res.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - let eventName = ""; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split("\n"); - buffer = lines.pop() ?? ""; - for (const line of lines) { - // A blank line terminates an SSE event block — reset the pending - // type so it doesn't bleed into the next event. - if (line === "") { - eventName = ""; - continue; - } - // A stream-level error (a thrown exception in the generator) is - // written as `event: error` with a payload that carries no `type` - // field, so applyEvent can't see it. The SSE event name is the - // real contract, so track it and treat `error` as a failed turn. - if (line.startsWith("event:")) { - eventName = line.slice(6).trim(); - continue; - } - if (!line.startsWith("data: ")) continue; - const data = line.slice(6).trim(); - if (!data || data === "[DONE]") continue; - if (eventName === "error") state.ok = false; - try { - applyEvent(JSON.parse(data), state, (id) => { - threadId = id; - }); - } catch { - // skip malformed event lines - } - } - } - } catch { - // timeout abort or a mid-stream transport error: a hung/broken turn, - // not a passing one. - state.ok = false; - } finally { - reader.releaseLock(); - } + await drainSse(res.body.getReader(), state, (id) => { + threadId = id; + }); return { reply: state.reply, diff --git a/packages/appkit/src/evals/mlflow-report.ts b/packages/appkit/src/evals/mlflow-report.ts index 859ffaf9d..6db1ffed3 100644 --- a/packages/appkit/src/evals/mlflow-report.ts +++ b/packages/appkit/src/evals/mlflow-report.ts @@ -1,5 +1,5 @@ import type { MlflowClient } from "../connectors/mlflow"; -import type { EvalResult } from "./types"; +import type { AssertionResult, EvalResult } from "./types"; /** A Feedback assessment in the MLflow REST proto-JSON shape. */ export interface Assessment { @@ -32,31 +32,30 @@ function sanitizeName(label: string): string { * render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. * Returns [] when there's no trace to attach to or the eval was skipped. */ -export function buildAssessments(result: EvalResult): Assessment[] { - if (!result.traceId || result.skipped) return []; - const traceId = result.traceId; - const out: Assessment[] = []; - const used = new Map(); - - for (const a of result.assertions) { - const isJudge = a.label.startsWith("judge."); - const base = sanitizeName(a.label); - const seen = used.get(base) ?? 0; - used.set(base, seen + 1); - out.push({ - trace_id: traceId, - assessment_name: seen === 0 ? base : `${base}_${seen + 1}`, - source: isJudge - ? { source_type: "LLM_JUDGE", source_id: "appkit-judge" } - : { source_type: "CODE", source_id: "appkit-eval" }, - // Judges report a numeric score; deterministic assertions a boolean. - feedback: { value: a.score ?? a.pass }, - rationale: a.detail, - metadata: { eval_id: result.id, severity: a.severity }, - }); - } +/** One Feedback assessment for a single assertion (judge assertions tagged `LLM_JUDGE`). */ +function assertionAssessment( + a: AssertionResult, + traceId: string, + name: string, + evalId: string, +): Assessment { + const isJudge = a.label.startsWith("judge."); + return { + trace_id: traceId, + assessment_name: name, + source: isJudge + ? { source_type: "LLM_JUDGE", source_id: "appkit-judge" } + : { source_type: "CODE", source_id: "appkit-eval" }, + // Judges report a numeric score; deterministic assertions a boolean. + feedback: { value: a.score ?? a.pass }, + rationale: a.detail, + metadata: { eval_id: evalId, severity: a.severity }, + }; +} - out.push({ +/** The overall `appkit_eval` pass/fail assessment for an eval result. */ +function overallAssessment(result: EvalResult, traceId: string): Assessment { + return { trace_id: traceId, assessment_name: "appkit_eval", source: { source_type: "CODE", source_id: "appkit-eval" }, @@ -67,8 +66,24 @@ export function buildAssessments(result: EvalResult): Assessment[] { ? "all gates passed" : "one or more gates failed", metadata: { eval_id: result.id }, - }); + }; +} + +export function buildAssessments(result: EvalResult): Assessment[] { + if (!result.traceId || result.skipped) return []; + const traceId = result.traceId; + const out: Assessment[] = []; + const used = new Map(); + + for (const a of result.assertions) { + const base = sanitizeName(a.label); + const seen = used.get(base) ?? 0; + used.set(base, seen + 1); + const name = seen === 0 ? base : `${base}_${seen + 1}`; + out.push(assertionAssessment(a, traceId, name, result.id)); + } + out.push(overallAssessment(result, traceId)); return out; } diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 7ee95b0ef..58a335c37 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -1,7 +1,7 @@ import { pathToFileURL } from "node:url"; import { MlflowClient } from "../connectors/mlflow"; -import { discoverEvalFiles } from "./discover"; +import { type DiscoveredEval, discoverEvalFiles } from "./discover"; import { createHttpDriver } from "./http-driver"; import { configureJudge } from "./judge"; import { type ReportOutcome, reportToMlflow } from "./mlflow-report"; @@ -95,6 +95,66 @@ export function resolveEvalDefault(mod: unknown): EvalDefinition | undefined { return undefined; } +/** + * Load and run a single discovered eval. Never throws — a load/run failure + * becomes a non-passing {@link EvalResult} so one bad eval can't abort the run. + */ +async function runOne( + d: DiscoveredEval, + id: string, + runId: string | undefined, + options: RunEvalsOptions, +): Promise { + try { + const def = await loadEval(d.file); + const driver = createHttpDriver({ + baseUrl: options.baseUrl, + agent: def.agent ?? d.agent, + headers: options.headers, + mlflowRunId: runId, + timeoutMs: options.timeoutMs, + }); + return await runEval(def, { id, driver, strict: options.strict }); + } catch (err) { + return { + id, + assertions: [], + passed: false, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +/** Configure the LLM judge when judge creds were supplied; otherwise a no-op. */ +async function maybeConfigureJudge(options: RunEvalsOptions): Promise { + if (!options.judge) return; + await configureJudge({ + client: new MlflowClient(options.judge.host, options.judge.token), + token: options.judge.token, + model: options.judge.model, + }); +} + +/** + * Report per-eval assessments and finish the MLflow run, when one was created. + * Returns the run summary, or `undefined` when there was no run to finalize. + */ +async function finalizeMlflow( + client: MlflowClient | undefined, + runId: string | undefined, + results: EvalResult[], + options: RunEvalsOptions, +): Promise { + if (!client || !runId) return undefined; + const report = await reportToMlflow(client, results); + const finish = await finishEvalRun(client, { + runId, + results, + endTime: options.now ?? Date.now(), + }); + return { runId, report, finish }; +} + /** * Discover, load, and run every eval under each agent's `evals/` dir, driving * the agents on a running app. Never throws for an individual eval — load/run @@ -118,13 +178,7 @@ export async function runEvalsInDir( const total = discovered.length; emit({ type: "discovered", total }); - if (options.judge) { - await configureJudge({ - client: new MlflowClient(options.judge.host, options.judge.token), - token: options.judge.token, - model: options.judge.model, - }); - } + await maybeConfigureJudge(options); // Create the MLflow evaluation run up front so each eval's trace can be // linked to it as it runs. One client is shared by run create/finish and the @@ -146,40 +200,13 @@ export async function runEvalsInDir( const d = discovered[index]; const id = `${d.agent}/${d.id}`; emit({ type: "start", id, index, total }); - - let result: EvalResult; - try { - const def = await loadEval(d.file); - const driver = createHttpDriver({ - baseUrl: options.baseUrl, - agent: def.agent ?? d.agent, - headers: options.headers, - mlflowRunId: runId, - timeoutMs: options.timeoutMs, - }); - result = await runEval(def, { id, driver, strict: options.strict }); - } catch (err) { - result = { - id, - assertions: [], - passed: false, - error: err instanceof Error ? err.message : String(err), - }; - } - + const result = await runOne(d, id, runId, options); results.push(result); emit({ type: "result", result, index, total }); } - if (mlflowClient && runId) { - const report = await reportToMlflow(mlflowClient, results); - const finish = await finishEvalRun(mlflowClient, { - runId, - results, - endTime: options.now ?? Date.now(), - }); - return { results, mlflow: { runId, report, finish } }; - } - - return { results }; + const summary: EvalRunSummary = { results }; + const mlflow = await finalizeMlflow(mlflowClient, runId, results, options); + if (mlflow) summary.mlflow = mlflow; + return summary; } diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index 003e43217..b5b0a4ce8 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -82,42 +82,38 @@ interface EvalOptions { judgeModel?: string; } -async function runAgentEval( - filter: string | undefined, - opts: EvalOptions, -): Promise { - const runner = await loadRunner(); - - // Resolve Databricks host + bearer the AppKit-native way: an explicit - // host/token (or DATABRICKS_* env) wins; otherwise the SDK mints an OAuth - // token from the CLI profile — so no hand-set PAT is required. - const auth = await runner.resolveDatabricksAuth({ - profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE, - host: opts.databricksHost ?? process.env.DATABRICKS_HOST, - token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN, - }); - const host = auth?.host; - const token = auth?.token; +/** Resolved Databricks host + bearer (either field may be absent). */ +type Auth = { host?: string; token?: string }; - // Create a native MLflow "Evaluation run" when creds + an experiment are - // available (traces live in the app; the run + scores are driven from here). +/** + * Native MLflow "Evaluation run" config — only when creds + an experiment are + * all present (traces live in the app; the run + scores are driven from here). + */ +function resolveMlflow(opts: EvalOptions, auth: Auth) { const experimentId = opts.experiment ?? process.env.MLFLOW_EXPERIMENT_ID; - const mlflow = - host && token && experimentId ? { host, token, experimentId } : undefined; - - // LLM-as-judge: reuse the Databricks creds + a judge serving endpoint. - const judgeModel = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL; - const judge = - judgeModel && host && token - ? { host, token, model: judgeModel } - : undefined; - - // Stream progress as evals run, instead of going silent until the end. - const onEvent = (event: EvalProgress): void => { + return auth.host && auth.token && experimentId + ? { host: auth.host, token: auth.token, experimentId } + : undefined; +} + +/** LLM-as-judge config — reuses the Databricks creds + a judge serving endpoint. */ +function resolveJudge(opts: EvalOptions, auth: Auth) { + const model = opts.judgeModel ?? process.env.APPKIT_JUDGE_MODEL; + return model && auth.host && auth.token + ? { host: auth.host, token: auth.token, model } + : undefined; +} + +/** Progress reporter: stream each eval as it runs instead of going silent. */ +function makeProgressReporter( + runner: EvalRunner, + url: string, +): (event: EvalProgress) => void { + return (event) => { switch (event.type) { case "discovered": console.log( - `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${opts.url}\n`, + `Running ${event.total} eval${event.total === 1 ? "" : "s"} against ${url}\n`, ); break; case "run-created": @@ -137,6 +133,54 @@ async function runAgentEval( } } }; +} + +function formatFailureLine(f: { + traceId: string; + status?: number; + error?: string; +}): string { + return ` ✗ trace ${f.traceId}: ${f.status ?? ""} ${f.error ?? ""}`.trim(); +} + +/** Print the MLflow assessment/finish outcome after a run that created one. */ +function printMlflowOutcome( + mlflow: NonNullable, +): void { + const { report, finish } = mlflow; + console.log( + `MLflow: ${report.written} assessment(s) written` + + (report.skipped ? `, ${report.skipped} skipped` : "") + + (report.failures.length ? `, ${report.failures.length} failed` : ""), + ); + for (const f of report.failures) { + console.error(formatFailureLine(f)); + } + if (finish.metricsError) { + console.error(` ⚠ metrics not logged: ${finish.metricsError}`); + } + if (!finish.finished) { + console.error( + ` ✗ run left RUNNING — failed to finish: ${finish.finishError ?? "unknown"}`, + ); + } +} + +async function runAgentEval( + filter: string | undefined, + opts: EvalOptions, +): Promise { + const runner = await loadRunner(); + + // Resolve Databricks host + bearer the AppKit-native way: an explicit + // host/token (or DATABRICKS_* env) wins; otherwise the SDK mints an OAuth + // token from the CLI profile — so no hand-set PAT is required. + const auth: Auth = + (await runner.resolveDatabricksAuth({ + profile: opts.profile ?? process.env.DATABRICKS_CONFIG_PROFILE, + host: opts.databricksHost ?? process.env.DATABRICKS_HOST, + token: opts.databricksToken ?? process.env.DATABRICKS_TOKEN, + })) ?? {}; const summary = await runner.runEvalsInDir({ rootDir: opts.root, @@ -144,32 +188,14 @@ async function runAgentEval( filter, strict: opts.strict, headers: opts.header ? parseHeaders(opts.header) : undefined, - mlflow, - judge, - onEvent, + mlflow: resolveMlflow(opts, auth), + judge: resolveJudge(opts, auth), + onEvent: makeProgressReporter(runner, opts.url), }); console.log(`\n${runner.formatSummaryLine(summary.results)}`); if (summary.mlflow) { - const { report, finish } = summary.mlflow; - console.log( - `MLflow: ${report.written} assessment(s) written` + - (report.skipped ? `, ${report.skipped} skipped` : "") + - (report.failures.length ? `, ${report.failures.length} failed` : ""), - ); - for (const f of report.failures) { - console.error( - ` ✗ trace ${f.traceId}: ${f.status ?? ""} ${f.error ?? ""}`.trim(), - ); - } - if (finish.metricsError) { - console.error(` ⚠ metrics not logged: ${finish.metricsError}`); - } - if (!finish.finished) { - console.error( - ` ✗ run left RUNNING — failed to finish: ${finish.finishError ?? "unknown"}`, - ); - } + printMlflowOutcome(summary.mlflow); } else { console.log( "\nMLflow evaluation run skipped — pass --experiment (or set" + From 1cbea6016c1ffac5ab38cc5c9faa2438926c8fd6 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Tue, 1 Sep 2026 17:48:38 +0200 Subject: [PATCH 12/14] refactor(appkit): reuse readSseEvents in eval http-driver Replace the bespoke SSE line parser (handleSseLine + drainSse) in the eval HTTP driver with the existing readSseEvents reader from stream/sse-reader. Net -29 lines, and gains the reader's maxLineChars/maxBufferChars DoS bounds that the hand-rolled buffer lacked. Behavior preserved: timeouts still yield succeeded:false (explicit signal.aborted check, since the reader cancels cleanly rather than throwing) and stream-level error frames still fail the turn via the event name. Signed-off-by: MarioCadenas --- packages/appkit/src/evals/http-driver.ts | 91 ++++++++---------------- 1 file changed, 31 insertions(+), 60 deletions(-) diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts index 14c571e9d..f8d6ce00f 100644 --- a/packages/appkit/src/evals/http-driver.ts +++ b/packages/appkit/src/evals/http-driver.ts @@ -1,3 +1,4 @@ +import { readSseEvents } from "../stream/sse-reader"; import type { DriveResult, EvalDriver } from "./types"; export interface HttpDriverOptions { @@ -87,61 +88,6 @@ function applyEvent( } } -/** - * Apply one raw SSE line to `state`, returning the (possibly updated) current - * event name. The event name is tracked because a stream-level error (a thrown - * exception in the generator) is framed as `event: error` with a payload that - * carries no `type` field — applyEvent can't see it, so the SSE event name is - * the real contract, and `error` marks the turn failed. - */ -function handleSseLine( - line: string, - eventName: string, - state: DriveState, - setThread: (id: string) => void, -): string { - if (line === "") return ""; // blank line terminates an SSE event block - if (line.startsWith("event:")) return line.slice(6).trim(); - if (!line.startsWith("data: ")) return eventName; - const data = line.slice(6).trim(); - if (!data || data === "[DONE]") return eventName; - if (eventName === "error") state.ok = false; - try { - applyEvent(JSON.parse(data), state, setThread); - } catch { - // skip malformed event lines - } - return eventName; -} - -/** Read the SSE stream to completion, applying each line into `state`. */ -async function drainSse( - reader: ReadableStreamDefaultReader, - state: DriveState, - setThread: (id: string) => void, -): Promise { - const decoder = new TextDecoder(); - let buffer = ""; - let eventName = ""; - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split("\n"); - buffer = lines.pop() ?? ""; - for (const line of lines) { - eventName = handleSseLine(line, eventName, state, setThread); - } - } - } catch { - // timeout abort or a mid-stream transport error: a hung/broken turn. - state.ok = false; - } finally { - reader.releaseLock(); - } -} - /** Build the chat request payload, including only the optional fields that are set. */ function buildRequestBody( message: string, @@ -169,15 +115,16 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { return { async send(message: string): Promise { + // Bounds connect + the entire read below. Passed to both the fetch and + // the SSE reader: on expiry the reader is cancelled and the turn fails. + const signal = AbortSignal.timeout(timeoutMs); let res: Response; try { res = await fetch(`${options.baseUrl}${chatPath}`, { method: "POST", headers: { "content-type": "application/json", ...options.headers }, body: buildRequestBody(message, options, threadId), - // bounds connect + the entire read below; on expiry the pending - // reader.read() rejects and we mark the turn failed. - signal: AbortSignal.timeout(timeoutMs), + signal, }); } catch { return { reply: "", toolCalls: [], succeeded: false }; @@ -198,9 +145,33 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { seen: new Set(), ok: true, }; - await drainSse(res.body.getReader(), state, (id) => { + const setThread = (id: string) => { threadId = id; - }); + }; + try { + for await (const { event, data } of readSseEvents(res.body, signal)) { + if (data === "[DONE]") continue; + // A stream-level error frame (a thrown exception in the generator) + // is framed as `event: error` with a payload that carries no `type`, + // so the SSE event name — not the payload — is the real signal. + if (event === "error") state.ok = false; + try { + applyEvent( + JSON.parse(data) as Record, + state, + setThread, + ); + } catch { + // skip malformed event payloads + } + } + } catch { + // mid-stream transport error or DoS-guard throw: a broken turn. + state.ok = false; + } + // A timeout ends the iterator cleanly (reader cancel) rather than + // throwing, so mark an aborted turn failed explicitly. + if (signal.aborted) state.ok = false; return { reply: state.reply, From e35f2abc3d9c5f474354f5cc0d45e672aa85173b Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Tue, 1 Sep 2026 18:07:38 +0200 Subject: [PATCH 13/14] fix(appkit): capture terminal message reply + harden eval http-driver - http-driver: reply now reads terminal `message` output-item text (replacing deltas), so non-streaming/LangChain adapters and delta-then-corrected turns no longer return an empty/stale reply. - http-driver: set `redirect: "manual"` so custom auth headers are not replayed across a cross-origin redirect. - agents: drop unused `traceTool` import (oxlint no-unused-vars). - deps: exact-pin `autoevals` (0.3.0) to match the repo convention. Adds http-driver tests for the message-only and delta-then-message paths. Signed-off-by: MarioCadenas --- packages/appkit/package.json | 2 +- packages/appkit/src/evals/http-driver.ts | 25 ++++++-- .../src/evals/tests/http-driver.test.ts | 58 +++++++++++++++++++ packages/appkit/src/plugins/agents/agents.ts | 1 - pnpm-lock.yaml | 2 +- 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/packages/appkit/package.json b/packages/appkit/package.json index 951d909a9..7a1b08926 100644 --- a/packages/appkit/package.json +++ b/packages/appkit/package.json @@ -88,7 +88,7 @@ "@opentelemetry/semantic-conventions": "1.38.0", "@types/semver": "7.7.1", "apache-arrow": "21.1.0", - "autoevals": "^0.3.0", + "autoevals": "0.3.0", "dotenv": "16.6.1", "drizzle-orm": "0.45.2", "express": "4.22.2", diff --git a/packages/appkit/src/evals/http-driver.ts b/packages/appkit/src/evals/http-driver.ts index f8d6ce00f..dcc15aba5 100644 --- a/packages/appkit/src/evals/http-driver.ts +++ b/packages/appkit/src/evals/http-driver.ts @@ -69,10 +69,22 @@ function applyEvent( type === "response.output_item.added" || type === "response.output_item.done" ) { - recordToolCall( - event.item as { type?: string; name?: string; call_id?: string }, - state, - ); + const item = event.item as { + type?: string; + name?: string; + call_id?: string; + content?: Array<{ text?: string }>; + }; + recordToolCall(item, state); + // A terminal `message` item carries the full reply text and *replaces* + // the accumulated deltas — the server emits no delta for a full message + // (event-translator `handleFullMessage`), so a non-streaming adapter's + // whole reply arrives only here. Guard on non-empty so a streaming turn's + // trailing done event can't clobber the deltas with "". + if (type === "response.output_item.done" && item.type === "message") { + const text = (item.content ?? []).map((c) => c.text ?? "").join(""); + if (text) state.reply = text; + } return; } if (type === "error" || type === "response.failed") { @@ -124,6 +136,11 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver { method: "POST", headers: { "content-type": "application/json", ...options.headers }, body: buildRequestBody(message, options, threadId), + // A chat endpoint never needs a redirect; following one would replay + // custom auth headers (`options.headers`) to the redirect target. + // A 3xx becomes an opaque response (res.ok === false), handled as a + // failed turn by the !res.ok guard below. + redirect: "manual", signal, }); } catch { diff --git a/packages/appkit/src/evals/tests/http-driver.test.ts b/packages/appkit/src/evals/tests/http-driver.test.ts index 011716fb4..99e9a1eed 100644 --- a/packages/appkit/src/evals/tests/http-driver.test.ts +++ b/packages/appkit/src/evals/tests/http-driver.test.ts @@ -38,6 +38,50 @@ const server: Server = createServer((req, res) => { res.end(); return; + // Non-streaming adapter (e.g. LangChain): a full `message` item and NO + // text deltas. The reply must come from the message item's content. + case "/message-only": + res.write( + `event: response.output_item.added\ndata: ${JSON.stringify({ + type: "response.output_item.added", + item: { type: "message", content: [] }, + })}\n\n`, + ); + res.write( + `event: response.output_item.done\ndata: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + content: [{ type: "output_text", text: "the answer" }], + }, + })}\n\n`, + ); + res.write(`data: [DONE]\n\n`); + res.end(); + return; + + // Deltas followed by a terminal `message` that corrects them: the message + // replaces the accumulated deltas rather than appending. + case "/delta-then-message": + res.write( + `event: response.output_text.delta\ndata: ${JSON.stringify({ + type: "response.output_text.delta", + delta: "partial", + })}\n\n`, + ); + res.write( + `event: response.output_item.done\ndata: ${JSON.stringify({ + type: "response.output_item.done", + item: { + type: "message", + content: [{ type: "output_text", text: "full final content" }], + }, + })}\n\n`, + ); + res.write(`data: [DONE]\n\n`); + res.end(); + return; + // A hung agent: heartbeat comments keep the socket alive but the stream // never ends. Deliberately never call res.end(). default: @@ -73,6 +117,20 @@ describe("createHttpDriver", () => { expect(result.succeeded).toBe(false); }); + test("captures reply from a message item when there are no deltas", async () => { + const driver = createHttpDriver({ baseUrl, path: "/message-only" }); + const result = await driver.send("hi"); + expect(result.succeeded).toBe(true); + expect(result.reply).toBe("the answer"); + }); + + test("a terminal message replaces accumulated deltas", async () => { + const driver = createHttpDriver({ baseUrl, path: "/delta-then-message" }); + const result = await driver.send("hi"); + expect(result.succeeded).toBe(true); + expect(result.reply).toBe("full final content"); + }); + test("times out a hung stream instead of hanging forever", async () => { const driver = createHttpDriver({ baseUrl, diff --git a/packages/appkit/src/plugins/agents/agents.ts b/packages/appkit/src/plugins/agents/agents.ts index 17f2371ee..1ee1da1e6 100644 --- a/packages/appkit/src/plugins/agents/agents.ts +++ b/packages/appkit/src/plugins/agents/agents.ts @@ -65,7 +65,6 @@ import { initAgentTracing, linkTraceToRun, traceAgent, - traceTool, updateTracePreview, } from "./mlflow"; import { composePromptForAgent } from "./prompt"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b809372ac..ef9c8c46d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -310,7 +310,7 @@ importers: specifier: 21.1.0 version: 21.1.0 autoevals: - specifier: ^0.3.0 + specifier: 0.3.0 version: 0.3.0(ws@8.21.0(bufferutil@4.0.9))(zod@4.3.6) dotenv: specifier: 16.6.1 From ea80542b4ff877bb7573d957ecb8cf2c2169dd77 Mon Sep 17 00:00:00 2001 From: MarioCadenas Date: Tue, 1 Sep 2026 18:20:16 +0200 Subject: [PATCH 14/14] feat(appkit): run evals concurrently with a bounded pool Replace the sequential loop in runEvalsInDir with a bounded worker pool (new internal `mapPool` helper) so independent eval turns overlap instead of summing their latencies. Adds a `concurrency` option to RunEvalsOptions (default 4, clamped to [1, total]) and a `--concurrency` CLI flag; the default stays at/below the server's maxConcurrentStreamsPerUser (5) to avoid the 429 guard. Results preserve discovery order. The CLI progress reporter now prints one full line per completion (via formatEvalHeadline) since concurrent starts would otherwise interleave. Regenerates the API reference, which also syncs pre-existing doc drift in the PR's own surface (HttpDriverOptions.timeoutMs, buildAssessments, resolveDatabricksAuth). Signed-off-by: MarioCadenas --- .../api/appkit/Function.buildAssessments.md | 5 -- .../appkit/Function.resolveDatabricksAuth.md | 9 -- .../api/appkit/Interface.HttpDriverOptions.md | 14 +++ .../api/appkit/Interface.RunEvalsOptions.md | 23 +++++ docs/docs/api/appkit/index.md | 22 ++++- docs/docs/api/appkit/typedoc-sidebar.ts | 90 +++++++++++++++++++ packages/appkit/src/evals/run-evals.ts | 63 +++++++++++-- .../appkit/src/evals/tests/run-evals.test.ts | 46 ++++++++++ .../shared/src/cli/commands/agent/eval.ts | 21 +++-- 9 files changed, 261 insertions(+), 32 deletions(-) create mode 100644 packages/appkit/src/evals/tests/run-evals.test.ts diff --git a/docs/docs/api/appkit/Function.buildAssessments.md b/docs/docs/api/appkit/Function.buildAssessments.md index eda73df95..6008e86db 100644 --- a/docs/docs/api/appkit/Function.buildAssessments.md +++ b/docs/docs/api/appkit/Function.buildAssessments.md @@ -4,11 +4,6 @@ function buildAssessments(result: EvalResult): Assessment[]; ``` -Build the Feedback assessments for an eval result: one per assertion (judge -assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they -render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. -Returns [] when there's no trace to attach to or the eval was skipped. - ## Parameters | Parameter | Type | diff --git a/docs/docs/api/appkit/Function.resolveDatabricksAuth.md b/docs/docs/api/appkit/Function.resolveDatabricksAuth.md index 0fc2f1030..c3036afbf 100644 --- a/docs/docs/api/appkit/Function.resolveDatabricksAuth.md +++ b/docs/docs/api/appkit/Function.resolveDatabricksAuth.md @@ -4,15 +4,6 @@ function resolveDatabricksAuth(options: ResolveDatabricksAuthOptions): Promise; ``` -Resolve `{host, token}` for the eval runner the same way the rest of AppKit -authenticates: construct a Databricks `WorkspaceClient` and let its config -mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set -PAT required. An explicit host/token still wins (PAT or CI env), so the SDK -is only consulted for whatever isn't supplied. - -Returns `undefined` when neither an explicit token nor a resolvable profile -yields a bearer, so the caller can treat auth as simply unavailable. - ## Parameters | Parameter | Type | diff --git a/docs/docs/api/appkit/Interface.HttpDriverOptions.md b/docs/docs/api/appkit/Interface.HttpDriverOptions.md index 8bb1f25e0..26e3fde35 100644 --- a/docs/docs/api/appkit/Interface.HttpDriverOptions.md +++ b/docs/docs/api/appkit/Interface.HttpDriverOptions.md @@ -49,3 +49,17 @@ optional path: string; ``` Chat endpoint path. Defaults to `/api/agents/chat`. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Max wall-clock time for a single turn before it is abandoned as a failed +turn (`succeeded: false`). Without this a hung agent — a blocked tool, a +stalled model — never ends the SSE stream (heartbeats keep it alive), so +the read loop spins forever and wedges the whole sequential suite. +Defaults to 120s. diff --git a/docs/docs/api/appkit/Interface.RunEvalsOptions.md b/docs/docs/api/appkit/Interface.RunEvalsOptions.md index 174d0e772..7c39528c9 100644 --- a/docs/docs/api/appkit/Interface.RunEvalsOptions.md +++ b/docs/docs/api/appkit/Interface.RunEvalsOptions.md @@ -12,6 +12,19 @@ Base URL of the running app to drive, e.g. `http://localhost:3000`. *** +### concurrency? + +```ts +optional concurrency: number; +``` + +Max evals to drive concurrently. Each eval opens one stream to the app as +the same user, so keep this at or below the app's +`maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the +429 guard. Defaults to 4; clamped to `[1, total]`. + +*** + ### filter? ```ts @@ -146,3 +159,13 @@ optional strict: boolean; ``` Soft assertion failures also fail the eval. + +*** + +### timeoutMs? + +```ts +optional timeoutMs: number; +``` + +Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index 08ffdace6..42afc4a50 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -49,6 +49,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [CacheConfig](Interface.CacheConfig.md) | Configuration for the CacheInterceptor. Controls TTL, size limits, storage backend, and probabilistic cleanup. | | [CustomJudgeSpec](Interface.CustomJudgeSpec.md) | A custom LLM-judge definition: a prompt template and choice→score mapping. | | [DatabaseCredential](Interface.DatabaseCredential.md) | Database credentials with OAuth token for Postgres connection | +| [DatabaseRegistry](Interface.DatabaseRegistry.md) | CANONICAL augmentation target. Empty by default; the generated `database.d.ts` augments it via `declare module "@databricks/appkit" { interface DatabaseRegistry { ... } }`. | | [DatabricksAuth](Interface.DatabricksAuth.md) | Resolved Databricks host + bearer token for the eval runner's REST calls. | | [DiscoveredEval](Interface.DiscoveredEval.md) | An eval file found under `server/agents//evals/`. | | [DriveResult](Interface.DriveResult.md) | What a driver returns for a single `t.send`. | @@ -96,6 +97,7 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [RunAgentResult](Interface.RunAgentResult.md) | - | | [RunEvalOptions](Interface.RunEvalOptions.md) | - | | [RunEvalsOptions](Interface.RunEvalsOptions.md) | - | +| [Schema](Interface.Schema.md) | One finalized schema. `TTableName` keeps the declared names in the type, so configuration that addresses a table by name is checked against the schema it was written for. Code that accepts any schema uses the default. | | [SearchRequest](Interface.SearchRequest.md) | - | | [SearchResponse](Interface.SearchResponse.md) | - | | [SearchResult](Interface.SearchResult.md) | - | @@ -129,12 +131,14 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [AgentToolsFn](TypeAlias.AgentToolsFn.md) | Function form of `AgentDefinition.tools`. Receives the typed [Plugins](TypeAlias.Plugins.md) map and returns a tool record. Invoked exactly once at setup (or once per `runAgent` call in standalone mode); the result is cached as the agent's resolved tool record. | | [BaseSystemPromptOption](TypeAlias.BaseSystemPromptOption.md) | - | | [ConfigSchema](TypeAlias.ConfigSchema.md) | Configuration schema definition for plugin config. Re-exported from the standard JSON Schema Draft 7 types. | +| [DatabaseExports](TypeAlias.DatabaseExports.md) | Typed database API published by the plugin. | | [EvalProgress](TypeAlias.EvalProgress.md) | - | | [ExecutionResult](TypeAlias.ExecutionResult.md) | Discriminated union for plugin execution results. | | [FileAction](TypeAlias.FileAction.md) | Every action the files plugin can perform. | | [FilePolicy](TypeAlias.FilePolicy.md) | A policy function that decides whether `user` may perform `action` on `resource`. Return `true` to allow, `false` to deny. | | [HostedTool](TypeAlias.HostedTool.md) | - | | [IAppRouter](TypeAlias.IAppRouter.md) | Express router type for plugin route registration | +| [IDatabaseConfig](TypeAlias.IDatabaseConfig.md) | Configuration for one schema-bound DatabasePlugin instance. | | [JobsExport](TypeAlias.JobsExport.md) | Public API shape of the jobs plugin. Callable to select a job by key. | | [Matcher](TypeAlias.Matcher.md) | A deterministic matcher: inspects a string value and returns a result. | | [PluginData](TypeAlias.PluginData.md) | Tuple of plugin class, config, and name. Created by `toPlugin()` and passed to `createApp()`. | @@ -168,7 +172,10 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [agentIdFromMarkdownPath](Function.agentIdFromMarkdownPath.md) | Derives the logical agent id from a markdown path. When the file is named `agent.md`, the id is the parent directory name (folder-based layout); otherwise the id is the file stem (e.g. legacy single-file paths). | | [appKitServingTypesPlugin](Function.appKitServingTypesPlugin.md) | Vite plugin to generate TypeScript types for AppKit serving endpoints. Fetches OpenAPI schemas from Databricks and generates a .d.ts with ServingEndpointRegistry module augmentation. | | [appKitTypesPlugin](Function.appKitTypesPlugin.md) | Vite plugin to generate types for AppKit queries. Calls generateFromEntryPoint under the hood. | -| [buildAssessments](Function.buildAssessments.md) | Build the Feedback assessments for an eval result: one per assertion (judge assertions tagged `LLM_JUDGE` with their numeric score + rationale, so they render as judge feedback in MLflow) plus an overall `appkit_eval` pass/fail. Returns [] when there's no trace to attach to or the eval was skipped. | +| [bigid](Function.bigid.md) | - | +| [bigint](Function.bigint.md) | - | +| [boolean](Function.boolean.md) | - | +| [buildAssessments](Function.buildAssessments.md) | - | | [configureJudge](Function.configureJudge.md) | Configure the judge once. Sets the OpenAI-compatible client env autoevals reads and the default judge model. No-op-safe: on failure, judging stays disabled and [isJudgeConfigured](Function.isJudgeConfigured.md) returns false. | | [createAgent](Function.createAgent.md) | Pure factory for agent definitions: cycle-detects the sub-agent graph and returns the same object, stamped with a non-enumerable AGENT\_BRAND so discovery recognizes it. Safe at module top-level; no adapter is built. Don't `Object.freeze` the definition before passing it in — the brand is written onto the argument. | | [createApp](Function.createApp.md) | Bootstraps AppKit with the provided configuration. | @@ -176,15 +183,19 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [createLakebasePool](Function.createLakebasePool.md) | Create a Lakebase pool with appkit's logger integration. Telemetry automatically uses appkit's OpenTelemetry configuration via global registry. | | [createLakebasePoolManager](Function.createLakebasePoolManager.md) | Create a pool manager that maintains per-key Lakebase connection pools. | | [createWorkspaceClient](Function.createWorkspaceClient.md) | Construct an AppKit workspace client. | +| [database](Function.database.md) | Create a typed database plugin registration for a finalized schema. | | [defineEval](Function.defineEval.md) | Define an agent eval. Default-export the result from a `server/agents//evals/*.eval.ts` file. | | [defineManifest](Function.defineManifest.md) | Validates a raw manifest (typically a `manifest.json` import) against the canonical Zod schema and returns it as a strict [PluginManifest](Interface.PluginManifest.md). | +| [defineSchema](Function.defineSchema.md) | Compile one declared schema. The returned type keeps the table names the builder returned, so `crudRoutes` and `hooks` can name only real tables. | | [defineTool](Function.defineTool.md) | Defines a single tool entry for a plugin's internal registry. | | [discoverEvalFiles](Function.discoverEvalFiles.md) | Discover evals under `/server/agents//evals/` — co-located with each agent's `agent.{md,ts}` (same folder-per-agent layout the agents plugin discovers). The agent id is the folder name; the eval id is the file path relative to that evals dir with `.eval.ts` stripped. Sorted + stable. | +| [enumColumn](Function.enumColumn.md) | - | | [equals](Function.equals.md) | Passes when the value equals `expected` exactly. | | [evalGlyph](Function.evalGlyph.md) | Status glyph for a single eval result. | | [executeFromRegistry](Function.executeFromRegistry.md) | Validates tool-call arguments against the entry's schema and invokes its handler. On validation failure, returns an LLM-friendly error string (matching the behavior of `tool()`) rather than throwing, so the model can self-correct on its next turn. | | [extractServingEndpoints](Function.extractServingEndpoints.md) | Extract serving endpoint config from a server file by AST-parsing it. Looks for `serving({ endpoints: { alias: { env: "..." }, ... } })` calls and extracts the endpoint alias names and their environment variable mappings. | | [findServerFile](Function.findServerFile.md) | Find the server entry file by checking candidate paths in order. | +| [fk](Function.fk.md) | Declare foreign-key to another column. | | [formatEvalDetail](Function.formatEvalDetail.md) | Indented detail lines for a failing eval (error + failing assertions). | | [formatEvalHeadline](Function.formatEvalHeadline.md) | The one-line header for a single eval result (no failure detail). | | [formatEvalResults](Function.formatEvalResults.md) | Render all results as a human-readable console report (non-streaming). | @@ -199,13 +210,16 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [getResourceRequirements](Function.getResourceRequirements.md) | Gets the resource requirements from a plugin's manifest. | | [getUsernameWithApiLookup](Function.getUsernameWithApiLookup.md) | Resolves the PostgreSQL username for a Lakebase connection. | | [getWorkspaceClient](Function.getWorkspaceClient.md) | Get workspace client from config or SDK default auth chain | +| [id](Function.id.md) | - | | [includes](Function.includes.md) | Passes when the value contains `substring`. | +| [integer](Function.integer.md) | - | | [isFunctionTool](Function.isFunctionTool.md) | - | | [isHostedTool](Function.isHostedTool.md) | - | | [isJudgeConfigured](Function.isJudgeConfigured.md) | - | | [isSQLTypeMarker](Function.isSQLTypeMarker.md) | Type guard to check if a value is a SQL type marker | | [isSupervisorTool](Function.isSupervisorTool.md) | Type guard for [HostedSupervisorTool](Interface.HostedSupervisorTool.md). Used by the agents plugin (`buildToolIndex`) and standalone `runAgent` (`classifyTool`) to route supervisor-hosted tools to the extensions payload rather than the adapter's `tools` array. | | [isToolkitEntry](Function.isToolkitEntry.md) | Type guard for `ToolkitEntry` — used by the agents plugin to differentiate toolkit references from inline tools in a mixed `tools` record. | +| [jsonb](Function.jsonb.md) | - | | [loadAgentFromFile](Function.loadAgentFromFile.md) | Loads a single markdown agent file and resolves its frontmatter against registered plugin toolkits + ambient tool library. | | [loadAgentsFromDir](Function.loadAgentsFromDir.md) | Scans a directory for one subdirectory per agent, each containing `agent.md` (frontmatter + body). Produces an `AgentDefinition` record keyed by agent id (folder name). Throws on frontmatter errors or unresolved references. Returns an empty map if the directory does not exist. | | [matches](Function.matches.md) | Passes when the value matches `pattern`. | @@ -213,11 +227,15 @@ surface with `@databricks/appkit/beta`. Not meant for application imports. | [normalizeHost](Function.normalizeHost.md) | Ensure the host has a scheme (Databricks env often lacks `https://`). | | [parseTextToolCalls](Function.parseTextToolCalls.md) | Parses text-based tool calls from model output. | | [reportToMlflow](Function.reportToMlflow.md) | Write one pass/fail assessment per eval result to the Databricks MLflow REST API. Never throws — failures are collected so the run still reports. | -| [resolveDatabricksAuth](Function.resolveDatabricksAuth.md) | Resolve `{host, token}` for the eval runner the same way the rest of AppKit authenticates: construct a Databricks `WorkspaceClient` and let its config mint (and later refresh) an OAuth bearer from the CLI profile — no hand-set PAT required. An explicit host/token still wins (PAT or CI env), so the SDK is only consulted for whatever isn't supplied. | +| [resolveDatabricksAuth](Function.resolveDatabricksAuth.md) | - | | [resolveHostedTools](Function.resolveHostedTools.md) | - | | [runAgent](Function.runAgent.md) | Standalone agent execution without `createApp`. Resolves the adapter, binds inline tools, and drives the adapter's `run()` loop to completion. | | [runEval](Function.runEval.md) | Run a single eval against a driver. Never throws for assertion or agent failures — those become a non-passing [EvalResult](Interface.EvalResult.md). Only a malformed eval definition surfaces as `result.error`. | | [runEvalsInDir](Function.runEvalsInDir.md) | Discover, load, and run every eval under each agent's `evals/` dir, driving the agents on a running app. Never throws for an individual eval — load/run failures become non-passing [EvalResult](Interface.EvalResult.md)s. | | [summarize](Function.summarize.md) | - | +| [text](Function.text.md) | - | +| [timestamp](Function.timestamp.md) | - | | [tool](Function.tool.md) | Factory for defining function tools with Zod schemas. | | [toolsFromRegistry](Function.toolsFromRegistry.md) | Produces the `AgentToolDefinition[]` a ToolProvider exposes to the LLM, deriving `parameters` JSON Schema from each entry's Zod schema. | +| [uuid](Function.uuid.md) | - | +| [varchar](Function.varchar.md) | - | diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index c26271827..6dd11de11 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -177,6 +177,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.DatabaseCredential", label: "DatabaseCredential" }, + { + type: "doc", + id: "api/appkit/Interface.DatabaseRegistry", + label: "DatabaseRegistry" + }, { type: "doc", id: "api/appkit/Interface.DatabricksAuth", @@ -412,6 +417,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.RunEvalsOptions", label: "RunEvalsOptions" }, + { + type: "doc", + id: "api/appkit/Interface.Schema", + label: "Schema" + }, { type: "doc", id: "api/appkit/Interface.SearchRequest", @@ -558,6 +568,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.ConfigSchema", label: "ConfigSchema" }, + { + type: "doc", + id: "api/appkit/TypeAlias.DatabaseExports", + label: "DatabaseExports" + }, { type: "doc", id: "api/appkit/TypeAlias.EvalProgress", @@ -588,6 +603,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/TypeAlias.IAppRouter", label: "IAppRouter" }, + { + type: "doc", + id: "api/appkit/TypeAlias.IDatabaseConfig", + label: "IDatabaseConfig" + }, { type: "doc", id: "api/appkit/TypeAlias.JobsExport", @@ -715,6 +735,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.appKitTypesPlugin", label: "appKitTypesPlugin" }, + { + type: "doc", + id: "api/appkit/Function.bigid", + label: "bigid" + }, + { + type: "doc", + id: "api/appkit/Function.bigint", + label: "bigint" + }, + { + type: "doc", + id: "api/appkit/Function.boolean", + label: "boolean" + }, { type: "doc", id: "api/appkit/Function.buildAssessments", @@ -755,6 +790,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.createWorkspaceClient", label: "createWorkspaceClient" }, + { + type: "doc", + id: "api/appkit/Function.database", + label: "database" + }, { type: "doc", id: "api/appkit/Function.defineEval", @@ -765,6 +805,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.defineManifest", label: "defineManifest" }, + { + type: "doc", + id: "api/appkit/Function.defineSchema", + label: "defineSchema" + }, { type: "doc", id: "api/appkit/Function.defineTool", @@ -775,6 +820,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.discoverEvalFiles", label: "discoverEvalFiles" }, + { + type: "doc", + id: "api/appkit/Function.enumColumn", + label: "enumColumn" + }, { type: "doc", id: "api/appkit/Function.equals", @@ -800,6 +850,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.findServerFile", label: "findServerFile" }, + { + type: "doc", + id: "api/appkit/Function.fk", + label: "fk" + }, { type: "doc", id: "api/appkit/Function.formatEvalDetail", @@ -870,11 +925,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.getWorkspaceClient", label: "getWorkspaceClient" }, + { + type: "doc", + id: "api/appkit/Function.id", + label: "id" + }, { type: "doc", id: "api/appkit/Function.includes", label: "includes" }, + { + type: "doc", + id: "api/appkit/Function.integer", + label: "integer" + }, { type: "doc", id: "api/appkit/Function.isFunctionTool", @@ -905,6 +970,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.isToolkitEntry", label: "isToolkitEntry" }, + { + type: "doc", + id: "api/appkit/Function.jsonb", + label: "jsonb" + }, { type: "doc", id: "api/appkit/Function.loadAgentFromFile", @@ -970,6 +1040,16 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Function.summarize", label: "summarize" }, + { + type: "doc", + id: "api/appkit/Function.text", + label: "text" + }, + { + type: "doc", + id: "api/appkit/Function.timestamp", + label: "timestamp" + }, { type: "doc", id: "api/appkit/Function.tool", @@ -979,6 +1059,16 @@ const typedocSidebar: SidebarsConfig = { type: "doc", id: "api/appkit/Function.toolsFromRegistry", label: "toolsFromRegistry" + }, + { + type: "doc", + id: "api/appkit/Function.uuid", + label: "uuid" + }, + { + type: "doc", + id: "api/appkit/Function.varchar", + label: "varchar" } ] } diff --git a/packages/appkit/src/evals/run-evals.ts b/packages/appkit/src/evals/run-evals.ts index 58a335c37..265d64e1b 100644 --- a/packages/appkit/src/evals/run-evals.ts +++ b/packages/appkit/src/evals/run-evals.ts @@ -22,6 +22,13 @@ export interface RunEvalsOptions { headers?: Record; /** Per-turn wall-clock timeout (ms) before a turn is failed. Defaults to 120s. */ timeoutMs?: number; + /** + * Max evals to drive concurrently. Each eval opens one stream to the app as + * the same user, so keep this at or below the app's + * `maxConcurrentStreamsPerUser` (default 5) or the surplus streams hit the + * 429 guard. Defaults to 4; clamped to `[1, total]`. + */ + concurrency?: number; /** * When set, create a native MLflow "Evaluation run": each eval's trace is * linked to the run, pass/fail is written as feedback, and aggregate metrics @@ -155,6 +162,39 @@ async function finalizeMlflow( return { runId, report, finish }; } +/** + * Default max evals in flight. Each eval opens one stream to the app as the + * same user; the server caps concurrent streams per user at 5 by default + * (`maxConcurrentStreamsPerUser`), so 4 leaves headroom under that limit. + */ +const DEFAULT_CONCURRENCY = 4; + +/** + * Run `fn` over `items` with at most `concurrency` calls in flight, preserving + * input order in the result array. `concurrency` is clamped to `[1, length]`. + * `fn` must not reject — a rejection abandons the other in-flight items. + */ +export async function mapPool( + items: T[], + concurrency: number, + fn: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length); + const limit = Math.max(1, Math.min(concurrency, items.length)); + let cursor = 0; + // Each worker pulls the next index off the shared cursor until exhausted. + // `cursor++` is atomic between awaits (single-threaded), so no index is + // handed to two workers. + const worker = async (): Promise => { + while (cursor < items.length) { + const index = cursor++; + results[index] = await fn(items[index], index); + } + }; + await Promise.all(Array.from({ length: limit }, () => worker())); + return results; +} + /** * Discover, load, and run every eval under each agent's `evals/` dir, driving * the agents on a running app. Never throws for an individual eval — load/run @@ -195,15 +235,20 @@ export async function runEvalsInDir( emit({ type: "run-created", runId }); } - const results: EvalResult[] = []; - for (let index = 0; index < discovered.length; index++) { - const d = discovered[index]; - const id = `${d.agent}/${d.id}`; - emit({ type: "start", id, index, total }); - const result = await runOne(d, id, runId, options); - results.push(result); - emit({ type: "result", result, index, total }); - } + // Run evals through a bounded pool so independent turns overlap instead of + // summing their latencies. runOne never throws, so a pool worker never + // rejects; results preserve discovery order (mapPool writes by index). + const results = await mapPool( + discovered, + options.concurrency ?? DEFAULT_CONCURRENCY, + async (d, index) => { + const id = `${d.agent}/${d.id}`; + emit({ type: "start", id, index, total }); + const result = await runOne(d, id, runId, options); + emit({ type: "result", result, index, total }); + return result; + }, + ); const summary: EvalRunSummary = { results }; const mlflow = await finalizeMlflow(mlflowClient, runId, results, options); diff --git a/packages/appkit/src/evals/tests/run-evals.test.ts b/packages/appkit/src/evals/tests/run-evals.test.ts new file mode 100644 index 000000000..2329c429a --- /dev/null +++ b/packages/appkit/src/evals/tests/run-evals.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "vitest"; + +import { mapPool } from "../run-evals"; + +describe("mapPool", () => { + test("preserves input order even when later items finish first", async () => { + // Earlier items sleep longer, so they resolve after later ones. + const out = await mapPool([30, 20, 10, 0], 2, async (ms, i) => { + await new Promise((r) => setTimeout(r, ms)); + return i; + }); + expect(out).toEqual([0, 1, 2, 3]); + }); + + test("never exceeds the concurrency limit", async () => { + let inFlight = 0; + let maxInFlight = 0; + await mapPool( + Array.from({ length: 10 }, (_, i) => i), + 3, + async () => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((r) => setTimeout(r, 5)); + inFlight--; + return null; + }, + ); + expect(maxInFlight).toBe(3); + }); + + test("processes every item exactly once", async () => { + const seen: number[] = []; + const out = await mapPool([1, 2, 3, 4, 5], 2, async (n) => { + seen.push(n); + return n * 2; + }); + expect(out).toEqual([2, 4, 6, 8, 10]); + expect(seen.sort((a, b) => a - b)).toEqual([1, 2, 3, 4, 5]); + }); + + test("clamps concurrency to [1, length] and handles empty input", async () => { + expect(await mapPool([1, 2], 99, async (n) => n)).toEqual([1, 2]); + expect(await mapPool([], 4, async (n) => n)).toEqual([]); + }); +}); diff --git a/packages/shared/src/cli/commands/agent/eval.ts b/packages/shared/src/cli/commands/agent/eval.ts index b5b0a4ce8..a11b8f969 100644 --- a/packages/shared/src/cli/commands/agent/eval.ts +++ b/packages/shared/src/cli/commands/agent/eval.ts @@ -27,6 +27,7 @@ interface EvalRunner { filter?: string; strict?: boolean; headers?: Record; + concurrency?: number; mlflow?: { host: string; token: string; experimentId: string }; judge?: { host: string; token: string; model: string }; onEvent?: (event: EvalProgress) => void; @@ -36,7 +37,7 @@ interface EvalRunner { host?: string; token?: string; }): Promise<{ host: string; token: string } | undefined>; - evalGlyph(result: unknown): string; + formatEvalHeadline(result: unknown): string; formatEvalDetail(result: unknown): string[]; formatSummaryLine(results: unknown[]): string; summarize(results: unknown[]): { allPassed: boolean }; @@ -80,6 +81,7 @@ interface EvalOptions { databricksToken?: string; experiment?: string; judgeModel?: string; + concurrency?: number; } /** Resolved Databricks host + bearer (either field may be absent). */ @@ -119,13 +121,12 @@ function makeProgressReporter( case "run-created": console.log(`MLflow evaluation run: ${event.runId}\n`); break; - case "start": - process.stdout.write( - `▸ [${event.index + 1}/${event.total}] ${event.id} … `, - ); - break; case "result": { - process.stdout.write(`${runner.evalGlyph(event.result)}\n`); + // One full line per completion — evals run concurrently, so a split + // "start … glyph" prefix would interleave into garbage. + console.log( + `[${event.index + 1}/${event.total}] ${runner.formatEvalHeadline(event.result)}`, + ); for (const line of runner.formatEvalDetail(event.result)) { console.log(line); } @@ -188,6 +189,7 @@ async function runAgentEval( filter, strict: opts.strict, headers: opts.header ? parseHeaders(opts.header) : undefined, + concurrency: opts.concurrency, mlflow: resolveMlflow(opts, auth), judge: resolveJudge(opts, auth), onEvent: makeProgressReporter(runner, opts.url), @@ -218,6 +220,11 @@ export const agentEvalCommand = new Command("eval") ) .option("--url ", "Base URL of the running app", "http://localhost:3000") .option("--strict", "Fail on soft-assertion misses too", false) + .option( + "--concurrency ", + "Max evals to run concurrently (default 4; keep at or below the app's max concurrent streams per user)", + (v) => Number.parseInt(v, 10), + ) .option( "--root ", "Project root containing server/agents/ (default: cwd)",