From 6bd1b665ca3ef33ae207b78f4220d9ac76708fc6 Mon Sep 17 00:00:00 2001 From: bobbai00 Date: Mon, 16 Nov 2026 01:26:29 -0800 Subject: [PATCH] test(agent-service): evaluate the agent on real prompts The agent's tests use a mock model, so nothing showed whether a change made the agent better or worse. `bun run eval` starts this checkout's agent service against a running Texera and a real model, runs six cases a few times each, and prints how many runs passed with the model calls, tokens and bytes per call, which a recorder in front of the LLM endpoint counts. The cases build a workflow, fix a failing operator, explain an execution error, answer from the docs, keep a change the user makes between two prompts, and explain the user's own failed execution. They work on orders made by a seeded generator, so every expected answer is computed. --- agent-service/eval/README.md | 96 ++++++++++ agent-service/eval/cases.ts | 228 ++++++++++++++++++++++++ agent-service/eval/checks.spec.ts | 94 ++++++++++ agent-service/eval/checks.ts | 60 +++++++ agent-service/eval/config.spec.ts | 73 ++++++++ agent-service/eval/config.ts | 72 ++++++++ agent-service/eval/llm-recorder.spec.ts | 84 +++++++++ agent-service/eval/llm-recorder.ts | 79 ++++++++ agent-service/eval/orders.spec.ts | 88 +++++++++ agent-service/eval/orders.ts | 85 +++++++++ agent-service/eval/report.spec.ts | 104 +++++++++++ agent-service/eval/report.ts | 104 +++++++++++ agent-service/eval/run.ts | 168 +++++++++++++++++ agent-service/eval/session.ts | 117 ++++++++++++ agent-service/eval/texera.ts | 130 ++++++++++++++ agent-service/package.json | 5 +- agent-service/tsconfig.json | 2 +- 17 files changed, 1586 insertions(+), 3 deletions(-) create mode 100644 agent-service/eval/README.md create mode 100644 agent-service/eval/cases.ts create mode 100644 agent-service/eval/checks.spec.ts create mode 100644 agent-service/eval/checks.ts create mode 100644 agent-service/eval/config.spec.ts create mode 100644 agent-service/eval/config.ts create mode 100644 agent-service/eval/llm-recorder.spec.ts create mode 100644 agent-service/eval/llm-recorder.ts create mode 100644 agent-service/eval/orders.spec.ts create mode 100644 agent-service/eval/orders.ts create mode 100644 agent-service/eval/report.spec.ts create mode 100644 agent-service/eval/report.ts create mode 100644 agent-service/eval/run.ts create mode 100644 agent-service/eval/session.ts create mode 100644 agent-service/eval/texera.ts diff --git a/agent-service/eval/README.md b/agent-service/eval/README.md new file mode 100644 index 00000000000..dce2624518f --- /dev/null +++ b/agent-service/eval/README.md @@ -0,0 +1,96 @@ + + +# Agent Evaluation + +The agent's unit tests use a mock model, so they cannot tell whether a change makes the agent better or worse. This evaluation runs a fixed set of cases against a real model on a running Texera, and prints how many runs passed and what they cost. + +## The Cases + +| Case | What the user does | The run passes when | +| --- | --- | --- | +| `build-workflow` | Asks for the three categories with the highest total sales | The answer names them, in order | +| `fix-failing-operator` | Asks to fix a filter on a misspelled column | The saved filter uses `amount`, and the answer gives the right count | +| `explain-execution-error` | Asks why a Python operator fails | The answer names the order the error names | +| `docs-question` | Asks which output port modes an operator can have | The answer names the three modes in the docs | +| `user-edit-between-prompts` | Asks for two changes, and adds a Limit in between | The user's Limit is still in the saved workflow | +| `user-execution-failed` | Runs the workflow, then asks why it failed, without running anything | The answer names the order the error names, and the agent ran nothing | + +The cases work on 240 orders that `orders.ts` makes the same way every time, and every expected answer is computed from those rows. The failing Python operator names the order it stops at, an id that is in the execution's error and nowhere in the workflow, so an answer that names it has read the execution. + +## How To Run It + +You need a running Texera and a model its LLM endpoint serves. The evaluation starts the agent service of this checkout on a free port, so it measures your working copy. It reads the variables the agent service reads (`src/config/env.ts`) and passes them on, and puts a recorder in front of `LLM_ENDPOINT` that counts every model call: its size in bytes and the tokens the provider reports. + +With `bin/local-dev.sh` the defaults reach every service; name the model and, where warehouses are on, a warehouse: + +```bash +cd agent-service +EVAL_MODEL=claude-haiku-4.5 EVAL_WAREHOUSE_ID=1 bun run eval +``` + +Behind a gateway such as `bin/single-node.sh`, send everything to the gateway except executions, which go to the computing unit: + +```bash +TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://localhost:8080 \ +WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://localhost:8080 \ +LLM_ENDPOINT=http://localhost:8080 \ +EVAL_FILE_SERVICE_ENDPOINT=http://localhost:8080 \ +WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://localhost:8085 \ +EVAL_MODEL=claude-haiku-4.5 EVAL_PASSWORD=... EVAL_WAREHOUSE_ID=1 \ +bun run eval +``` + +`bin/single-node.sh` does not publish the computing unit's port 8085; publish it with a Compose override first. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `EVAL_MODEL` | (required) | The model, as `/api/models` lists it | +| `EVAL_USERNAME`, `EVAL_PASSWORD` | `texera`, `texera` | The user the cases run as | +| `EVAL_FILE_SERVICE_ENDPOINT` | `http://localhost:9092` | Where the orders dataset is made | +| `EVAL_COMPUTING_UNIT_ID` | `1` | A running computing unit of the user | +| `EVAL_WAREHOUSE_ID` | none | The warehouse executions write to, where warehouses are on | +| `EVAL_REPEAT` | `3` | Runs per case, since a real model does not answer the same way twice | +| `EVAL_CASES` | all | Comma-separated case ids | +| `EVAL_OUT` | none | A file for the full report as JSON | + +Each run gets its own workflow and agent, and both are deleted when the run ends. The orders stay in a dataset named `agent-eval-orders`, which later runs reuse. + +## What It Prints + +One line per run, then a table with the costs averaged per run. This is `main` with Claude Haiku 5.5 on November 16, 2026: + +``` +Case Passed Model calls Input tokens Output tokens KB per call Max KB +------------------------- ------ ----------- ------------ ------------- ----------- ------ +build-workflow 3/3 4.3 56,984 1,402 37.9 39.8 +fix-failing-operator 3/3 2.7 34,732 914 37.7 39.3 +explain-execution-error 3/3 2.0 26,013 854 37.8 38.6 +docs-question 0/3 1.0 12,205 839 35.8 35.8 +user-edit-between-prompts 0/3 4.0 52,409 1,037 37.8 39.8 +user-execution-failed 0/3 1.0 12,693 739 37.0 37.0 +------------------------- ------ ----------- ------------ ------------- ----------- ------ +total 9/18 2.5 32,506 964 37.6 39.8 + +Score: 9 of 18 runs passed (50%). Costs are per run, averaged. +``` + +## Adding A Case + +A case is an entry of `CASES` in `cases.ts`: the workflow it starts from, its prompts, what the user does before or between them, and a check that reads the answers and the saved workflow. Compute what the check expects from the orders rather than writing it down, and make the check pass only on what the agent could not have guessed. diff --git a/agent-service/eval/cases.ts b/agent-service/eval/cases.ts new file mode 100644 index 00000000000..9442e3a3328 --- /dev/null +++ b/agent-service/eval/cases.ts @@ -0,0 +1,228 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { executeOperatorAndFormat } from "../src/agent/tools/workflow-execution-tools"; +import { WorkflowSystemMetadata } from "../src/agent/util/workflow-system-metadata"; +import { WorkflowUtilService } from "../src/agent/util/workflow-utils"; +import { WorkflowState } from "../src/agent/workflow-state"; +import { persistWorkflow, retrieveWorkflow } from "../src/api/workflow-api"; +import type { OperatorPredicate, WorkflowContent } from "../src/types/workflow"; +import { allOf, fail, mentionsAll, mentionsInOrder, mentionsNumber, pass, type Verdict } from "./checks"; +import { countAbove, firstAbove, topCategoriesBySales, type Order } from "./orders"; +import type { Turn } from "./session"; + +/** What one run of a case works with. */ +export interface CaseContext { + token: string; + /** The orders file, as a file scan takes it. */ + ordersPath: string; + orders: Order[]; + /** The workflow made for this run; the agent is delegated to it. */ + workflowId: number; + computingUnitId: number; + warehouseId?: number; +} + +export interface EvalCase { + id: string; + /** The workflow before the first prompt; an empty one when absent. */ + workflow?(ordersPath: string): WorkflowContent; + /** What the user does once the workflow is saved, before the first prompt. */ + before?(ctx: CaseContext): Promise; + prompts(ctx: CaseContext): string[]; + /** What the user does between prompt `index` and the next one. */ + between?(ctx: CaseContext, index: number): Promise; + check(ctx: CaseContext, turns: Turn[]): Promise; +} + +interface OperatorSpec { + type: string; + name: string; + properties: Record; +} + +/** A new operator of the given type, with its schema's defaults under the given properties. */ +function operator(state: WorkflowState, spec: OperatorSpec, id?: string): OperatorPredicate { + const made = new WorkflowUtilService(WorkflowSystemMetadata.getInstance(), state).getNewOperatorPredicate( + spec.type, + spec.name + ); + return { + ...made, + operatorID: id ?? made.operatorID, + operatorProperties: { ...made.operatorProperties, ...spec.properties }, + }; +} + +function link(state: WorkflowState, from: OperatorPredicate, to: OperatorPredicate): void { + state.addLink({ + linkID: state.generateLinkId(), + source: { operatorID: from.operatorID, portID: from.outputPorts[0].portID }, + target: { operatorID: to.operatorID, portID: to.inputPorts[0].portID }, + }); +} + +/** A workflow of operators in a row, each fed by the one before it. */ +function chain(...specs: OperatorSpec[]): WorkflowContent { + const state = new WorkflowState(); + let previous: OperatorPredicate | undefined; + specs.forEach((spec, i) => { + const op = operator(state, spec); + state.addOperator(op, { x: 200 + 250 * i, y: 200 }); + if (previous) link(state, previous, op); + previous = op; + }); + return state.getWorkflowContent(); +} + +const scanOrders = (path: string): OperatorSpec => ({ + type: "CSVFileScan", + name: "Load orders", + properties: { fileName: path, hasHeader: true }, +}); + +/** The amount above which `checkAmounts` fails. */ +const AMOUNT_LIMIT = 250; + +/** + * A Python operator that fails on the first order above the limit, and names that order in its + * error. The order's id is in the execution's error and nowhere in the workflow, so an answer that + * names it has read the execution rather than guessed from the operators. + */ +const checkAmounts: OperatorSpec = { + type: "PythonUDFV2", + name: "Check amounts", + properties: { + code: [ + "from pytexera import *", + "", + "", + "class ProcessTupleOperator(UDFOperatorV2):", + " @overrides", + " def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:", + ` if tuple_["amount"] > ${AMOUNT_LIMIT}:`, + ` raise ValueError(f"order {tuple_['order_id']} is over the limit")`, + " yield tuple_", + ].join("\n"), + }, +}; + +/** The answer names the order the failed execution stopped at. */ +const namesFailingOrder = (ctx: CaseContext, answer: string): Verdict => { + const order = firstAbove(ctx.orders, AMOUNT_LIMIT); + if (!order) return fail(`no order is above ${AMOUNT_LIMIT}, so nothing fails`); + return mentionsNumber(answer, order.order_id).pass + ? pass(`names order ${order.order_id}, the one the execution failed on`) + : fail(`does not name order ${order.order_id}, the one the execution failed on`); +}; + +const saved = async (ctx: CaseContext): Promise => + (await retrieveWorkflow(ctx.token, ctx.workflowId)).content; + +/** The id the user's own operator gets in `user-edit-between-prompts`. */ +const USER_LIMIT_ID = "user-limit"; + +export const CASES: EvalCase[] = [ + { + id: "build-workflow", + prompts: ctx => [ + `Load ${ctx.ordersPath} and tell me the three product categories with the highest total sales, highest first. Build it as a workflow and run it.`, + ], + check: async (ctx, turns) => mentionsInOrder(turns[0].answer, topCategoriesBySales(ctx.orders, 3)), + }, + { + id: "fix-failing-operator", + workflow: path => + chain(scanOrders(path), { + type: "Filter", + name: "Large orders", + properties: { predicates: [{ attribute: "amout", condition: ">", value: "100" }] }, + }), + prompts: () => [ + "The filter in this workflow is broken. Fix it so it keeps the orders with an amount above 100, run it, and tell me how many orders it keeps.", + ], + check: async (ctx, turns) => { + const filter = (await saved(ctx)).operators.find(op => op.operatorType === "Filter"); + const attribute = filter?.operatorProperties.predicates?.[0]?.attribute; + return allOf( + mentionsNumber(turns[0].answer, countAbove(ctx.orders, 100)), + attribute === "amount" ? pass("fixed the filter") : fail(`left the filter on ${JSON.stringify(attribute)}`) + ); + }, + }, + { + id: "explain-execution-error", + workflow: path => chain(scanOrders(path), checkAmounts), + prompts: () => ["Run this workflow and tell me why it fails."], + check: async (ctx, turns) => namesFailingOrder(ctx, turns[0].answer), + }, + { + id: "docs-question", + prompts: () => ["What output port modes can a Texera operator have? Name each of them."], + check: async (_, turns) => mentionsAll(turns[0].answer, ["set snapshot", "delta", "single snapshot"]), + }, + { + id: "user-edit-between-prompts", + prompts: ctx => [ + `Load ${ctx.ordersPath} with a CSV scan and run it.`, + "Add a filter after the CSV scan that keeps the orders with an amount above 100, and run it.", + ], + // The user adds a Limit after the agent's CSV scan and saves, as the canvas does. + between: async ctx => { + const workflow = await retrieveWorkflow(ctx.token, ctx.workflowId); + const state = new WorkflowState(); + state.setWorkflowContent(workflow.content); + const scan = state.getAllOperators().find(op => op.operatorType === "CSVFileScan"); + if (!scan) return; + const limit = operator(state, { type: "Limit", name: "My limit", properties: { limit: 10 } }, USER_LIMIT_ID); + state.addOperator(limit, { x: 450, y: 400 }); + link(state, scan, limit); + await persistWorkflow(ctx.token, ctx.workflowId, workflow.name, state.getWorkflowContent()); + }, + check: async ctx => + (await saved(ctx)).operators.some(op => op.operatorID === USER_LIMIT_ID) + ? pass("kept the user's Limit") + : fail("lost the Limit the user added between the prompts"), + }, + { + id: "user-execution-failed", + workflow: path => chain(scanOrders(path), checkAmounts), + // The user runs the workflow, and it fails. + before: async ctx => { + const state = new WorkflowState(); + state.setWorkflowContent(await saved(ctx)); + const last = state.getAllOperators().find(op => op.operatorType === "PythonUDFV2")!; + await executeOperatorAndFormat( + state, + { + userToken: ctx.token, + workflowId: ctx.workflowId, + computingUnitId: ctx.computingUnitId, + warehouseId: ctx.warehouseId, + }, + last.operatorID + ); + }, + prompts: () => ["My last execution of this workflow failed. Without running anything, tell me why."], + check: async (ctx, turns) => { + const executed = turns[0].steps.some(step => step.toolCalls?.some(call => /execute/i.test(call.toolName))); + return executed ? fail("ran the workflow itself") : namesFailingOrder(ctx, turns[0].answer); + }, + }, +]; diff --git a/agent-service/eval/checks.spec.ts b/agent-service/eval/checks.spec.ts new file mode 100644 index 00000000000..674ef903de8 --- /dev/null +++ b/agent-service/eval/checks.spec.ts @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { describe, expect, test } from "bun:test"; +import { allOf, fail, mentionsAll, mentionsInOrder, mentionsNumber, pass } from "./checks"; + +describe("mentionsInOrder", () => { + test("passes when every name appears in the given order, whatever the case", () => { + expect( + mentionsInOrder("1. technology, 2. Furniture, 3. Office Supplies", ["Technology", "Furniture", "Office Supplies"]) + .pass + ).toBe(true); + }); + + test("fails when the names appear in another order", () => { + const v = mentionsInOrder("Furniture first, then Technology", ["Technology", "Furniture"]); + expect(v).toEqual({ pass: false, reason: "names Technology, Furniture out of order" }); + }); + + test("names the ones that are missing", () => { + expect(mentionsInOrder("Technology only", ["Technology", "Furniture", "Toys"])).toEqual({ + pass: false, + reason: "does not mention Furniture, Toys", + }); + }); + + test("fails on an empty answer", () => { + expect(mentionsInOrder("", ["Technology"]).pass).toBe(false); + }); +}); + +describe("mentionsNumber", () => { + test.each([ + ["It keeps 120 orders.", 120], + ["It keeps **120** orders.", 120], + ["120", 120], + ["kept 1,234 rows", 1234], + ["kept 1234 rows", 1234], + ])("finds the number in %p", (text, value) => { + expect(mentionsNumber(text, value).pass).toBe(true); + }); + + test.each([ + ["It keeps 1120 orders.", 120], + ["It keeps 120.5 orders on average.", 120], + ["between 12,0 and 13", 120], + ["a total of 1,120", 120], + ["no number here", 120], + ["", 0], + ])("does not take part of another number in %p", (text, value) => { + expect(mentionsNumber(text, value).pass).toBe(false); + }); +}); + +describe("mentionsAll", () => { + test("passes when every phrase appears, ignoring case", () => { + expect( + mentionsAll("Set Snapshot, delta updates and SINGLE SNAPSHOT", ["set snapshot", "delta", "single snapshot"]).pass + ).toBe(true); + }); + + test("lists the missing phrases", () => { + expect(mentionsAll("Set Snapshot only", ["set snapshot", "delta"])).toEqual({ + pass: false, + reason: "does not mention delta", + }); + }); +}); + +describe("allOf", () => { + test("passes with every reason when all pass", () => { + expect(allOf(pass("a"), pass("b"))).toEqual({ pass: true, reason: "a; b" }); + }); + + test("gives the first failure", () => { + expect(allOf(pass("a"), fail("b"), fail("c"))).toEqual({ pass: false, reason: "b" }); + }); +}); diff --git a/agent-service/eval/checks.ts b/agent-service/eval/checks.ts new file mode 100644 index 00000000000..96be0b48d71 --- /dev/null +++ b/agent-service/eval/checks.ts @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// What the cases check an answer for. Each check reads plain text and says why it passed or failed. + +export interface Verdict { + pass: boolean; + reason: string; +} + +export const pass = (reason: string): Verdict => ({ pass: true, reason }); +export const fail = (reason: string): Verdict => ({ pass: false, reason }); + +/** Every name appears, and they first appear in the given order. */ +export function mentionsInOrder(text: string, names: string[]): Verdict { + const lower = text.toLowerCase(); + const positions = names.map(name => lower.indexOf(name.toLowerCase())); + const missing = names.filter((_, i) => positions[i] < 0); + if (missing.length > 0) return fail(`does not mention ${missing.join(", ")}`); + const ordered = positions.every((p, i) => i === 0 || p > positions[i - 1]); + return ordered ? pass(`names ${names.join(", ")} in order`) : fail(`names ${names.join(", ")} out of order`); +} + +/** The number appears on its own, written with or without thousands separators. */ +export function mentionsNumber(text: string, value: number): Verdict { + const forms = [String(value), value.toLocaleString("en-US")]; + // Digits and commas only, so the forms need no escaping. A neighbouring digit, or a decimal or + // thousands separator followed by one, means the match is part of a different number. + const found = forms.some(form => new RegExp(`(^|[^\\d.,])${form}(?!\\d|[.,]\\d)`).test(text)); + return found ? pass(`says ${value}`) : fail(`does not say ${value}`); +} + +/** Every phrase appears, ignoring case. */ +export function mentionsAll(text: string, phrases: string[]): Verdict { + const lower = text.toLowerCase(); + const missing = phrases.filter(phrase => !lower.includes(phrase.toLowerCase())); + return missing.length === 0 ? pass(`mentions ${phrases.join(", ")}`) : fail(`does not mention ${missing.join(", ")}`); +} + +/** Every check passes; the first that fails gives the reason. */ +export function allOf(...verdicts: Verdict[]): Verdict { + const failed = verdicts.find(v => !v.pass); + return failed ?? pass(verdicts.map(v => v.reason).join("; ")); +} diff --git a/agent-service/eval/config.spec.ts b/agent-service/eval/config.spec.ts new file mode 100644 index 00000000000..6cd4ab76e5f --- /dev/null +++ b/agent-service/eval/config.spec.ts @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { describe, expect, test } from "bun:test"; +import { readEvalConfig } from "./config"; + +describe("readEvalConfig", () => { + test("needs only a model, and fills in local-dev defaults", () => { + expect(readEvalConfig({ EVAL_MODEL: "claude-haiku-5-5" })).toEqual({ + model: "claude-haiku-5-5", + username: "texera", + password: "texera", + fileServiceEndpoint: "http://localhost:9092", + computingUnitId: 1, + warehouseId: undefined, + repeat: 3, + caseIds: undefined, + out: undefined, + }); + }); + + test("reads every setting, trims the case list and the endpoint's trailing slash", () => { + const config = readEvalConfig({ + EVAL_MODEL: "m", + EVAL_USERNAME: "u", + EVAL_PASSWORD: "p", + EVAL_FILE_SERVICE_ENDPOINT: "http://gateway:8080/", + EVAL_COMPUTING_UNIT_ID: "4", + EVAL_WAREHOUSE_ID: "2", + EVAL_REPEAT: "5", + EVAL_CASES: " build-workflow, ,docs-question ", + EVAL_OUT: "/tmp/eval.json", + }); + expect(config).toEqual({ + model: "m", + username: "u", + password: "p", + fileServiceEndpoint: "http://gateway:8080", + computingUnitId: 4, + warehouseId: 2, + repeat: 5, + caseIds: ["build-workflow", "docs-question"], + out: "/tmp/eval.json", + }); + }); + + test.each([ + [{}], + [{ EVAL_MODEL: "" }], + [{ EVAL_MODEL: "m", EVAL_REPEAT: "0" }], + [{ EVAL_MODEL: "m", EVAL_REPEAT: "two" }], + [{ EVAL_MODEL: "m", EVAL_COMPUTING_UNIT_ID: "-1" }], + [{ EVAL_MODEL: "m", EVAL_FILE_SERVICE_ENDPOINT: "not a url" }], + ])("refuses %p", source => { + expect(() => readEvalConfig(source)).toThrow(); + }); +}); diff --git a/agent-service/eval/config.ts b/agent-service/eval/config.ts new file mode 100644 index 00000000000..c0e966c9a34 --- /dev/null +++ b/agent-service/eval/config.ts @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { z } from "zod"; + +/** + * What the evaluation needs besides the agent service's own settings. Those (the Dashboard + * Service, compiling service, computing unit and LLM endpoints) are read from the environment as + * the agent service reads them, and passed on to the agent service the evaluation starts. + */ +const EvalEnvSchema = z.object({ + /** The model the agents run with, as `/api/models` lists it. */ + EVAL_MODEL: z.string().min(1), + EVAL_USERNAME: z.string().default("texera"), + EVAL_PASSWORD: z.string().default("texera"), + /** Where datasets are created: the file service, or a gateway in front of it. */ + EVAL_FILE_SERVICE_ENDPOINT: z.string().url().default("http://localhost:9092"), + EVAL_COMPUTING_UNIT_ID: z.coerce.number().int().positive().default(1), + /** The warehouse executions write to; leave it unset where warehouses are off. */ + EVAL_WAREHOUSE_ID: z.coerce.number().int().positive().optional(), + /** How many times each case runs: a real model does not answer the same way twice. */ + EVAL_REPEAT: z.coerce.number().int().positive().default(3), + /** Comma-separated case ids to run; all of them when unset. */ + EVAL_CASES: z.string().optional(), + /** Where to write the full report as JSON; not written when unset. */ + EVAL_OUT: z.string().optional(), +}); + +export interface EvalConfig { + model: string; + username: string; + password: string; + fileServiceEndpoint: string; + computingUnitId: number; + warehouseId?: number; + repeat: number; + caseIds?: string[]; + out?: string; +} + +export function readEvalConfig(source: Record = process.env): EvalConfig { + const e = EvalEnvSchema.parse(source); + return { + model: e.EVAL_MODEL, + username: e.EVAL_USERNAME, + password: e.EVAL_PASSWORD, + fileServiceEndpoint: e.EVAL_FILE_SERVICE_ENDPOINT.replace(/\/$/, ""), + computingUnitId: e.EVAL_COMPUTING_UNIT_ID, + warehouseId: e.EVAL_WAREHOUSE_ID, + repeat: e.EVAL_REPEAT, + caseIds: e.EVAL_CASES?.split(",") + .map(id => id.trim()) + .filter(Boolean), + out: e.EVAL_OUT, + }; +} diff --git a/agent-service/eval/llm-recorder.spec.ts b/agent-service/eval/llm-recorder.spec.ts new file mode 100644 index 00000000000..940cdc90241 --- /dev/null +++ b/agent-service/eval/llm-recorder.spec.ts @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { LlmRecorder } from "./llm-recorder"; + +/** A stand-in LLM endpoint that echoes what it got and answers with fixed usage. */ +let upstream: ReturnType; +let seen: Array<{ path: string; auth: string | null; body: string }>; +let recorder: LlmRecorder; +let url: string; + +beforeEach(() => { + seen = []; + upstream = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch: async request => { + const path = new URL(request.url).pathname; + seen.push({ path, auth: request.headers.get("authorization"), body: await request.text() }); + if (path === "/api/chat/completions") + return Response.json({ choices: [], usage: { prompt_tokens: 120, completion_tokens: 8 } }); + if (path === "/api/broken") return new Response("not json", { status: 502 }); + return Response.json({ data: [] }); + }, + }); + recorder = new LlmRecorder(`http://127.0.0.1:${upstream.port}/`); + url = recorder.start(); +}); + +afterEach(() => { + recorder.stop(); + upstream.stop(true); +}); + +describe("LlmRecorder", () => { + test("passes a chat completion through unchanged and records its size and usage", async () => { + const body = JSON.stringify({ model: "m", messages: [{ role: "user", content: "hi" }] }); + const response = await fetch(`${url}/api/chat/completions`, { + method: "POST", + headers: { Authorization: "Bearer user-jwt", "Content-Type": "application/json" }, + body, + }); + expect(await response.json()).toEqual({ choices: [], usage: { prompt_tokens: 120, completion_tokens: 8 } }); + expect(seen).toEqual([{ path: "/api/chat/completions", auth: "Bearer user-jwt", body }]); + expect(recorder.calls).toEqual([ + { requestBytes: new TextEncoder().encode(body).length, status: 200, inputTokens: 120, outputTokens: 8 }, + ]); + }); + + test("passes other requests through without recording them", async () => { + const response = await fetch(`${url}/api/models?x=1`); + expect(response.status).toBe(200); + expect(seen.map(s => s.path)).toEqual(["/api/models"]); + expect(recorder.calls).toEqual([]); + }); + + test("keeps an upstream failure as it is, and records a completion with no usage as zero tokens", async () => { + const broken = await fetch(`${url}/api/broken`); + expect(broken.status).toBe(502); + expect(await broken.text()).toBe("not json"); + + upstream.reload({ fetch: async () => new Response("bad gateway", { status: 502 }) }); + const failed = await fetch(`${url}/api/chat/completions`, { method: "POST", body: "{}" }); + expect(failed.status).toBe(502); + expect(recorder.calls).toEqual([{ requestBytes: 2, status: 502, inputTokens: 0, outputTokens: 0 }]); + }); +}); diff --git a/agent-service/eval/llm-recorder.ts b/agent-service/eval/llm-recorder.ts new file mode 100644 index 00000000000..faced9cafb4 --- /dev/null +++ b/agent-service/eval/llm-recorder.ts @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** One model call, as it went over the wire. */ +export interface ModelCall { + /** Bytes of the request: system prompt, tool definitions and messages, as sent. */ + requestBytes: number; + status: number; + inputTokens: number; + outputTokens: number; +} + +/** + * Sits between the agent service and its LLM endpoint, passes every request through unchanged, + * and records each chat completion: what it sent and the tokens the provider counted. + */ +export class LlmRecorder { + readonly calls: ModelCall[] = []; + private server?: ReturnType; + + constructor(private readonly target: string) {} + + /** Starts listening on a free port and returns the URL to use as the LLM endpoint. */ + start(): string { + this.server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: request => this.forward(request) }); + return `http://127.0.0.1:${this.server.port}`; + } + + stop(): void { + this.server?.stop(true); + } + + private async forward(request: Request): Promise { + const url = new URL(request.url); + const body = await request.arrayBuffer(); + const headers = new Headers(request.headers); + headers.delete("host"); + const response = await fetch(`${this.target.replace(/\/$/, "")}${url.pathname}${url.search}`, { + method: request.method, + headers, + body: body.byteLength > 0 ? body : undefined, + }); + const text = await response.text(); + if (url.pathname.endsWith("/chat/completions")) { + const usage = parseUsage(text); + this.calls.push({ requestBytes: body.byteLength, status: response.status, ...usage }); + } + // The body is passed on already decoded, so the encoding headers no longer describe it. + const out = new Headers(response.headers); + out.delete("content-encoding"); + out.delete("content-length"); + return new Response(text, { status: response.status, headers: out }); + } +} + +function parseUsage(text: string): { inputTokens: number; outputTokens: number } { + try { + const usage = JSON.parse(text)?.usage; + return { inputTokens: usage?.prompt_tokens ?? 0, outputTokens: usage?.completion_tokens ?? 0 }; + } catch { + return { inputTokens: 0, outputTokens: 0 }; + } +} diff --git a/agent-service/eval/orders.spec.ts b/agent-service/eval/orders.spec.ts new file mode 100644 index 00000000000..9ebc63b1eca --- /dev/null +++ b/agent-service/eval/orders.spec.ts @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { describe, expect, test } from "bun:test"; +import { + CATEGORIES, + countAbove, + firstAbove, + generateOrders, + ordersCsv, + topCategoriesBySales, + type Order, +} from "./orders"; + +describe("generateOrders", () => { + test("makes the same rows on every call", () => { + expect(generateOrders()).toEqual(generateOrders()); + }); + + test("makes 240 orders by default, every category alike, with positive amounts", () => { + const orders = generateOrders(); + expect(orders.length).toBe(240); + for (const category of CATEGORIES) expect(orders.filter(o => o.category === category).length).toBe(40); + expect(orders.every(o => o.amount > 0)).toBe(true); + expect(new Set(orders.map(o => o.order_id)).size).toBe(240); + }); + + test("makes no rows when asked for none", () => { + expect(generateOrders(0)).toEqual([]); + }); +}); + +describe("the expected answers", () => { + const orders = generateOrders(); + + test("rank the categories by their weights, far enough apart that rounding cannot swap them", () => { + expect(topCategoriesBySales(orders, 3)).toEqual(["Technology", "Furniture", "Office Supplies"]); + }); + + test("count only the orders strictly above the threshold", () => { + const few: Order[] = [ + { order_id: 1, order_date: "2024-01-01", category: "Toys", region: "East", amount: 100 }, + { order_id: 2, order_date: "2024-01-01", category: "Toys", region: "East", amount: 100.01 }, + ]; + expect(countAbove(few, 100)).toBe(1); + const above = countAbove(orders, 100); + expect(above).toBeGreaterThan(0); + expect(above).toBeLessThan(orders.length); + }); + + test("find the first order above a threshold, in file order", () => { + // The order the cases' failing Python operator stops at: its error names this id. + expect(firstAbove(orders, 250)?.order_id).toBe(100115); + expect(orders.filter(o => o.order_id < 100115).every(o => o.amount <= 250)).toBe(true); + expect(firstAbove(orders, 10_000)).toBeUndefined(); + }); + + test("handle no orders", () => { + expect(topCategoriesBySales([], 3)).toEqual([]); + expect(countAbove([], 100)).toBe(0); + expect(firstAbove([], 100)).toBeUndefined(); + }); +}); + +describe("ordersCsv", () => { + test("writes a header, one line per order with two decimals, and a final newline", () => { + const csv = ordersCsv([ + { order_id: 7, order_date: "2024-02-03", category: "Office Supplies", region: "West", amount: 12.5 }, + ]); + expect(csv).toBe("order_id,order_date,category,region,amount\n7,2024-02-03,Office Supplies,West,12.50\n"); + }); +}); diff --git a/agent-service/eval/orders.ts b/agent-service/eval/orders.ts new file mode 100644 index 00000000000..b2be99f1e6a --- /dev/null +++ b/agent-service/eval/orders.ts @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The data the cases work on: a table of orders made by a seeded generator, so every run uploads + * the same file and every expected answer is computed from the same rows rather than written down. + */ + +export const CATEGORIES = ["Technology", "Furniture", "Office Supplies", "Apparel", "Grocery", "Toys"] as const; +const REGIONS = ["East", "West", "South", "Central"] as const; +/** Each category's amounts are scaled by its weight, which keeps the category totals far apart. */ +const WEIGHTS = [9, 7, 5, 4, 3, 2]; + +export interface Order { + order_id: number; + order_date: string; + category: string; + region: string; + amount: number; +} + +/** A linear congruential generator: the same seed gives the same numbers on every machine. */ +function seeded(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 2 ** 32; + }; +} + +export function generateOrders(count = 240): Order[] { + const next = seeded(9454); + const start = Date.UTC(2024, 0, 1); + return Array.from({ length: count }, (_, i) => { + const c = i % CATEGORIES.length; + return { + order_id: 100001 + i, + order_date: new Date(start + (i % 365) * 86_400_000).toISOString().slice(0, 10), + category: CATEGORIES[c], + region: REGIONS[Math.floor(next() * REGIONS.length)], + amount: Math.round((5 + next() * 25) * WEIGHTS[c] * 100) / 100, + }; + }); +} + +export function ordersCsv(orders: Order[]): string { + const rows = orders.map(o => [o.order_id, o.order_date, o.category, o.region, o.amount.toFixed(2)].join(",")); + return ["order_id,order_date,category,region,amount", ...rows].join("\n") + "\n"; +} + +/** The categories with the highest total amount, highest first. */ +export function topCategoriesBySales(orders: Order[], n: number): string[] { + const totals = new Map(); + for (const o of orders) totals.set(o.category, (totals.get(o.category) ?? 0) + o.amount); + return [...totals.entries()] + .sort((a, b) => b[1] - a[1]) + .slice(0, n) + .map(([category]) => category); +} + +/** The first order, in file order, whose amount is strictly above the threshold. */ +export function firstAbove(orders: Order[], threshold: number): Order | undefined { + return orders.find(o => o.amount > threshold); +} + +/** How many orders have an amount strictly above the threshold. */ +export function countAbove(orders: Order[], threshold: number): number { + return orders.filter(o => o.amount > threshold).length; +} diff --git a/agent-service/eval/report.spec.ts b/agent-service/eval/report.spec.ts new file mode 100644 index 00000000000..dd879efa21f --- /dev/null +++ b/agent-service/eval/report.spec.ts @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { describe, expect, test } from "bun:test"; +import { fail, pass } from "./checks"; +import { formatReport, summarize, type RunRecord } from "./report"; + +const call = (requestBytes: number, inputTokens: number, outputTokens: number) => ({ + requestBytes, + status: 200, + inputTokens, + outputTokens, +}); + +const records: RunRecord[] = [ + { caseId: "a", run: 1, verdict: pass("ok"), calls: [call(10_000, 100, 10), call(20_000, 200, 20)], seconds: 3 }, + { caseId: "a", run: 2, verdict: fail("no"), calls: [call(30_000, 300, 30)], seconds: 2 }, + { caseId: "b", run: 1, verdict: pass("ok"), calls: [], seconds: 1 }, +]; + +describe("summarize", () => { + test("counts passes and averages the costs per run, by case and in total", () => { + const { cases, total } = summarize(records); + expect(cases.map(c => c.caseId)).toEqual(["a", "b"]); + expect(cases[0]).toEqual({ + caseId: "a", + passed: 1, + runs: 2, + modelCalls: 1.5, + inputTokens: 300, + outputTokens: 30, + kbPerCall: 20, + maxKbPerCall: 30, + }); + // A run without model calls costs nothing and divides nothing by zero. + expect(cases[1]).toEqual({ + caseId: "b", + passed: 1, + runs: 1, + modelCalls: 0, + inputTokens: 0, + outputTokens: 0, + kbPerCall: 0, + maxKbPerCall: 0, + }); + expect(total).toMatchObject({ passed: 2, runs: 3, modelCalls: 1, inputTokens: 200, kbPerCall: 20 }); + }); + + test("has nothing to say about no runs", () => { + expect(summarize([])).toEqual({ + cases: [], + total: { + caseId: "total", + passed: 0, + runs: 0, + modelCalls: 0, + inputTokens: 0, + outputTokens: 0, + kbPerCall: 0, + maxKbPerCall: 0, + }, + }); + }); +}); + +describe("formatReport", () => { + test("prints one aligned row per case, the total, and the score", () => { + const lines = formatReport(summarize(records)).split("\n"); + expect(lines[0]).toMatch(/^Case\s+Passed\s+Model calls\s+Input tokens\s+Output tokens\s+KB per call\s+Max KB$/); + expect(lines[2]).toMatch(/^a\s+1\/2\s+1\.5\s+300\s+30\s+20\.0\s+30\.0$/); + expect(lines[5]).toMatch(/^total\s+2\/3\s+1\.0\s+200\s+20\s+20\.0\s+30\.0$/); + expect(lines.at(-1)).toBe("Score: 2 of 3 runs passed (67%). Costs are per run, averaged."); + expect( + new Set( + lines + .slice(0, 6) + .filter(l => l.length > 0) + .map(l => l.length) + ).size + ).toBe(1); + }); + + test("scores no runs as zero", () => { + expect(formatReport(summarize([])).split("\n").at(-1)).toBe( + "Score: 0 of 0 runs passed (0%). Costs are per run, averaged." + ); + }); +}); diff --git a/agent-service/eval/report.ts b/agent-service/eval/report.ts new file mode 100644 index 00000000000..70c039b1943 --- /dev/null +++ b/agent-service/eval/report.ts @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { Verdict } from "./checks"; +import type { ModelCall } from "./llm-recorder"; + +/** One run of one case. */ +export interface RunRecord { + caseId: string; + run: number; + verdict: Verdict; + /** The model calls the agent made during the run. */ + calls: ModelCall[]; + seconds: number; +} + +/** A case's runs, with the costs averaged per run. */ +export interface CaseSummary { + caseId: string; + passed: number; + runs: number; + modelCalls: number; + inputTokens: number; + outputTokens: number; + /** The average size of one request, in KB (1,000 bytes). */ + kbPerCall: number; + /** The largest request of any run, in KB. */ + maxKbPerCall: number; +} + +function summarizeRuns(caseId: string, runs: RunRecord[]): CaseSummary { + const calls = runs.flatMap(r => r.calls); + const perRun = (n: number) => (runs.length === 0 ? 0 : n / runs.length); + const sum = (pick: (c: ModelCall) => number) => calls.reduce((n, c) => n + pick(c), 0); + return { + caseId, + passed: runs.filter(r => r.verdict.pass).length, + runs: runs.length, + modelCalls: perRun(calls.length), + inputTokens: perRun(sum(c => c.inputTokens)), + outputTokens: perRun(sum(c => c.outputTokens)), + kbPerCall: calls.length === 0 ? 0 : sum(c => c.requestBytes) / calls.length / 1000, + maxKbPerCall: calls.reduce((max, c) => Math.max(max, c.requestBytes / 1000), 0), + }; +} + +/** One summary per case, in the order the cases first ran, and one for all runs together. */ +export function summarize(records: RunRecord[]): { cases: CaseSummary[]; total: CaseSummary } { + const ids = [...new Set(records.map(r => r.caseId))]; + return { + cases: ids.map(id => + summarizeRuns( + id, + records.filter(r => r.caseId === id) + ) + ), + total: summarizeRuns("total", records), + }; +} + +export function formatReport(summary: { cases: CaseSummary[]; total: CaseSummary }): string { + const header = ["Case", "Passed", "Model calls", "Input tokens", "Output tokens", "KB per call", "Max KB"]; + const row = (s: CaseSummary) => [ + s.caseId, + `${s.passed}/${s.runs}`, + s.modelCalls.toFixed(1), + Math.round(s.inputTokens).toLocaleString("en-US"), + Math.round(s.outputTokens).toLocaleString("en-US"), + s.kbPerCall.toFixed(1), + s.maxKbPerCall.toFixed(1), + ]; + const rows = [header, ...summary.cases.map(row), row(summary.total)]; + const widths = header.map((_, i) => Math.max(...rows.map(r => r[i].length))); + const line = (cells: string[]) => + cells.map((cell, i) => (i === 0 ? cell.padEnd(widths[i]) : cell.padStart(widths[i]))).join(" "); + const rule = widths.map(w => "-".repeat(w)).join(" "); + const { passed, runs } = summary.total; + const score = runs === 0 ? 0 : Math.round((100 * passed) / runs); + return [ + line(header), + rule, + ...summary.cases.map(s => line(row(s))), + rule, + line(row(summary.total)), + "", + `Score: ${passed} of ${runs} runs passed (${score}%). Costs are per run, averaged.`, + ].join("\n"); +} diff --git a/agent-service/eval/run.ts b/agent-service/eval/run.ts new file mode 100644 index 00000000000..c3b85487e74 --- /dev/null +++ b/agent-service/eval/run.ts @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Runs the cases against a real model and prints how the agent did. See eval/README.md. + +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { WorkflowSystemMetadata } from "../src/agent/util/workflow-system-metadata"; +import { getBackendConfig } from "../src/api/backend-api"; +import { CASES, type CaseContext, type EvalCase } from "./cases"; +import { fail, type Verdict } from "./checks"; +import { readEvalConfig, type EvalConfig } from "./config"; +import { LlmRecorder } from "./llm-recorder"; +import { generateOrders, ordersCsv } from "./orders"; +import { formatReport, summarize, type RunRecord } from "./report"; +import { AgentSession, type Turn } from "./session"; +import { createWorkflow, deleteWorkflows, EMPTY_WORKFLOW, ensureDataset, login } from "./texera"; + +/** The agent saves its workflow 500 ms after a change; give that save time before reading. */ +const SAVE_SETTLE_MS = 2000; + +interface AgentService { + url: string; + log: string; + stop(): void; +} + +/** Starts this checkout's agent service on a free port, with its model calls sent to `llmEndpoint`. */ +async function startAgentService(llmEndpoint: string): Promise { + const probe = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + const log = join(tmpdir(), `texera-agent-eval-${port}.log`); + const child = Bun.spawn(["bun", "run", "src/server.ts"], { + cwd: join(import.meta.dir, ".."), + env: { ...process.env, PORT: String(port), LLM_ENDPOINT: llmEndpoint }, + stdout: Bun.file(log), + stderr: Bun.file(log), + }); + const url = `http://127.0.0.1:${port}`; + for (let tries = 0; tries < 60; tries++) { + const ok = await fetch(`${url}/api/healthcheck`).then( + r => r.ok, + () => false + ); + if (ok) return { url, log, stop: () => child.kill() }; + await Bun.sleep(500); + } + child.kill(); + throw new Error(`the agent service did not start; see ${log}`); +} + +async function runCase( + evalCase: EvalCase, + run: number, + shared: Omit, + config: EvalConfig, + agentUrl: string, + recorder: LlmRecorder +): Promise { + const started = performance.now(); + const firstCall = recorder.calls.length; + const content = evalCase.workflow?.(shared.ordersPath) ?? EMPTY_WORKFLOW; + const workflowId = await createWorkflow(shared.token, `agent-eval ${evalCase.id} ${run}`, content); + const ctx: CaseContext = { ...shared, workflowId }; + let session: AgentSession | undefined; + let verdict: Verdict; + try { + await evalCase.before?.(ctx); + session = await AgentSession.start(agentUrl, shared.token, { + name: `eval ${evalCase.id} ${run}`, + model: config.model, + workflowId, + computingUnitId: config.computingUnitId, + }); + const prompts = evalCase.prompts(ctx); + const turns: Turn[] = []; + for (const [i, prompt] of prompts.entries()) { + turns.push(await session.prompt(prompt, config.warehouseId)); + await Bun.sleep(SAVE_SETTLE_MS); + if (i < prompts.length - 1) await evalCase.between?.(ctx, i); + } + verdict = await evalCase.check(ctx, turns); + } catch (error) { + verdict = fail(`the run broke: ${error instanceof Error ? error.message : String(error)}`); + } finally { + // The agent goes first, so its pending save cannot land on the deleted workflow. + await session?.close(); + await deleteWorkflows(shared.token, [workflowId]); + } + return { + caseId: evalCase.id, + run, + verdict, + calls: recorder.calls.slice(firstCall), + seconds: (performance.now() - started) / 1000, + }; +} + +async function main(): Promise { + const config = readEvalConfig(); + const unknown = (config.caseIds ?? []).filter(id => !CASES.some(c => c.id === id)); + if (unknown.length > 0) throw new Error(`no such case: ${unknown.join(", ")}`); + const cases = config.caseIds ? CASES.filter(c => config.caseIds!.includes(c.id)) : CASES; + + const recorder = new LlmRecorder(getBackendConfig().modelsEndpoint); + const service = await startAgentService(recorder.start()); + console.log(`Agent service of this checkout at ${service.url} (log: ${service.log}), model ${config.model}`); + try { + const token = await login(config.username, config.password); + await WorkflowSystemMetadata.getInstance().initializeFromBackend(); + const orders = generateOrders(); + const ordersPath = await ensureDataset( + token, + config.fileServiceEndpoint, + "agent-eval-orders", + "orders.csv", + ordersCsv(orders) + ); + const shared = { + token, + ordersPath, + orders, + computingUnitId: config.computingUnitId, + warehouseId: config.warehouseId, + }; + + const records: RunRecord[] = []; + for (const evalCase of cases) { + for (let run = 1; run <= config.repeat; run++) { + const record = await runCase(evalCase, run, shared, config, service.url, recorder); + records.push(record); + console.log( + `${record.verdict.pass ? "PASS" : "FAIL"} ${evalCase.id} ${run}/${config.repeat}: ${record.verdict.reason} ` + + `(${record.calls.length} model calls, ${record.seconds.toFixed(0)} s)` + ); + } + } + + const summary = summarize(records); + console.log(`\n${formatReport(summary)}`); + if (config.out) { + await Bun.write(config.out, JSON.stringify({ model: config.model, summary, records }, null, 2)); + console.log(`Wrote ${config.out}`); + } + } finally { + service.stop(); + recorder.stop(); + } +} + +await main(); diff --git a/agent-service/eval/session.ts b/agent-service/eval/session.ts new file mode 100644 index 00000000000..30c4c219a15 --- /dev/null +++ b/agent-service/eval/session.ts @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createAuthHeaders } from "../src/api/auth-api"; +import { AgentState, type ReActStep } from "../src/types/agent"; +import { WsClientPromptCommand } from "../src/types/ws/client"; +import type { WsServerEvent } from "../src/types/ws/server"; + +/** One prompt and what the agent did with it. */ +export interface Turn { + prompt: string; + /** The turn's steps, the user's first; each step once, as it ended. */ + steps: ReActStep[]; + /** The text of the turn's last step. */ + answer: string; +} + +const TURN_TIMEOUT_MS = 10 * 60_000; + +/** + * One agent, driven the way the chat drives it: created over HTTP, then prompted over its WebSocket + * with the service's own command frames. + */ +export class AgentSession { + private readonly events: WsServerEvent[] = []; + private listener: (event: WsServerEvent) => void = () => {}; + + private constructor( + private readonly agentUrl: string, + private readonly agentId: string, + private readonly socket: WebSocket + ) { + socket.onmessage = message => { + const event = JSON.parse(String(message.data)) as WsServerEvent; + this.events.push(event); + this.listener(event); + }; + } + + static async start( + agentUrl: string, + token: string, + agent: { name: string; model: string; workflowId: number; computingUnitId: number } + ): Promise { + const response = await fetch(`${agentUrl}/api/agents`, { + method: "POST", + headers: createAuthHeaders(token), + body: JSON.stringify({ + name: agent.name, + modelType: agent.model, + workflowId: agent.workflowId, + computingUnitId: agent.computingUnitId, + }), + }); + const created = (await response.json()) as { id?: string }; + if (!response.ok || !created.id) { + throw new Error(`creating an agent failed: ${response.status} ${JSON.stringify(created)}`); + } + const socket = new WebSocket(`${agentUrl.replace(/^http/, "ws")}/api/agents/${created.id}/react`); + await new Promise((resolve, reject) => { + socket.onopen = () => resolve(); + socket.onerror = () => reject(new Error(`could not open the WebSocket of ${created.id}`)); + }); + return new AgentSession(agentUrl, created.id, socket); + } + + /** Sends a prompt and resolves once the agent is available again. */ + async prompt(content: string, warehouseId?: number): Promise { + const start = this.events.length; + const finished = new Promise((resolve, reject) => { + let generating = false; + const timer = setTimeout(() => reject(new Error("the turn did not end in time")), TURN_TIMEOUT_MS); + this.listener = event => { + if (event.type === "WsServerStatusEvent" && event.state === AgentState.GENERATING) generating = true; + if (event.type === "WsServerStatusEvent" && event.state === AgentState.AVAILABLE && generating) { + clearTimeout(timer); + resolve(); + } + if (event.type === "WsServerErrorEvent") { + clearTimeout(timer); + reject(new Error(event.error)); + } + }; + }); + this.socket.send(JSON.stringify(new WsClientPromptCommand(content, "chat", warehouseId))); + await finished; + + // A step is sent again when it is marked as the turn's last, so keep the latest copy of each. + const byId = new Map(); + for (const event of this.events.slice(start)) { + if (event.type === "WsServerStepEvent") byId.set(event.step.id, event.step); + } + const steps = [...byId.values()]; + return { prompt: content, steps, answer: steps.at(-1)?.content ?? "" }; + } + + async close(): Promise { + this.socket.close(); + await fetch(`${this.agentUrl}/api/agents/${this.agentId}`, { method: "DELETE" }); + } +} diff --git a/agent-service/eval/texera.ts b/agent-service/eval/texera.ts new file mode 100644 index 00000000000..983ea855ec7 --- /dev/null +++ b/agent-service/eval/texera.ts @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// The calls the evaluation makes to Texera as the user: sign in, make and remove the workflows the +// cases run on, and put the orders file in a dataset. Workflows are read and saved with the agent +// service's own client (`src/api/workflow-api.ts`), so the evaluation sees them as the agent does. + +import { getBackendConfig } from "../src/api/backend-api"; +import { createAuthHeaders, extractUserFromToken } from "../src/api/auth-api"; +import type { WorkflowContent } from "../src/types/workflow"; + +async function call(token: string | undefined, url: string, init: RequestInit = {}): Promise { + const headers = { ...(token ? createAuthHeaders(token) : { "Content-Type": "application/json" }), ...init.headers }; + const response = await fetch(url, { ...init, headers }); + if (!response.ok) { + throw new Error(`${init.method ?? "GET"} ${url} failed: ${response.status} ${await response.text()}`); + } + return response; +} + +const dashboard = () => `${getBackendConfig().apiEndpoint}/api`; + +export async function login(username: string, password: string): Promise { + const response = await call(undefined, `${dashboard()}/auth/login`, { + method: "POST", + body: JSON.stringify({ username, password }), + }); + const { accessToken } = (await response.json()) as { accessToken?: string }; + if (!accessToken) throw new Error(`login as ${username} returned no token`); + return accessToken; +} + +export const EMPTY_WORKFLOW: WorkflowContent = { + operators: [], + operatorPositions: {}, + links: [], + commentBoxes: [], + settings: { dataTransferBatchSize: 400 } as WorkflowContent["settings"], +}; + +export async function createWorkflow(token: string, name: string, content: WorkflowContent): Promise { + const response = await call(token, `${dashboard()}/workflow/create`, { + method: "POST", + body: JSON.stringify({ name, content: JSON.stringify(content) }), + }); + const created = (await response.json()) as { workflow: { wid: number } }; + return created.workflow.wid; +} + +export async function deleteWorkflows(token: string, wids: number[]): Promise { + if (wids.length === 0) return; + await call(token, `${dashboard()}/workflow/delete`, { method: "POST", body: JSON.stringify({ wids }) }); +} + +/** + * Makes sure the user owns a dataset holding the given file, and returns the file's path in the + * form the file scan operators take: `////`. A dataset of that + * name the user already owns is reused as it is. + */ +export async function ensureDataset( + token: string, + fileService: string, + datasetName: string, + fileName: string, + content: string +): Promise { + const owner = extractUserFromToken(token).email; + const api = `${fileService}/api/dataset`; + const listed = (await (await call(token, `${api}/list`)).json()) as Array<{ + dataset: { did: number; name: string }; + isOwner: boolean; + }>; + let did = listed.find(d => d.isOwner && d.dataset.name === datasetName)?.dataset.did; + + if (did === undefined) { + const created = (await ( + await call(token, `${api}/create`, { + method: "POST", + body: JSON.stringify({ + datasetName, + datasetDescription: "Orders made by the agent evaluation (agent-service/eval).", + isDatasetPublic: false, + }), + }) + ).json()) as { dataset: { did: number } }; + did = created.dataset.did; + + const bytes = new TextEncoder().encode(content); + const target = `ownerEmail=${encodeURIComponent(owner)}&datasetName=${encodeURIComponent(datasetName)}&filePath=${encodeURIComponent(fileName)}`; + await call( + token, + `${api}/multipart-upload?type=init&${target}&fileSizeBytes=${bytes.length}&partSizeBytes=${bytes.length}`, + { + method: "POST", + } + ); + await call(token, `${api}/multipart-upload/part?${target}&partNumber=1`, { + method: "POST", + headers: { "Content-Type": "application/octet-stream" }, + body: bytes, + }); + await call(token, `${api}/multipart-upload?type=finish&${target}`, { method: "POST" }); + await call(token, `${api}/${did}/version/create`, { + method: "POST", + headers: { "Content-Type": "text/plain" }, + body: "", + }); + } + + const latest = (await (await call(token, `${api}/${did}/version/latest`)).json()) as { + datasetVersion: { name: string }; + }; + return `/${owner}/${datasetName}/${latest.datasetVersion.name}/${fileName}`; +} diff --git a/agent-service/package.json b/agent-service/package.json index 9e9eb874b34..ac142a8ee4d 100644 --- a/agent-service/package.json +++ b/agent-service/package.json @@ -10,9 +10,10 @@ "start": "bun run src/server.ts", "start:node": "npx tsx src/server.ts", "test": "bun test", + "eval": "bun run eval/run.ts", "typecheck": "tsc --noEmit", - "format": "prettier --write \"src/**/*.{ts,tsx,json}\"", - "format:check": "prettier --check \"src/**/*.{ts,tsx,json}\"" + "format": "prettier --write \"src/**/*.{ts,tsx,json}\" \"eval/**/*.ts\"", + "format:check": "prettier --check \"src/**/*.{ts,tsx,json}\" \"eval/**/*.ts\"" }, "dependencies": { "@ai-sdk/openai": "4.0.118", diff --git a/agent-service/tsconfig.json b/agent-service/tsconfig.json index 5f0da1e5bee..8327078fb11 100644 --- a/agent-service/tsconfig.json +++ b/agent-service/tsconfig.json @@ -19,5 +19,5 @@ "noEmit": true }, - "include": ["src/**/*"] + "include": ["src/**/*", "eval/**/*"] }