From 4ab695bbd6d825dfddb829ea57c81cd2e32d0103 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:49:35 +0000 Subject: [PATCH 1/4] refactor(sdk)!: Remove deprecated tools and configurations --- .../durable-eval-webhook/scenario.ts | 7 +- js/package.json | 1 - js/src/durable-eval.test.ts | 71 +-- js/src/durable-eval.ts | 208 ++----- js/src/eval-parameters.ts | 178 ------ js/src/exports.ts | 1 - js/src/framework.test.ts | 393 ++++++------- js/src/framework.ts | 170 +----- js/src/framework2.ts | 531 +++-------------- js/src/logger.test.ts | 175 +----- js/src/logger.ts | 547 ++---------------- js/src/parameters.test.ts | 295 ---------- js/src/prompt-cache/cache-config.ts | 44 +- js/src/prompt-cache/cache-mode.test.ts | 91 +-- js/src/prompt-cache/parameters-cache.test.ts | 69 --- js/src/prompt-cache/parameters-cache.ts | 123 ---- js/src/prompt-schemas.ts | 54 -- js/src/prompt-types.test.ts | 22 +- js/src/public-types.test.ts | 56 +- js/src/reporters/types.ts | 2 +- js/src/sdk-contracts.test.ts | 64 +- js/src/sdk-schemas.ts | 5 - js/src/sdk-types.ts | 24 - pnpm-lock.yaml | 3 - 24 files changed, 501 insertions(+), 2633 deletions(-) delete mode 100644 js/src/eval-parameters.ts delete mode 100644 js/src/parameters.test.ts delete mode 100644 js/src/prompt-cache/parameters-cache.test.ts delete mode 100644 js/src/prompt-cache/parameters-cache.ts delete mode 100644 js/src/prompt-schemas.ts diff --git a/e2e/scenarios/durable-eval-webhook/scenario.ts b/e2e/scenarios/durable-eval-webhook/scenario.ts index 6a1b54899..5e096aec4 100644 --- a/e2e/scenarios/durable-eval-webhook/scenario.ts +++ b/e2e/scenarios/durable-eval-webhook/scenario.ts @@ -1,3 +1,4 @@ +import assert from "node:assert/strict"; import { BatchScorer, BatchTask, @@ -23,11 +24,13 @@ async function main() { number, number, number, - { testRunId: string; kind: string }, - Record + { testRunId: string; kind: string } >({ batchSize: 2, async submit(items) { + for (const item of items) { + assert.equal("parameters" in item, false); + } const id = `task-${jobs.size + 1}`; jobs.set(id, items); return { id }; diff --git a/js/package.json b/js/package.json index 27b87cfa5..940273dbd 100644 --- a/js/package.json +++ b/js/package.json @@ -175,7 +175,6 @@ "@vercel/functions": "^1.0.2", "acorn": "^8.16.0", "acorn-import-attributes": "^1.9.5", - "ajv": "^8.20.0", "astring": "^1.9.0", "cjs-module-lexer": "^2.2.0", "dotenv": "^16.4.5", diff --git a/js/src/durable-eval.test.ts b/js/src/durable-eval.test.ts index f814e638c..f6a05813e 100644 --- a/js/src/durable-eval.test.ts +++ b/js/src/durable-eval.test.ts @@ -193,20 +193,9 @@ describe("defineDurableEval", () => { test("merges batch task metadata with case metadata", async () => { type Metadata = { fromCase?: boolean; fromTask?: boolean }; - let taskItems: DurableBatchTaskItem< - number, - void, - Metadata, - Record - >[] = []; + let taskItems: DurableBatchTaskItem[] = []; let scorerMetadata: Metadata | undefined; - const task = new BatchTask< - number, - number, - void, - Metadata, - Record - >({ + const task = new BatchTask({ async submit(items) { taskItems = items; return null; @@ -259,14 +248,7 @@ describe("defineDurableEval", () => { return { value, created: true }; }, }; - const task = new BatchTask< - number, - number, - void, - void, - Record, - { id: string } - >({ + const task = new BatchTask({ async submit() { return { id: "provider-job" }; }, @@ -308,6 +290,7 @@ describe("defineDurableEval", () => { caseIds: ["one:trial:0", "two:trial:0"], }); expect(run).not.toHaveProperty("schemaVersion"); + expect(run).not.toHaveProperty("parameters"); expect(run).not.toHaveProperty("projectName"); expect(run).not.toHaveProperty("evalName"); expect(run).not.toHaveProperty("cases"); @@ -325,7 +308,7 @@ describe("defineDurableEval", () => { test("polls each existing task and scorer sub-batch once", async () => { const taskJobs = new Map< string, - DurableBatchTaskItem>[] + DurableBatchTaskItem[] >(); const scoreJobs = new Map< string, @@ -334,14 +317,7 @@ describe("defineDurableEval", () => { const taskPoll = vi.fn(async () => ({ status: "complete" as const })); const scorePoll = vi.fn(async () => ({ status: "complete" as const })); - const task = new BatchTask< - number, - number, - number, - void, - Record, - { id: string } - >({ + const task = new BatchTask({ batchSize: 2, async submit(items) { const id = `task-${taskJobs.size + 1}`; @@ -445,7 +421,7 @@ describe("defineDurableEval", () => { const store = new DurableEvalMemoryStore(); const taskJobs = new Map< string, - DurableBatchTaskItem>[] + DurableBatchTaskItem[] >(); const scoreJobs = new Map< string, @@ -456,14 +432,7 @@ describe("defineDurableEval", () => { const taskBatchesCollecting = new Promise((resolve) => { releaseTaskCollect = resolve; }); - const task = new BatchTask< - number, - number, - number, - void, - Record, - { id: string } - >({ + const task = new BatchTask({ batchSize: 2, async submit(items) { const id = `task-provider-${taskJobs.size + 1}`; @@ -560,25 +529,13 @@ describe("defineDurableEval", () => { }); test("claims downstream work once across concurrent webhook deliveries", async () => { - let taskItems: DurableBatchTaskItem< - number, - number, - void, - Record - >[] = []; + let taskItems: DurableBatchTaskItem[] = []; let collectCount = 0; let releaseCollect!: () => void; const bothCollecting = new Promise((resolve) => { releaseCollect = resolve; }); - const task = new BatchTask< - number, - number, - number, - void, - Record, - { id: string } - >({ + const task = new BatchTask({ async submit(items) { taskItems = items; return { id: "task-provider" }; @@ -654,13 +611,7 @@ describe("defineDurableEval", () => { return { status: "complete" as const }; }, }; - const task = new BatchTask< - number, - number, - number, - void, - Record - >({ + const task = new BatchTask({ async submit(items) { jobs.set("task", items); return { id: "task" }; diff --git a/js/src/durable-eval.ts b/js/src/durable-eval.ts index c27f63d26..ba2b49d0b 100644 --- a/js/src/durable-eval.ts +++ b/js/src/durable-eval.ts @@ -4,11 +4,7 @@ import { SpanTypeAttribute, uint8ArrayToBase64, } from "../util/index"; -import { - type EvalParameters, - type InferParameters, - validateParameters, -} from "./eval-parameters"; + import { _internalInitEvaluatorExperiment, _internalPrepareEvaluatorClassification, @@ -228,7 +224,6 @@ export interface DurableBatchTaskItem< Input, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, > { /** Stable identifier for this case and trial within the durable run. */ id: string; @@ -240,8 +235,6 @@ export interface DurableBatchTaskItem< metadata: Metadata; /** Tags associated with the evaluation case. */ tags: string[] | undefined; - /** Parameters supplied when the durable evaluation started. */ - parameters: InferParameters; /** Zero-based trial index for this case. */ trialIndex: number; } @@ -299,12 +292,11 @@ interface DurableBatchTask< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, SubmissionData extends JsonValue, > { readonly kind: typeof BATCH_TASK_KIND; readonly processor: DurableBatchProcessor< - DurableBatchTaskItem, + DurableBatchTaskItem, DurableBatchTaskResult, SubmissionData >; @@ -336,21 +328,19 @@ export class BatchTask< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, - Parameters extends EvalParameters = EvalParameters, SubmissionData extends JsonValue = JsonValue, > implements DurableBatchTask< Input, Output, Expected, Metadata, - Parameters, SubmissionData > { readonly kind: typeof BATCH_TASK_KIND = BATCH_TASK_KIND; constructor( readonly processor: DurableBatchProcessor< - DurableBatchTaskItem, + DurableBatchTaskItem, DurableBatchTaskResult, SubmissionData >, @@ -394,9 +384,8 @@ type DurableEvaluator< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, - Parameters extends EvalParameters = EvalParameters, > = Omit< - Evaluator, + Evaluator, "task" | "scores" | "timeout" | "signal" | "maxConcurrency" | "update" > & { store: DurableEvalStore; @@ -408,25 +397,15 @@ type DurableEvaluator< datum: EvalCase, ) => string | Promise; task: - | EvalTask - | DurableBatchTask< - Input, - Output, - Expected, - Metadata, - Parameters, - JsonValue - >; + | EvalTask + | DurableBatchTask; scores?: Array< | EvalScorer | DurableBatchScorer >; }; -interface DurableEvalStartOptions< - Parameters extends EvalParameters = EvalParameters, -> { - parameters?: InferParameters; +interface DurableEvalStartOptions { noSendLogs?: boolean; } @@ -460,23 +439,14 @@ interface DurableEvalRuntimeDefinition< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, - Parameters extends EvalParameters = EvalParameters, > { readonly projectName: string; readonly evalName: string; - readonly evaluator: DurableEvaluator< - Input, - Output, - Expected, - Metadata, - Parameters - >; + readonly evaluator: DurableEvaluator; } -interface DurableEvalDefinition { - start( - options?: DurableEvalStartOptions, - ): Promise; +interface DurableEvalDefinition { + start(options?: DurableEvalStartOptions): Promise; status(options: { runId: string }): Promise; poll(options: { runId: string }): Promise; processBatchResult(result: DurableBatchResult): Promise; @@ -527,7 +497,6 @@ type DurableRunState = { runId: string; experimentName: string; noSendLogs: boolean; - parameters: JsonValue; status: "running" | "completed"; summary?: ExperimentSummary; cases: DurableCaseRecord[]; @@ -762,17 +731,15 @@ export function defineDurableEval< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, - Parameters extends EvalParameters = EvalParameters, >( projectName: string, - evaluator: DurableEvaluator, -): DurableEvalDefinition { + evaluator: DurableEvaluator, +): DurableEvalDefinition { const definition: DurableEvalRuntimeDefinition< Input, Output, Expected, - Metadata, - Parameters + Metadata > = { projectName, evalName: evaluator.experimentName ?? projectName, @@ -792,25 +759,14 @@ async function startDurableEval< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, - options: DurableEvalStartOptions, + definition: DurableEvalRuntimeDefinition, + options: DurableEvalStartOptions, ): Promise { const store = definition.evaluator.store; const runId = newId(); const key = runKey(definition.projectName, definition.evalName, runId); const { data } = callEvaluatorData(definition.evaluator.data); - const parameters = await validateParameters( - options.parameters ?? {}, - definition.evaluator.parameters, - ); const experimentName = definition.evaluator.experimentName ?? `${definition.evalName}-${runId}`; const experiment = await _internalInitEvaluatorExperiment( @@ -819,8 +775,7 @@ async function startDurableEval< Input, Output, Expected, - Metadata, - Parameters + Metadata >, data, { @@ -840,20 +795,13 @@ async function startDurableEval< projectName: definition.projectName, evalName: definition.evalName, data, - } as unknown as EvaluatorDef< - Input, - Output, - Expected, - Metadata, - Parameters - >, + } as unknown as EvaluatorDef, { start: () => undefined, stop: () => undefined, increment: () => undefined, }, undefined, - parameters, true, true, ); @@ -861,7 +809,6 @@ async function startDurableEval< runId, experimentName, noSendLogs: options.noSendLogs ?? false, - parameters: assertJsonValue(parameters, "eval parameters"), status: "completed", summary: result.summary, cases: [], @@ -875,7 +822,6 @@ async function startDurableEval< runId, experimentName, noSendLogs: options.noSendLogs ?? false, - parameters: assertJsonValue(parameters, "eval parameters"), status: "running", cases: await materializeCases(definition, data, experiment), batches: [], @@ -890,15 +836,8 @@ async function getDurableEvalStatus< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, + definition: DurableEvalRuntimeDefinition, options: { runId: string }, ): Promise { const store = definition.evaluator.store; @@ -916,15 +855,8 @@ async function processDurableBatchResult< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, + definition: DurableEvalRuntimeDefinition, result: DurableBatchResult, ): Promise { if (!result.batchId && !result.externalId) { @@ -966,15 +898,8 @@ async function pollDurableEval< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, + definition: DurableEvalRuntimeDefinition, options: { runId: string }, ): Promise { const store = definition.evaluator.store; @@ -1031,7 +956,7 @@ async function pollDurableEval< } async function openDurableExperiment( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, ) { const data: EvalCase[] = []; @@ -1041,8 +966,7 @@ async function openDurableExperiment( unknown, unknown, unknown, - BaseMetadata, - EvalParameters + BaseMetadata >, data, { @@ -1058,15 +982,8 @@ async function advanceDurableEval< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1121,7 +1038,7 @@ async function advanceDurableEval< } function currentStatus( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, ): DurableEvalResult { if (state.status === "completed") { @@ -1147,7 +1064,7 @@ function currentStatus( } async function startCaseRoot( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, experiment: Experiment | null, @@ -1173,11 +1090,11 @@ async function startCaseRoot( } async function logTaskResult( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, experiment: Experiment | null, - task?: EvalTask, + task?: EvalTask, ) { const datum = record.datum as EvalCase; const root = await startCaseRoot(definition, state, record, experiment); @@ -1185,13 +1102,7 @@ async function logTaskResult( if (task) { const result = await root.traced( (span) => - _internalRunEvaluatorTask( - task, - datum, - record.trialIndex, - state.parameters as Record, - span, - ), + _internalRunEvaluatorTask(task, datum, record.trialIndex, span), { name: "task", spanId: deterministicId(`${state.runId}:${record.id}:task`), @@ -1238,7 +1149,7 @@ async function logTaskResult( } async function logCompletedTasks( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1262,15 +1173,8 @@ async function runTaskStage< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1285,8 +1189,7 @@ async function runTaskStage< Input, Output, Expected, - Metadata, - Parameters + Metadata >; const changed: DurableCaseRecord[] = []; for (const record of state.cases) { @@ -1306,15 +1209,8 @@ async function runScoreStages< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1400,7 +1296,7 @@ function scorerArgs(record: DurableCaseRecord) { } function resumeCaseRoot( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, record: DurableCaseRecord, experiment: Experiment | null, ) { @@ -1415,7 +1311,7 @@ function resumeCaseRoot( } async function evaluateAndLogScore( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, name: string, @@ -1465,7 +1361,7 @@ async function evaluateAndLogScore( } async function evaluateAndLogClassification( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, name: string, @@ -1514,7 +1410,7 @@ async function evaluateAndLogClassification( } async function ensureBatches( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1546,9 +1442,7 @@ async function ensureBatches( if (!claim.created) continue; const context = { runId: state.runId, batchId }; const items = records.map((record) => - kind === "task" - ? taskBatchItem(record, state.parameters) - : scorerBatchItem(record), + kind === "task" ? taskBatchItem(record) : scorerBatchItem(record), ); const submissionData = assertJsonValue( await processor.submit(items, context), @@ -1576,7 +1470,7 @@ async function ensureBatches( } async function collectBatch( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, batch: DurableBatchRecord, ) { @@ -1634,7 +1528,7 @@ async function collectBatch( } function processorForStage( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, kind: "task" | "score", scorerName?: string, ): DurableBatchProcessor { @@ -1662,16 +1556,9 @@ async function materializeCases< Output, Expected, Metadata extends BaseMetadata, - Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition< - Input, - Output, - Expected, - Metadata, - Parameters - >, - data: Evaluator["data"], + definition: DurableEvalRuntimeDefinition, + data: Evaluator["data"], experiment: Experiment | null, ): Promise { const evaluator = definition.evaluator; @@ -1728,7 +1615,7 @@ async function materializeCases< return records; } -function taskBatchItem(record: DurableCaseRecord, parameters: JsonValue) { +function taskBatchItem(record: DurableCaseRecord) { const datum = record.datum as EvalCase; return { id: record.id, @@ -1736,7 +1623,6 @@ function taskBatchItem(record: DurableCaseRecord, parameters: JsonValue) { expected: "expected" in datum ? datum.expected : undefined, metadata: record.metadata, tags: record.tags, - parameters, trialIndex: record.trialIndex, }; } @@ -1755,7 +1641,7 @@ function scorerBatchItem(record: DurableCaseRecord) { } async function finishExperiment( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, experiment: Experiment | null, ) { @@ -1888,7 +1774,7 @@ type DurableBatchPlan = Omit< >; function plannedBatches( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, runId: string, caseIds: string[], ) { @@ -1927,7 +1813,7 @@ function plannedBatches( } async function readCaseRecord( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, store: DurableEvalStore, key: string, id: string, @@ -2018,7 +1904,7 @@ async function readCaseRecord( } async function readRunState( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, store: DurableEvalStore, key: string, ) { @@ -2210,7 +2096,7 @@ function resultItemId(value: unknown) { function isBatchTask( value: unknown, -): value is DurableBatchTask { +): value is DurableBatchTask { return ( typeof value === "object" && value !== null && diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts deleted file mode 100644 index 9a21208ad..000000000 --- a/js/src/eval-parameters.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { z } from "zod/v3"; -import Ajv from "ajv"; -import { Prompt, RemoteEvalParameters } from "./logger"; -import { - promptDefinitionToPromptData, - type PromptDefinitionWithTools, -} from "./prompt-schemas"; -import { PromptData as promptDataSchema } from "./sdk-schemas"; - -export type EvalParameters = Record< - string, - | { - type: "prompt"; - default?: PromptDefinitionWithTools; - description?: string; - } - | { - type: "model"; - default?: string; - description?: string; - } - | z.ZodTypeAny ->; - -// Type helper to infer the type of a parameter value -type InferParameterValue = T extends { type: "prompt" } - ? Prompt - : T extends { type: "model" } - ? string - : T extends z.ZodType - ? z.infer - : never; - -// Type helper to infer the full parameters type -export type InferParameters = { - [K in keyof T]: InferParameterValue; -}; - -export async function validateParameters< - Parameters extends EvalParameters = EvalParameters, - T extends Record = InferParameters, ->( - parameters: Record, - parameterSchema: Parameters | Promise | unknown, -): Promise { - let resolvedSchema = parameterSchema; - if (resolvedSchema instanceof Promise) { - resolvedSchema = await resolvedSchema; - } - - // If no schema is provided, return parameters as-is - if (resolvedSchema === undefined || resolvedSchema === null) { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return parameters as T; - } - - if (RemoteEvalParameters.isParameters(resolvedSchema)) { - const mergedParameters = - parameters && Object.keys(parameters).length > 0 - ? { - ...resolvedSchema.data, - ...parameters, - } - : resolvedSchema.data; - - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return validateParametersWithJsonSchema( - mergedParameters, - resolvedSchema.schema, - ) as T; - } - - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return validateParametersWithZod( - parameters, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - resolvedSchema as Parameters, - ) as T; -} - -function validateParametersWithZod< - Parameters extends EvalParameters = EvalParameters, ->( - parameters: Record, - parameterSchema: Parameters, -): InferParameters { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return Object.fromEntries( - Object.entries(parameterSchema).map(([name, schema]) => { - const value = parameters[name]; - try { - if ("type" in schema && schema.type === "prompt") { - const promptData = value - ? promptDataSchema.parse(value) - : schema.default - ? promptDefinitionToPromptData( - schema.default, - schema.default.tools, - ) - : undefined; - if (!promptData) { - throw new Error(`Parameter '${name}' is required`); - } - return [name, Prompt.fromPromptData(name, promptData)]; - } else if ("type" in schema && schema.type === "model") { - const model = value ?? schema.default; - if (model === undefined) { - throw new Error(`Parameter '${name}' is required`); - } - if (typeof model !== "string") { - throw new Error( - `Parameter '${name}' must be a string model identifier`, - ); - } - return [name, model]; - } else { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const schemaCasted = schema as z.ZodSchema; - return [name, schemaCasted.parse(value)]; - } - } catch (e) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error("Error validating parameter", name, e); - throw Error( - `Invalid parameter '${name}': ${e instanceof Error ? e.message : String(e)}`, - ); - } - }), - ) as InferParameters; -} - -function validateParametersWithJsonSchema>( - parameters: Record, - schema: Record, -): T { - const ajv = new Ajv({ coerceTypes: true, useDefaults: true, strict: false }); - const validate = ajv.compile(schema); - - if (!validate(parameters)) { - const errorMessages = validate.errors - ?.map((err) => { - const path = err.instancePath || "root"; - return `${path}: ${err.message}`; - }) - .join(", "); - throw Error(`Invalid parameters: ${errorMessages}`); - } - - return rehydrateRemoteParameters(parameters, schema) as T; -} - -function rehydrateRemoteParameters( - parameters: Record, - schema: Record, -): Record { - const schemaProperties = schema.properties; - if (typeof schemaProperties !== "object" || schemaProperties === null) { - return parameters; - } - - return Object.fromEntries( - Object.entries(parameters).map(([name, value]) => { - const propertySchema = Reflect.get(schemaProperties, name); - if (typeof propertySchema !== "object" || propertySchema === null) { - return [name, value]; - } - - if (Reflect.get(propertySchema, "x-bt-type") === "prompt") { - return [ - name, - Prompt.fromPromptData(name, promptDataSchema.parse(value)), - ]; - } - - return [name, value]; - }), - ); -} diff --git a/js/src/exports.ts b/js/src/exports.ts index f112a5f7b..511fe1c43 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -37,7 +37,6 @@ export { injectTraceContext, initDataset, initLogger, - loadParameters, loadPrompt, logError, login, diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index a61b578c2..6727361ec 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -15,6 +15,7 @@ import { } from "./framework"; import { _exportsForTestingOnly, + _internalGetGlobalState, BraintrustState, initLogger, TestBackgroundLogger, @@ -53,7 +54,6 @@ test("meta (write) is passed to task", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -90,7 +90,6 @@ test("metadata (read/write) is passed to task", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -129,7 +128,6 @@ test("expected (read/write) is passed to task", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -171,7 +169,6 @@ test("EvalCase id and tags are passed to scorers", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -564,7 +561,6 @@ describe("runEvaluator", () => { }, new NoopProgressReporter(), undefined, - undefined, true, ), ).rejects.toThrow("Encountered 1 unhandled task errors"); @@ -749,7 +745,6 @@ test("trialIndex is passed to task", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -790,7 +785,6 @@ test("trialIndex with multiple inputs", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -835,7 +829,6 @@ test("per-input trialCount overrides global trialCount", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -881,7 +874,6 @@ test("per-input trialCount works without global trialCount", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -1047,7 +1039,6 @@ test("runEvaluator forwards baseExperimentId to summary", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -1089,7 +1080,6 @@ test("runEvaluator forwards persisted baseExperimentName id to summary", async ( }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -1130,7 +1120,6 @@ test("tags can be appended and logged to root span", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); expect(result.results[0].tags).toEqual(expectedTags); @@ -1178,7 +1167,6 @@ test.each([ }, new NoopProgressReporter(), undefined, - undefined, true, ); expect(result.results[0].tags).toEqual(expectedTags); @@ -1217,7 +1205,6 @@ test("tags are persisted with a failing scorer", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); expect(result.results[0].tags).toEqual(expectedTags); @@ -1249,7 +1236,6 @@ test("tags remain empty when not set", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); expect(result.results[0].tags).toEqual(undefined); @@ -1283,7 +1269,6 @@ test("scorer spans have purpose='scorer' attribute", async () => { }, new NoopProgressReporter(), undefined, - undefined, true, ); @@ -1320,59 +1305,173 @@ import { z } from "zod/v3"; import { projects, CodePrompt } from "./framework2"; describe("framework2 metadata support", () => { + test("publishing resolves the project once for prompts and scorers", async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + const project = projects.create({ name: "test-project" }); + const projectId = "00000000-0000-4000-8000-000000000001"; + project.prompts.create({ + name: "completion", + model: "test-model", + prompt: "Hello {{name}}", + templateFormat: "nunjucks", + params: { temperature: 0.5 }, + }); + project.scorers.create({ + name: "quality", + model: "test-model", + messages: [{ role: "user", content: "Rate {{output}}" }], + templateFormat: "mustache", + useCot: true, + choiceScores: { good: 1, bad: 0 }, + }); + const register = vi + .spyOn(_internalGetGlobalState().appConn(), "post_json") + .mockResolvedValue({ + project: { id: projectId, org_id: projectId, name: "test-project" }, + }); + const insert = vi + .spyOn(_internalGetGlobalState().apiConn(), "post_json") + .mockResolvedValue({}); + try { + await project.publish(); + expect(register).toHaveBeenCalledExactlyOnceWith("api/project/register", { + project_name: "test-project", + }); + expect(insert).toHaveBeenCalledExactlyOnceWith("insert-functions", { + functions: [ + expect.objectContaining({ + project_id: projectId, + prompt_data: { + prompt: { type: "completion", content: "Hello {{name}}" }, + options: { model: "test-model", params: { temperature: 0.5 } }, + template_format: "nunjucks", + }, + }), + expect.objectContaining({ + project_id: projectId, + function_type: "scorer", + prompt_data: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "Rate {{output}}" }], + }, + options: { model: "test-model" }, + template_format: "mustache", + parser: { + type: "llm_classifier", + use_cot: true, + choice_scores: { good: 1, bad: 0 }, + }, + }, + }), + ], + }); + } finally { + register.mockRestore(); + insert.mockRestore(); + } + }); + + test("inline provider tools are built and published with prompts", async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + const project = projects.create({ id: "project-123" }); + const tools = [ + { + type: "function" as const, + function: { + name: "weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ]; + const prompt = project.prompts.create({ + name: "weather-prompt", + model: "test-model", + messages: [{ role: "user", content: "Weather in {{city}}?" }], + tools, + }); + expect(prompt.build({ city: "Paris" }).tools).toEqual(tools); + const insert = vi + .spyOn(_internalGetGlobalState().apiConn(), "post_json") + .mockResolvedValue({}); + try { + await project.publish(); + expect(insert).toHaveBeenCalledWith("insert-functions", { + functions: [ + expect.objectContaining({ + prompt_data: expect.objectContaining({ + prompt: expect.objectContaining({ tools: JSON.stringify(tools) }), + }), + }), + ], + }); + const definition = insert.mock.calls[0][1] as { + functions: { prompt_data: object }[]; + }; + expect(definition.functions[0].prompt_data).not.toHaveProperty( + "tool_functions", + ); + } finally { + insert.mockRestore(); + } + }); + describe("CodeFunction metadata", () => { - test("tool stores metadata correctly", () => { + test("classifier stores metadata correctly", () => { const project = projects.create({ name: "test-project" }); const metadata = { version: "1.0", author: "test" }; - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), metadata, }); - expect(tool.metadata).toEqual(metadata); - expect(tool.name).toBe("test-tool"); - expect(tool.slug).toBe("test-tool"); + expect(classifier.metadata).toEqual(metadata); + expect(classifier.name).toBe("test-classifier"); + expect(classifier.slug).toBe("test-classifier"); }); - test("tool works without metadata", () => { + test("classifier works without metadata", () => { const project = projects.create({ name: "test-project" }); - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), }); - expect(tool.metadata).toBeUndefined(); + expect(classifier.metadata).toBeUndefined(); }); - test("tool stores tags correctly", () => { + test("classifier stores tags correctly", () => { const project = projects.create({ name: "test-project" }); const tags = ["ci", "sdk"]; - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), tags, }); - expect(tool.tags).toEqual(tags); + expect(classifier.tags).toEqual(tags); }); - test("tool works without tags", () => { + test("classifier works without tags", () => { const project = projects.create({ name: "test-project" }); - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), }); - expect(tool.tags).toBeUndefined(); + expect(classifier.tags).toBeUndefined(); }); test("classifier registers as a code function", () => { @@ -1466,7 +1565,7 @@ describe("framework2 metadata support", () => { expect(prompts[0].metadata).toBeUndefined(); }); - test("toFunctionDefinition includes metadata when present", async () => { + test("toFunctionDefinition includes metadata when present", () => { const project = projects.create({ name: "test-project" }); const metadata = { version: "2.0", tag: "production" }; @@ -1476,28 +1575,21 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any { name: "test-prompt", slug: "test-prompt", metadata, - } as any, + }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.metadata).toEqual(metadata); expect(funcDef.name).toBe("test-prompt"); expect(funcDef.project_id).toBe("project-123"); }); - test("toFunctionDefinition excludes metadata when undefined", async () => { + test("toFunctionDefinition excludes metadata when undefined", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1506,22 +1598,15 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt" } as any, + { name: "test-prompt", slug: "test-prompt" }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.metadata).toBeUndefined(); }); - test("toFunctionDefinition includes environments for single environment", async () => { + test("toFunctionDefinition includes environments for single environment", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1530,26 +1615,19 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any { name: "test-prompt", slug: "test-prompt", environments: ["production"], - } as any, + }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.environments).toEqual([{ slug: "production" }]); }); - test("toFunctionDefinition includes environments for multiple environments", async () => { + test("toFunctionDefinition includes environments for multiple environments", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1558,21 +1636,14 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any { name: "test-prompt", slug: "test-prompt", environments: ["staging", "production"], - } as any, + }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.environments).toEqual([ { slug: "staging" }, @@ -1580,7 +1651,7 @@ describe("framework2 metadata support", () => { ]); }); - test("toFunctionDefinition excludes environments when undefined", async () => { + test("toFunctionDefinition excludes environments when undefined", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1589,17 +1660,10 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt" } as any, + { name: "test-prompt", slug: "test-prompt" }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.environments).toBeUndefined(); }); @@ -1623,7 +1687,7 @@ describe("framework2 metadata support", () => { expect(prompts[0].tags).toEqual(tags); }); - test("toFunctionDefinition includes tags when present", async () => { + test("toFunctionDefinition includes tags when present", () => { const project = projects.create({ name: "test-project" }); const tags = ["ci", "production"]; @@ -1633,24 +1697,17 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt", tags } as any, + { name: "test-prompt", slug: "test-prompt", tags }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.tags).toEqual(tags); expect(funcDef.name).toBe("test-prompt"); expect(funcDef.project_id).toBe("project-123"); }); - test("toFunctionDefinition excludes tags when undefined", async () => { + test("toFunctionDefinition excludes tags when undefined", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1659,141 +1716,15 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt" } as any, + { name: "test-prompt", slug: "test-prompt" }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.tags).toBeUndefined(); }); }); - describe("CodeParameters defaults", () => { - test("toFunctionDefinition initializes data with schema defaults", async () => { - const project = projects.create({ name: "test-project" }); - project.parameters.create({ - name: "test-parameters", - schema: { - title: z.string().default("default-title"), - numSamples: z.number().default(10), - enabled: z.boolean().default(true), - tags: z.array(z.string()).default(["default-tag"]), - config: z - .object({ - retryCount: z.number(), - strategy: z.string(), - }) - .default({ - retryCount: 3, - strategy: "balanced", - }), - datasetName: z.string(), - main: { - type: "prompt", - default: { - messages: [{ role: "user", content: "{{input}}" }], - model: "gpt-4", - }, - }, - model: { - type: "model", - default: "gpt-5-mini", - }, - }, - }); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const parameters = (project as any)._publishableParameters; - expect(parameters).toHaveLength(1); - - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await parameters[0].toFunctionDefinition(mockProjectMap); - - expect(funcDef.function_data.type).toBe("parameters"); - expect(funcDef.function_data.data).toMatchObject({ - title: "default-title", - numSamples: 10, - enabled: true, - tags: ["default-tag"], - config: { - retryCount: 3, - strategy: "balanced", - }, - main: { - prompt: { - type: "chat", - messages: [{ role: "user", content: "{{input}}" }], - }, - options: { - model: "gpt-4", - }, - }, - model: "gpt-5-mini", - }); - expect(funcDef.function_data.__schema.properties.model).toMatchObject({ - type: "string", - "x-bt-type": "model", - default: "gpt-5-mini", - }); - expect(funcDef.function_data.data).not.toHaveProperty("datasetName"); - }); - - test("toFunctionDefinition does not initialize data when schema has no defaults", async () => { - const project = projects.create({ name: "test-project" }); - project.parameters.create({ - name: "test-parameters-no-defaults", - schema: { - title: z.string(), - numSamples: z.number(), - enabled: z.boolean(), - tags: z.array(z.string()), - config: z.object({ - retryCount: z.number(), - strategy: z.string(), - }), - main: { - type: "prompt", - }, - model: { - type: "model", - }, - }, - }); - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const parameters = (project as any)._publishableParameters; - expect(parameters).toHaveLength(1); - - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await parameters[0].toFunctionDefinition(mockProjectMap); - - expect(funcDef.function_data.type).toBe("parameters"); - expect(funcDef.function_data.data).toEqual({}); - expect(funcDef.function_data.data).not.toHaveProperty("title"); - expect(funcDef.function_data.data).not.toHaveProperty("numSamples"); - expect(funcDef.function_data.data).not.toHaveProperty("enabled"); - expect(funcDef.function_data.data).not.toHaveProperty("tags"); - expect(funcDef.function_data.data).not.toHaveProperty("config"); - expect(funcDef.function_data.data).not.toHaveProperty("main"); - expect(funcDef.function_data.data).not.toHaveProperty("model"); - }); - }); - describe("Scorer metadata", () => { test("code scorer stores metadata correctly", () => { const project = projects.create({ name: "test-project" }); diff --git a/js/src/framework.ts b/js/src/framework.ts index 2e7fea418..2fe0f4847 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -19,7 +19,7 @@ import { queue } from "async"; import iso from "./isomorph"; import { debugLogger } from "./debug-logger"; import { GenericFunction } from "./framework-types"; -import { CodeFunction, CodePrompt, CodeParameters } from "./framework2"; +import { CodeFunction, CodePrompt } from "./framework2"; import { Trace, LocalTrace } from "./trace"; import { BaseMetadata, @@ -31,8 +31,6 @@ import { ExperimentSummary, FullInitOptions, NOOP_SPAN, - type ParametersRef, - RemoteEvalParameters, Span, StartSpanArgs, init as _initExperiment, @@ -53,11 +51,6 @@ import type { ReporterDef, } from "./reporters/types"; import { isEmpty, InternalAbortError } from "./util"; -import { - EvalParameters, - InferParameters, - validateParameters, -} from "./eval-parameters"; export type BaseExperiment< Input, @@ -105,15 +98,9 @@ export type EvalData< | Promise[]> | BaseExperiment); -export type EvalTask< - Input, - Output, - Expected, - Metadata extends BaseMetadata, - Parameters extends EvalParameters, -> = ( +export type EvalTask = ( input: Input, - hooks: EvalHooks, + hooks: EvalHooks, ) => Output | Promise; type TaskProgressEvent = Omit< @@ -121,11 +108,7 @@ type TaskProgressEvent = Omit< "id" | "origin" | "object_type" | "name" >; -export interface EvalHooks< - Expected, - Metadata extends BaseMetadata, - Parameters extends EvalParameters, -> { +export interface EvalHooks { /** * The metadata object for the current evaluation. You can mutate this object to add or remove metadata. */ @@ -138,11 +121,6 @@ export interface EvalHooks< * The task's span. */ span: Span; - /** - * The current parameters being used for this specific task execution. - * Array parameters are converted to single values. - */ - parameters: InferParameters; /** * Report progress that will show up in the playground. */ @@ -218,7 +196,6 @@ export interface Evaluator< Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, - Parameters extends EvalParameters = EvalParameters, > { /** * A function that returns a list of inputs, expected outputs, and metadata. @@ -228,7 +205,7 @@ export interface Evaluator< /** * A function that takes an input and returns an output. */ - task: EvalTask; + task: EvalTask; /** * A set of functions that take an input, output, and expected value and return a {@link Score}. @@ -243,20 +220,6 @@ export interface Evaluator< */ classifiers?: EvalClassifier[]; - /** - * A set of parameters that will be passed to the evaluator. - * Can be: - * - A raw EvalParameters schema (Zod schemas) - * - A Parameters instance from loadParameters() - * - A Promise from loadParameters() - */ - parameters?: - | Parameters - | RemoteEvalParameters> - | Promise< - RemoteEvalParameters> - >; - /** * An optional name for the experiment. */ @@ -397,11 +360,10 @@ export type EvaluatorDef< Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, - Parameters extends EvalParameters = EvalParameters, > = { projectName: string; evalName: string; -} & Evaluator; +} & Evaluator; type EvaluatorFile = { functions: CodeFunction< @@ -410,16 +372,9 @@ type EvaluatorFile = { GenericFunction >[]; prompts: CodePrompt[]; - parameters?: CodeParameters[]; evaluators: { [evalName: string]: { - evaluator: EvaluatorDef< - unknown, - unknown, - unknown, - BaseMetadata, - EvalParameters - >; + evaluator: EvaluatorDef; reporter?: ReporterDef | string; }; }; @@ -437,37 +392,9 @@ function initExperiment( }); } -async function getExperimentParametersRef( - parameters: - | EvalParameters - | RemoteEvalParameters - | Promise> - | undefined, -): Promise { - if (!parameters) { - return undefined; - } - - const resolvedParameters = - parameters instanceof Promise ? await parameters : parameters; - - if (!RemoteEvalParameters.isParameters(resolvedParameters)) { - return undefined; - } - - if (resolvedParameters.id === undefined) { - return undefined; - } - - return { - id: resolvedParameters.id, - version: resolvedParameters.version, - }; -} - export async function _internalInitEvaluatorExperiment( projectName: string, - evaluator: Evaluator, + evaluator: Evaluator, data: EvalData, options: { disabled?: boolean; @@ -477,7 +404,6 @@ export async function _internalInitEvaluatorExperiment( ): Promise { if (options.disabled) return null; const { baseExperiment } = callEvaluatorData(data); - const parameters = await getExperimentParametersRef(evaluator.parameters); return initExperiment(evaluator.state, { ...(evaluator.projectId ? { projectId: evaluator.projectId } @@ -493,7 +419,6 @@ export async function _internalInitEvaluatorExperiment( gitMetadataSettings: evaluator.gitMetadataSettings, repoInfo: evaluator.repoInfo, dataset: Dataset.isDataset(data) ? data : undefined, - parameters, }); } @@ -547,7 +472,7 @@ function isIterable(value: unknown): value is Iterable { export async function _internalResolveEvaluatorData( evaluator: Pick< - EvaluatorDef, + EvaluatorDef, "data" | "projectName" | "projectId" | "state" >, experiment: Experiment | null, @@ -615,12 +540,11 @@ declare global { globalThis._evals = { functions: [], prompts: [], - parameters: [], evaluators: {}, reporters: {}, }; -interface EvalOptions { +interface EvalOptions { /** * A `Reporter` which you can use to summarize progress after an Eval() runs. */ @@ -653,10 +577,6 @@ interface EvalOptions { * is somewhat outdated, and may be removed in the future. */ progress?: ProgressReporter; - /** - * The parameters to use for the evaluator. - */ - parameters?: InferParameters; /** * Whether to retain the per-example Eval results and return them from Eval(). * @@ -711,16 +631,12 @@ export async function Eval< Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, EvalReport = boolean, - Parameters extends EvalParameters = EvalParameters, >( name: string, - evaluator: Evaluator, - reporterOrOpts?: - | ReporterDef - | string - | EvalOptions, + evaluator: Evaluator, + reporterOrOpts?: ReporterDef | string | EvalOptions, ): Promise> { - const options: EvalOptions = isEmpty(reporterOrOpts) + const options: EvalOptions = isEmpty(reporterOrOpts) ? {} : typeof reporterOrOpts === "string" ? { reporter: reporterOrOpts } @@ -739,13 +655,7 @@ export async function Eval< evalName, projectName: name, ...evaluator, - } as EvaluatorDef< - unknown, - unknown, - unknown, - BaseMetadata, - EvalParameters - >, + } as EvaluatorDef, reporter: options.reporter, }; @@ -816,7 +726,6 @@ export async function Eval< evalDef, progressReporter, options.stream, - options.parameters, shouldCollectResults, enableCache, ), @@ -828,7 +737,6 @@ export async function Eval< evalDef, progressReporter, options.stream, - options.parameters, shouldCollectResults, enableCache, ); @@ -885,10 +793,9 @@ export function classifierName( } export async function _internalRunEvaluatorTask( - task: EvalTask, + task: EvalTask, datum: EvalCase, trialIndex: number, - parameters: Record, span: Span, reportProgress: (event: TaskProgressEvent) => void = () => undefined, ): Promise<{ @@ -899,11 +806,10 @@ export async function _internalRunEvaluatorTask( const metadata: Record = { ...("metadata" in datum ? datum.metadata : {}), }; - const hooks: EvalHooks, EvalParameters> = { + const hooks: EvalHooks> = { metadata, expected: "expected" in datum ? datum.expected : undefined, span, - parameters, reportProgress, trialIndex, tags: [...(datum.tags ?? [])], @@ -1108,33 +1014,6 @@ function logScoringFailures( return Object.keys(errorMap); } -export async function runEvaluator( - experiment: Experiment | null, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, - progressReporter: ProgressReporter, - stream: ((data: SSEProgressEventData) => void) | undefined, - parameters?: InferParameters, - collectResults = true, - enableCache = true, - // eslint-disable-next-line @typescript-eslint/no-explicit-any -): Promise> { - if (!evaluator.scores && !evaluator.classifiers) { - throw new Error( - "Evaluator must include at least one of `scores` or `classifiers`", - ); - } - return await runEvaluatorInternal( - experiment, - evaluator, - progressReporter, - stream, - parameters, - collectResults, - enableCache, - ); -} - export const defaultErrorScoreHandler: ErrorScoreHandler = ({ rootSpan, data: _, @@ -1145,27 +1024,27 @@ export const defaultErrorScoreHandler: ErrorScoreHandler = ({ return scores; }; -async function runEvaluatorInternal( +export async function runEvaluator( experiment: Experiment | null, // eslint-disable-next-line @typescript-eslint/no-explicit-any evaluator: EvaluatorDef, progressReporter: ProgressReporter, stream: ((data: SSEProgressEventData) => void) | undefined, - parameters: InferParameters | undefined, - collectResults: boolean, - enableCache: boolean, + collectResults = true, + enableCache = true, // eslint-disable-next-line @typescript-eslint/no-explicit-any ): Promise> { + if (!evaluator.scores && !evaluator.classifiers) { + throw new Error( + "Evaluator must include at least one of `scores` or `classifiers`", + ); + } // Start span cache for this eval (it's disabled by default to avoid temp files outside of evals) // Only start if enableCache is true (default) if (enableCache) { (evaluator.state ?? _internalGetGlobalState())?.spanCache?.start(); } try { - parameters = await validateParameters( - parameters ?? {}, - evaluator.parameters, - ); const dataIterable = await _internalResolveEvaluatorData( evaluator, experiment, @@ -1303,7 +1182,6 @@ async function runEvaluatorInternal( evaluator.task, datum, trialIndex, - parameters ?? {}, span, (event) => { stream?.({ diff --git a/js/src/framework2.ts b/js/src/framework2.ts index fb9ee26ea..12eb8b897 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -7,11 +7,10 @@ import { Project as projectSchema } from "./sdk-schemas"; import type { FunctionTypeEnum as FunctionType, IfExists, - SavedFunctionId, - PromptBlockData, + ChatCompletionMessageParam, + ModelParams, PromptData, ToolFunctionDefinition, - ExtendedSavedFunctionId, } from "./sdk-types"; import { loadPrettyXact, TransactionId } from "../util/index"; import { @@ -21,26 +20,12 @@ import { PromptRowWithId, } from "./logger"; import type { BaseFnOpts, GenericFunction } from "./framework-types"; -import type { EvalParameters } from "./eval-parameters"; -import { - promptDefinitionToPromptData, - type PromptDefinition, -} from "./prompt-schemas"; -import { zodToJsonSchema } from "./zod/utils"; -type ParametersSchema = { - type: "object"; - properties: Record>; - required?: string[]; - additionalProperties?: boolean; -}; // Safe access to __filename (only exists in Node.js CJS) const currentFilename = typeof __filename !== "undefined" ? __filename : "unknown"; -type NameOrId = { name: string } | { id: string }; - -type CreateProjectOpts = NameOrId; +type CreateProjectOpts = { name: string } | { id: string }; class ProjectBuilder { create(opts: CreateProjectOpts) { return new Project(opts); @@ -51,9 +36,7 @@ export const projects = new ProjectBuilder(); class Project { public readonly name?: string; public readonly id?: string; - public tools: ToolBuilder; public prompts: PromptBuilder; - public parameters: ParametersBuilder; public scorers: ScorerBuilder; public classifiers: ClassifierBuilder; @@ -66,15 +49,12 @@ class Project { GenericFunction >[] = []; private _publishablePrompts: CodePrompt[] = []; - private _publishableParameters: CodeParameters[] = []; constructor(args: CreateProjectOpts) { _initializeSpanContext(); this.name = "name" in args ? args.name : undefined; this.id = "id" in args ? args.id : undefined; - this.tools = new ToolBuilder(this); this.prompts = new PromptBuilder(this); - this.parameters = new ParametersBuilder(this); this.scorers = new ScorerBuilder(this); this.classifiers = new ClassifierBuilder(this); } @@ -86,16 +66,6 @@ class Project { } } - public addParameters(parameters: CodeParameters) { - this._publishableParameters.push(parameters); - if (globalThis._lazy_load) { - if (globalThis._evals.parameters == null) - globalThis._evals.parameters = []; - - globalThis._evals.parameters.push(parameters); - } - } - public addCodeFunction( fn: CodeFunction< // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -119,7 +89,6 @@ class Project { return; } await login(); - const projectMap = new ProjectNameIdMap(); const functionDefinitions: PromptFunctionEvent[] = []; if (this._publishableCodeFunctions.length > 0) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. @@ -128,10 +97,17 @@ class Project { ); } if (this._publishablePrompts.length > 0) { + const projectId = + this.id ?? + z.object({ project: projectSchema }).parse( + await _internalGetGlobalState() + .appConn() + .post_json("api/project/register", { + project_name: this.name, + }), + ).project.id; for (const prompt of this._publishablePrompts) { - const functionDefinition = - await prompt.toFunctionDefinition(projectMap); - functionDefinitions.push(functionDefinition); + functionDefinitions.push(prompt.toFunctionDefinition(projectId)); } } @@ -141,76 +117,6 @@ class Project { } } -class ToolBuilder { - private taskCounter = 0; - constructor(private readonly project: Project) {} - - public create< - // eslint-disable-next-line @typescript-eslint/no-explicit-any - TParams extends { _output: any; _input: any; _def: any }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - TReturns extends { _output: any; _input: any; _def: any }, - THandler extends GenericFunction, - >( - opts: Partial & { - handler: THandler; - parameters: TParams; - returns: TReturns; - }, - ): CodeFunction; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - public create>( - opts: Partial & { - handler: THandler; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - parameters?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - returns?: any; - }, // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): CodeFunction; - - // This type definition is just a catch all so that the implementation can be - // less specific than the two more specific declarations above. - public create( - opts: Partial & { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - handler: GenericFunction; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - parameters?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - returns?: any; - }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): CodeFunction { - this.taskCounter++; - opts = opts ?? {}; - - const { handler, name, slug, parameters, returns, ...rest } = opts; - let resolvedName = name ?? handler.name; - - if (resolvedName.trim().length === 0) { - resolvedName = `Tool ${iso.basename(currentFilename)} ${this.taskCounter}`; - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const tool: CodeFunction = new CodeFunction(this.project, { - handler, - name: resolvedName, - slug: slug ?? slugify(resolvedName, { lower: true, strict: true }), - type: "tool", - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - parameters: parameters as any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - returns: returns as any, - ...rest, - }); - - this.project.addCodeFunction(tool); - return tool; - } -} - class ScorerBuilder { private taskCounter = 0; constructor(private readonly project: Project) {} @@ -250,35 +156,17 @@ class ScorerBuilder { }); this.project.addCodeFunction(scorer); } else { - const promptBlock: PromptBlockData = - "messages" in opts - ? { - type: "chat", - messages: opts.messages, - } - : { - type: "completion", - content: opts.prompt, - }; const promptData: PromptData = { - prompt: promptBlock, - options: { - model: opts.model, - params: opts.params, - }, + ...promptDefinitionToPromptData(opts), parser: { type: "llm_classifier", use_cot: opts.useCot, choice_scores: opts.choiceScores, }, - ...(opts.templateFormat - ? { template_format: opts.templateFormat } - : {}), }; const codePrompt = new CodePrompt( this.project, promptData, - [], { ...opts, name: resolvedName, @@ -304,7 +192,7 @@ class ClassifierBuilder { Exact>, Returns >, - >(opts: ClassifierOpts) { + >(opts: CodeOpts>, Returns, Fn>) { this.taskCounter++; let resolvedName = opts.name ?? opts.handler.name; @@ -329,25 +217,23 @@ class ClassifierBuilder { } } -type Schema = Partial<{ - parameters: z.ZodSchema; - returns: z.ZodSchema; -}>; - type CodeOpts< Params, Returns, Fn extends GenericFunction, > = Partial & { handler: Fn; - metadata?: Record; -} & Schema; + parameters?: z.ZodSchema; + returns?: z.ZodSchema; +}; type ScorerPromptOpts = Partial & - PromptOpts & { + PromptDefinition & { + name: string; + id?: string; + version?: TransactionId; useCot: boolean; choiceScores: Record; - metadata?: Record; }; // A more correct ScorerArgs than that in core/js/src/score.ts. @@ -365,7 +251,7 @@ type Exact = T extends Shape : never : never; -type ScorerOptsUnion< +type ScorerOpts< Output, Input, Params, @@ -375,26 +261,6 @@ type ScorerOptsUnion< | CodeOpts>, Returns, Fn> | ScorerPromptOpts; -type ScorerOpts< - Output, - Input, - Params, - Returns, - Fn extends GenericFunction>, Returns>, -> = ScorerOptsUnion & { - metadata?: Record; -}; - -type ClassifierOpts< - Output, - Input, - Params, - Returns, - Fn extends GenericFunction>, Returns>, -> = CodeOpts>, Returns, Fn> & { - metadata?: Record; -}; - export class CodeFunction< Input, Output, @@ -447,17 +313,6 @@ export class CodeFunction< } } -type GenericCodeFunction = CodeFunction< - // This has to be marked as any because we want to support arrays of - // functions that return different things. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - GenericFunction ->; - export class CodePrompt { public readonly project: Project; public readonly name: string; @@ -467,7 +322,6 @@ export class CodePrompt { public readonly description?: string; public readonly id?: string; public readonly functionType?: FunctionType; - public readonly toolFunctions: (SavedFunctionId | GenericCodeFunction)[]; public readonly tags?: string[]; public readonly metadata?: Record; public readonly environmentSlugs?: string[]; @@ -475,10 +329,11 @@ export class CodePrompt { constructor( project: Project, prompt: PromptData, - toolFunctions: (SavedFunctionId | GenericCodeFunction)[], - opts: Omit, "name" | "slug"> & { + opts: Partial & { name: string; slug: string; + id?: string; + environments?: string[]; }, functionType?: FunctionType, ) { @@ -486,7 +341,6 @@ export class CodePrompt { this.name = opts.name; this.slug = opts.slug; this.prompt = prompt; - this.toolFunctions = toolFunctions; this.ifExists = opts.ifExists; this.description = opts.description; this.id = opts.id; @@ -496,35 +350,9 @@ export class CodePrompt { this.environmentSlugs = opts.environments; } - async toFunctionDefinition( - projectNameToId: ProjectNameIdMap, - ): Promise { - const prompt_data = { - ...this.prompt, - }; - if (this.toolFunctions.length > 0) { - const resolvableToolFunctions: ExtendedSavedFunctionId[] = - await Promise.all( - this.toolFunctions.map(async (fn) => { - if ("slug" in fn) { - return { - type: "slug", - project_id: await projectNameToId.resolve(fn.project), - slug: fn.slug, - }; - } else { - return fn; - } - }), - ); - - // This is a hack because these will be resolved on the server side. - prompt_data.tool_functions = - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - resolvableToolFunctions as SavedFunctionId[]; - } + toFunctionDefinition(projectId: string): PromptFunctionEvent { return { - project_id: await projectNameToId.resolve(this.project), + project_id: projectId, name: this.name, slug: this.slug, description: this.description ?? "", @@ -532,7 +360,7 @@ export class CodePrompt { type: "prompt", }, function_type: this.functionType, - prompt_data, + prompt_data: this.prompt, if_exists: this.ifExists, tags: this.tags, metadata: this.metadata, @@ -544,34 +372,27 @@ export class CodePrompt { } } -interface PromptId { - id: string; -} - -interface PromptVersion { - version: TransactionId; -} - -interface PromptTools { - tools: (GenericCodeFunction | SavedFunctionId | ToolFunctionDefinition)[]; -} - -interface PromptNoTrace { - noTrace: boolean; -} +type PromptDefinition = ( + | { prompt: string } + | { messages: ChatCompletionMessageParam[] } +) & { + model: string; + params?: ModelParams; + templateFormat?: "mustache" | "nunjucks" | "none"; + environments?: string[]; +}; type PromptOpts< HasId extends boolean, HasVersion extends boolean, - HasTools extends boolean = true, - HasNoTrace extends boolean = true, -> = (Partial> & { name: string }) & - (HasId extends true ? PromptId : Partial) & - (HasVersion extends true ? PromptVersion : Partial) & - // eslint-disable-next-line @typescript-eslint/no-empty-object-type - (HasTools extends true ? Partial : {}) & - // eslint-disable-next-line @typescript-eslint/no-empty-object-type - (HasNoTrace extends true ? Partial : {}) & +> = Partial & { + name: string; + tools?: ToolFunctionDefinition[]; + noTrace?: boolean; +} & (HasId extends true ? { id: string } : { id?: string }) & + (HasVersion extends true + ? { version: TransactionId } + : { version?: TransactionId }) & PromptDefinition; class PromptBuilder { @@ -581,22 +402,12 @@ class PromptBuilder { HasId extends boolean = false, HasVersion extends boolean = false, >(opts: PromptOpts): Prompt { - const toolFunctions: (SavedFunctionId | GenericCodeFunction)[] = []; - const rawTools: ToolFunctionDefinition[] = []; - - for (const tool of opts.tools ?? []) { - if (tool instanceof CodeFunction) { - toolFunctions.push(tool); - } else if ("type" in tool && !("function" in tool)) { - toolFunctions.push(tool); - } else { - rawTools.push(tool); - } - } - const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true }); - const promptData: PromptData = promptDefinitionToPromptData(opts, rawTools); + const promptData: PromptData = promptDefinitionToPromptData( + opts, + opts.tools, + ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const promptRow: PromptRowWithId = { @@ -615,7 +426,7 @@ class PromptBuilder { opts.noTrace ?? false, ); - const codePrompt = new CodePrompt(this.project, promptData, toolFunctions, { + const codePrompt = new CodePrompt(this.project, promptData, { ...opts, slug, }); @@ -625,157 +436,9 @@ class PromptBuilder { } } -interface ParametersOpts { - name: string; - slug?: string; - description?: string; - schema: S; - ifExists?: IfExists; - metadata?: Record; -} - -export class CodeParameters { - public readonly project: Project; - public readonly name: string; - public readonly slug: string; - public readonly description?: string; - public readonly schema: EvalParameters; - public readonly ifExists?: IfExists; - public readonly metadata?: Record; - - constructor( - project: Project, - opts: { - name: string; - slug: string; - description?: string; - schema: EvalParameters; - ifExists?: IfExists; - metadata?: Record; - }, - ) { - this.project = project; - this.name = opts.name; - this.slug = opts.slug; - this.description = opts.description; - this.schema = opts.schema; - this.ifExists = opts.ifExists; - this.metadata = opts.metadata; - } - - async toFunctionDefinition( - projectNameToId: ProjectNameIdMap, - ): Promise { - const schema = serializeEvalParameterstoParametersSchema(this.schema); - return { - project_id: await projectNameToId.resolve(this.project), - name: this.name, - slug: this.slug, - description: this.description ?? "", - function_type: "parameters", - function_data: { - type: "parameters", - data: getDefaultDataFromParametersSchema(schema), - __schema: schema, - }, - if_exists: this.ifExists, - metadata: this.metadata, - }; - } -} - -class ParametersBuilder { - constructor(private readonly project: Project) {} - - public create(opts: ParametersOpts): S { - const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true }); - - const codeParameters = new CodeParameters(this.project, { - name: opts.name, - slug, - description: opts.description, - schema: opts.schema, - ifExists: opts.ifExists, - metadata: opts.metadata, - }); - - this.project.addParameters(codeParameters); - - return opts.schema; - } -} - -function serializeEvalParameterstoParametersSchema( - parameters: EvalParameters, -): ParametersSchema { - const properties: Record> = {}; - const required: string[] = []; - - for (const [name, value] of Object.entries(parameters)) { - if ("type" in value && value.type === "prompt") { - const defaultPromptData = value.default - ? promptDefinitionToPromptData(value.default) - : undefined; - - properties[name] = { - type: "object", - "x-bt-type": "prompt", - ...(value.description ? { description: value.description } : {}), - ...(defaultPromptData ? { default: defaultPromptData } : {}), - }; - - if (!defaultPromptData) { - required.push(name); - } - } else if ("type" in value && value.type === "model") { - properties[name] = { - type: "string", - "x-bt-type": "model", - ...(value.description ? { description: value.description } : {}), - ...("default" in value ? { default: value.default } : {}), - }; - - if (!("default" in value)) { - required.push(name); - } - } else { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const schemaObj = zodToJsonSchema(value as z.ZodType) as Record< - string, - unknown - >; - - properties[name] = schemaObj; - - if (!("default" in schemaObj)) { - required.push(name); - } - } - } - - return { - type: "object", - properties, - ...(required.length > 0 ? { required } : {}), - additionalProperties: true, - }; -} - -function getDefaultDataFromParametersSchema( - schema: ParametersSchema, -): Record { - return Object.fromEntries( - Object.entries(schema.properties).flatMap(([name, value]) => { - if (!("default" in value)) { - return []; - } - - return [[name, value.default]]; - }), - ); -} - -interface FunctionEventBase { +interface PromptFunctionEvent { + prompt_data: PromptData; + function_data: { type: "prompt" }; project_id: string; slug: string; name: string; @@ -787,65 +450,33 @@ interface FunctionEventBase { environments?: { slug: string }[]; } -interface PromptFunctionEvent extends FunctionEventBase { - prompt_data: PromptData; - function_data: { type: "prompt" }; -} +function promptDefinitionToPromptData( + promptDefinition: PromptDefinition, + rawTools?: ToolFunctionDefinition[], +): PromptData { + const promptBlock: PromptData["prompt"] = + "messages" in promptDefinition + ? { + type: "chat", + messages: promptDefinition.messages, + tools: + rawTools && rawTools.length > 0 + ? JSON.stringify(rawTools) + : undefined, + } + : { + type: "completion", + content: promptDefinition.prompt, + }; -interface ParametersFunctionEvent extends FunctionEventBase { - function_type: "parameters"; - function_data: { - type: "parameters"; - data: Record; - __schema: ParametersSchema; + return { + prompt: promptBlock, + options: { + model: promptDefinition.model, + params: promptDefinition.params, + }, + ...(promptDefinition.templateFormat + ? { template_format: promptDefinition.templateFormat } + : {}), }; } - -class ProjectNameIdMap { - private nameToId: Record = {}; - private idToName: Record = {}; - - async getId(projectName: string): Promise { - if (!(projectName in this.nameToId)) { - const response = await _internalGetGlobalState() - .appConn() - .post_json("api/project/register", { - project_name: projectName, - }); - - const result = z - .object({ - project: projectSchema, - }) - .parse(response); - - const projectId = result.project.id; - - this.nameToId[projectName] = projectId; - this.idToName[projectId] = projectName; - } - return this.nameToId[projectName]; - } - - async getName(projectId: string): Promise { - if (!(projectId in this.idToName)) { - const response = await _internalGetGlobalState() - .appConn() - .post_json("api/project/get", { - id: projectId, - }); - const result = z.array(projectSchema).nonempty().parse(response); - const projectName = result[0].name; - this.idToName[projectId] = projectName; - this.nameToId[projectName] = projectId; - } - return this.idToName[projectId]; - } - - async resolve(project: Project): Promise { - if (project.id) { - return project.id; - } - return this.getId(project.name!); - } -} diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index ffe6c1030..f4acfcab4 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -17,11 +17,9 @@ import { initLogger, extractTraceContextFromHeaders, Prompt, - RemoteEvalParameters, BraintrustState, FailedHTTPResponse, loadPrompt, - loadParameters, wrapTraced, traced, currentSpan, @@ -48,10 +46,6 @@ import { type PromptDiskCacheEntry, type PromptMemoryCacheEntry, } from "./prompt-cache/prompt-cache"; -import { - ParametersCache, - type ParametersMemoryCacheEntry, -} from "./prompt-cache/parameters-cache"; import { DiskCache } from "./prompt-cache/disk-cache"; import { LRUCache } from "./lru-cache"; @@ -2100,7 +2094,7 @@ test("init surfaces dataset environment lookup errors instead of falling back to vi.restoreAllMocks(); }); -describe("prompt and parameters loaders", () => { +describe("prompt loaders", () => { let state: BraintrustState; let getJson: ReturnType; const promptRow = { @@ -2138,44 +2132,6 @@ describe("prompt and parameters loaders", () => { options: { model: string }; }; }; - const parametersRow = { - id: "44444444-4444-4444-8444-444444444444", - _xact_id: "v1", - project_id: "55555555-5555-4555-8555-555555555555", - name: "Saved parameters", - slug: "saved-parameters", - description: null, - function_type: "parameters", - function_data: { - type: "parameters", - data: { prefix: "hello" }, - __schema: { - type: "object", - properties: { - prefix: { type: "string", default: "hello" }, - }, - additionalProperties: true, - }, - }, - } satisfies { - id: string; - _xact_id: string; - project_id: string; - name: string; - slug: string; - description: null; - function_type: "parameters"; - function_data: { - type: "parameters"; - data: { prefix: string }; - __schema: { - type: string; - properties: { prefix: { type: string; default: string } }; - additionalProperties: boolean; - }; - }; - }; - function jsonResponse(data: unknown): Response { return new Response(JSON.stringify(data), { headers: { "Content-Type": "application/json" }, @@ -2890,135 +2846,6 @@ describe("prompt and parameters loaders", () => { "Organization prompt-org-name not found. Must be one of other-org-name", ); }); - - test("loadParameters uses request-local credentials and isolated caches", async () => { - state.parametersCache = new ParametersCache({ - memoryCache: new LRUCache({ - max: 10, - }), - }); - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - loginResponse({ - id: "parameters-org-id", - name: "test-org-name", - apiUrl: "https://parameters-api.test", - }), - ) - .mockResolvedValueOnce(jsonResponse({ objects: [parametersRow] })) - .mockResolvedValueOnce( - loginResponse({ - id: "parameters-org-id", - name: "test-org-name", - apiUrl: "https://parameters-api.test", - }), - ) - .mockResolvedValueOnce(serverErrorResponse()); - - await loadParameters({ - projectName: "test-project", - slug: "saved-parameters", - apiKey: "first-parameters-api-key", - fetch: fetchMock, - state, - }); - - await expect( - loadParameters({ - projectName: "test-project", - slug: "saved-parameters", - apiKey: "second-parameters-api-key", - fetch: fetchMock, - state, - }), - ).rejects.toThrow("not found on server or in local cache"); - expect(state.loginToken).toBe("___TEST_API_KEY__THIS_IS_NOT_REAL___"); - }); - - test("loadParameters never falls back on ambient authentication rejection", async () => { - getJson - .mockResolvedValueOnce({ objects: [parametersRow] }) - .mockRejectedValueOnce( - new FailedHTTPResponse(403, "Forbidden", "Revoked API key"), - ); - - await loadParameters({ - projectName: "test-project", - slug: "saved-parameters", - state, - }); - - await expect( - loadParameters({ - projectName: "test-project", - slug: "saved-parameters", - state, - }), - ).rejects.toThrow("403: Forbidden"); - }); - - test("resolved parameter caches validate the organization identity", async () => { - const cache = new ParametersCache({ - memoryCache: new LRUCache({ - max: 10, - }), - }); - const parameters = new RemoteEvalParameters(parametersRow); - const key = { - projectName: "test-project", - slug: "saved-parameters", - version: "latest", - }; - - await cache.withNamespace("credential", "first-org").set(key, parameters); - - expect( - await cache.withNamespace("credential", "first-org").get(key), - ).toBe(parameters); - expect(await cache.withNamespace("credential").get(key)).toBe(parameters); - expect( - await cache.withNamespace("credential", "second-org").get(key), - ).toBeUndefined(); - }); - }); - - test("loadParameters prefers version over environment for project lookup", async () => { - getJson.mockResolvedValue({ objects: [parametersRow] }); - - await loadParameters({ - projectName: "test-project", - slug: "saved-parameters", - version: "v1", - environment: "production", - state, - }); - - expect(getJson).toHaveBeenCalledWith( - "v1/function", - expect.objectContaining({ - project_name: "test-project", - slug: "saved-parameters", - version: "v1", - function_type: "parameters", - }), - ); - expect(getJson.mock.calls[0][1]).not.toHaveProperty("environment"); - }); - - test("loadParameters prefers version over environment for id lookup", async () => { - getJson.mockResolvedValue(parametersRow); - - await loadParameters({ - id: parametersRow.id, - version: "v1", - environment: "production", - state, - }); - - expect(getJson).toHaveBeenCalledWith(`v1/function/${parametersRow.id}`, { - version: "v1", - }); }); }); diff --git a/js/src/logger.ts b/js/src/logger.ts index b7bd554c0..83a4075b2 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -145,39 +145,6 @@ export type DatasetRestoreResult = { rows_deleted: number; }; -const parametersRowSchema = z.object({ - id: z.string().uuid(), - _xact_id: z.string(), - project_id: z.string().uuid(), - name: z.string(), - slug: z.string(), - description: z.union([z.string(), z.null()]).optional(), - function_type: z.literal("parameters"), - function_data: z.object({ - type: z.literal("parameters"), - data: z.record(z.unknown()).optional(), - __schema: z.record(z.unknown()), - }), - metadata: z - .union([z.object({}).partial().passthrough(), z.null()]) - .optional(), -}); -type ParametersRow = { - id: string; - _xact_id: string; - project_id: string; - name: string; - slug: string; - description?: string | null; - function_type: "parameters"; - function_data: { - type: "parameters"; - data?: Record; - __schema: Record; - }; - metadata?: Record | null; -}; - import { waitUntil } from "@vercel/functions"; import { parseTemplateFormat, @@ -192,17 +159,9 @@ import { devNullWritableStream, } from "./functions/stream"; import iso, { IsoAsyncLocalStorage } from "./isomorph"; -import { createCacheLayers } from "./prompt-cache/cache-config"; -import { - PromptCache, - type PromptDiskCacheEntry, - type PromptMemoryCacheEntry, -} from "./prompt-cache/prompt-cache"; -import { - ParametersCache, - type ParametersDiskCacheEntry, - type ParametersMemoryCacheEntry, -} from "./prompt-cache/parameters-cache"; +import { createPromptCache } from "./prompt-cache/cache-config"; +import type { PromptCache } from "./prompt-cache/prompt-cache"; + import { LRUCache } from "./lru-cache"; import { addAzureBlobHeaders, @@ -216,7 +175,6 @@ import { import { lintTemplate as _lintMustacheTemplate } from "./template/mustache-utils"; import { prettifyXact } from "../util/index"; import { SpanCache, CachedSpan } from "./span-cache"; -import type { EvalParameters, InferParameters } from "./eval-parameters"; import { detectSpanOriginEnvironment, getSpanInstrumentationName, @@ -785,11 +743,6 @@ type ResolvedLoaderLoginOptions = Required< existingState?: BraintrustState; }; -type LoaderCacheViews = { - promptCache: PromptCache; - parametersCache: ParametersCache; -}; - type LoaderRequestState = Pick; // proxyUrl may point at the universal proxy (`{apiUrl}/v1/proxy`) for @@ -833,7 +786,6 @@ export class BraintrustState { private _proxyConn: HTTPConnection | null = null; public promptCache: PromptCache; - public parametersCache: ParametersCache; public spanCache: SpanCache; private _idGenerator: IDGenerator | null = null; private _contextManager: ContextManager | null = null; @@ -886,34 +838,7 @@ export class BraintrustState { this.resetLoginInfo(); - const { memoryCache, diskCache } = createCacheLayers< - PromptMemoryCacheEntry, - PromptDiskCacheEntry - >({ - memoryMaxEnvVar: "BRAINTRUST_PROMPT_CACHE_MEMORY_MAX", - diskCacheDirEnvVar: "BRAINTRUST_PROMPT_CACHE_DIR", - diskMaxEnvVar: "BRAINTRUST_PROMPT_CACHE_DISK_MAX", - getDefaultDiskCacheDir: () => - `${iso.getEnv("HOME") ?? iso.homedir!()}/.braintrust/prompt_cache`, - }); - this.promptCache = new PromptCache({ memoryCache, diskCache }); - - const { - memoryCache: parametersMemoryCache, - diskCache: parametersDiskCache, - } = createCacheLayers( - { - memoryMaxEnvVar: "BRAINTRUST_PARAMETERS_CACHE_MEMORY_MAX", - diskCacheDirEnvVar: "BRAINTRUST_PARAMETERS_CACHE_DIR", - diskMaxEnvVar: "BRAINTRUST_PARAMETERS_CACHE_DISK_MAX", - getDefaultDiskCacheDir: () => - `${iso.getEnv("HOME") ?? iso.homedir!()}/.braintrust/parameters_cache`, - }, - ); - this.parametersCache = new ParametersCache({ - memoryCache: parametersMemoryCache, - diskCache: parametersDiskCache, - }); + this.promptCache = createPromptCache(); this.spanCache = new SpanCache({ disabled: loginParams.disableSpanCache }); this.spanOriginEnvironment = detectSpanOriginEnvironment(); @@ -1025,10 +950,10 @@ export class BraintrustState { } /** @internal */ - public _internalGetLoaderCacheViews( + public _internalGetPromptCache( loginOptions: ResolvedLoaderLoginOptions, requestState?: LoaderRequestState, - ): LoaderCacheViews { + ): PromptCache { const expectedResolvedOrgIdentity = requestState?.orgId && requestState.appUrl ? JSON.stringify([ @@ -1038,16 +963,10 @@ export class BraintrustState { ]) : undefined; - return { - promptCache: this.promptCache.withNamespace( - loginOptions.credentialCacheNamespace, - expectedResolvedOrgIdentity, - ), - parametersCache: this.parametersCache.withNamespace( - loginOptions.credentialCacheNamespace, - expectedResolvedOrgIdentity, - ), - }; + return this.promptCache.withNamespace( + loginOptions.credentialCacheNamespace, + expectedResolvedOrgIdentity, + ); } /** @internal */ @@ -4054,17 +3973,11 @@ export type DatasetRef = { id: string; } & DatasetSelection; -export interface ParametersRef { - id: string; - version?: string; -} - export type InitOptions = FullLoginOptions & { experiment?: string; description?: string; dataset?: AnyDataset | DatasetRef; _internal_btql?: Record; - parameters?: ParametersRef | RemoteEvalParameters; update?: boolean; baseExperiment?: string; isPublic?: boolean; @@ -4144,7 +4057,6 @@ export function init( description, dataset, _internal_btql, - parameters, baseExperiment, isPublic, open, @@ -4292,18 +4204,6 @@ export function init( }; } - if (parameters !== undefined) { - if (RemoteEvalParameters.isParameters(parameters)) { - args["parameters_id"] = parameters.id; - args["parameters_version"] = parameters.version; - } else { - args["parameters_id"] = parameters.id; - if (parameters.version !== undefined) { - args["parameters_version"] = parameters.version; - } - } - } - if (isPublic !== undefined) { args["public"] = isPublic; } @@ -4918,99 +4818,6 @@ export type LoadPromptOptions = FullLoginOptions & { state?: BraintrustState; }; -type LoadParametersBaseOptions = FullLoginOptions & { - state?: BraintrustState; -}; - -export type LoadParametersByIdOptions = LoadParametersBaseOptions & { - id: string; - version?: string; -}; - -export type LoadParametersByIdWithEnvOptions = LoadParametersBaseOptions & { - id: string; - environment: string; - version?: string; -}; - -export type LoadParametersByProjectNameOptions = LoadParametersBaseOptions & { - projectName: string; - slug: string; - version?: string; -}; - -export type LoadParametersByProjectNameWithEnvOptions = - LoadParametersBaseOptions & { - projectName: string; - slug: string; - environment: string; - version?: string; - }; - -export type LoadParametersByProjectIdOptions = LoadParametersBaseOptions & { - projectId: string; - slug: string; - version?: string; -}; - -export type LoadParametersByProjectIdWithEnvOptions = - LoadParametersBaseOptions & { - projectId: string; - slug: string; - environment: string; - version?: string; - }; - -export type LoadParametersOptions = - | LoadParametersByIdOptions - | LoadParametersByIdWithEnvOptions - | LoadParametersByProjectNameOptions - | LoadParametersByProjectNameWithEnvOptions - | LoadParametersByProjectIdOptions - | LoadParametersByProjectIdWithEnvOptions; - -type LoadParametersImplementationOptions = LoadParametersBaseOptions & { - projectName?: string; - projectId?: string; - slug?: string; - version?: string; - id?: string; - environment?: string; -}; - -type LoaderRequestResult = - | ({ ok: true; response: T } & LoaderCacheViews) - | ({ ok: false; error: unknown } & LoaderCacheViews); - -async function runCredentialScopedLoaderRequest( - state: BraintrustState, - loginOptions: Pick< - FullLoginOptions, - "apiKey" | "appUrl" | "orgName" | "fetch" | "forceLogin" - >, - request: (requestState: LoaderRequestState) => Promise, -): Promise> { - const resolvedLoginOptions = - await state._internalResolveLoaderLoginOptions(loginOptions); - let cacheViews = state._internalGetLoaderCacheViews(resolvedLoginOptions); - - try { - const requestState = - await state._internalGetLoaderState(resolvedLoginOptions); - cacheViews = state._internalGetLoaderCacheViews( - resolvedLoginOptions, - requestState, - ); - return { - ok: true, - response: await request(requestState), - ...cacheViews, - }; - } catch (error) { - return { ok: false, error, ...cacheViews }; - } -} - /** * Load a prompt from the specified project. * @@ -5069,34 +4876,39 @@ export async function loadPrompt({ } const state = stateArg ?? _globalState; - const result = await runCredentialScopedLoaderRequest( - state, - { orgName, apiKey, appUrl, fetch, forceLogin }, - async (requestState) => { - let response; - if (id) { - // Load prompt by ID using the /v1/prompt/{id} endpoint - response = await requestState - .apiConn() - .get_json(`v1/prompt/${id}`, versionOrEnvironment); - // Wrap single prompt response in objects array to match list API format - if (response) { - response = { objects: [response] }; - } - } else { - response = await requestState.apiConn().get_json("v1/prompt", { - project_name: projectName, - project_id: projectId, - slug, - ...versionOrEnvironment, - }); + const resolvedLoginOptions = await state._internalResolveLoaderLoginOptions({ + orgName, + apiKey, + appUrl, + fetch, + forceLogin, + }); + let promptCache = state._internalGetPromptCache(resolvedLoginOptions); + let response; + try { + const requestState = + await state._internalGetLoaderState(resolvedLoginOptions); + promptCache = state._internalGetPromptCache( + resolvedLoginOptions, + requestState, + ); + if (id) { + // Wrap the single prompt response to match the list API format. + response = await requestState + .apiConn() + .get_json(`v1/prompt/${id}`, versionOrEnvironment); + if (response) { + response = { objects: [response] }; } - return response; - }, - ); - const { promptCache } = result; - if (!result.ok) { - const e = result.error; + } else { + response = await requestState.apiConn().get_json("v1/prompt", { + project_name: projectName, + project_id: projectId, + slug, + ...versionOrEnvironment, + }); + } + } catch (e) { if (!isLoaderCacheFallbackError(e)) { throw e; } @@ -5133,8 +4945,6 @@ export async function loadPrompt({ } return prompt; } - const { response } = result; - if (!("objects" in response) || response.objects.length === 0) { if (id) { throw new Error(`Prompt with id ${id} not found.`); @@ -5174,186 +4984,6 @@ export async function loadPrompt({ return prompt; } -/** - * Load parameters from the specified project. - * - * @param options Options for configuring loadParameters(). - * @param options.projectName The name of the project to load the parameters from. Must specify at least one of `projectName` or `projectId`. - * @param options.projectId The id of the project to load the parameters from. This takes precedence over `projectName` if specified. - * @param options.slug The slug of the parameters to load. - * @param options.version An optional version of the parameters (to read). If not specified, the latest version will be used. - * @param options.environment Fetch the version of the parameters assigned to the specified environment (e.g. "production", "staging"). If both `version` and `environment` are provided, `version` takes precedence. - * @param options.id The id of specific parameters to load. If specified, this takes precedence over all other parameters (project and slug). - * @param options.appUrl The URL of the Braintrust App. Defaults to https://www.braintrust.dev. - * @param options.apiKey The API key to use for this request, independently of any existing global login. If the parameter is not specified, will use an existing login or try the `BRAINTRUST_API_KEY` environment variable. In Node.js, if that is unset, will try the nearest `.env.braintrust` file in the current working directory or parent directories. - * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. - * @returns The parameters object. - * @throws If the parameters are not found. - * @throws If multiple parameters are found with the same slug in the same project (this should never happen). - * - * @example - * ```typescript - * const params = await loadParameters({ - * projectName: "My Project", - * slug: "my-parameters", - * }); - * ``` - */ -export function loadParameters( - options: LoadParametersByProjectNameOptions, -): Promise>>; -export function loadParameters( - options: LoadParametersByProjectNameWithEnvOptions, -): Promise>>; -export function loadParameters( - options: LoadParametersByProjectIdOptions, -): Promise>>; -export function loadParameters( - options: LoadParametersByProjectIdWithEnvOptions, -): Promise>>; -export function loadParameters( - options: LoadParametersByIdOptions, -): Promise>>; -export function loadParameters( - options: LoadParametersByIdWithEnvOptions, -): Promise>>; -export async function loadParameters< - S extends EvalParameters = EvalParameters, ->({ - projectName, - projectId, - slug, - version, - environment, - id, - appUrl, - apiKey, - orgName, - fetch, - forceLogin, - state: stateArg, -}: LoadParametersImplementationOptions): Promise< - RemoteEvalParameters> -> { - const versionOrEnvironment = version - ? { version } - : environment - ? { environment } - : {}; - if (id) { - // When loading by ID, we don't need project or slug - } else if (isEmpty(projectName) && isEmpty(projectId)) { - throw new Error("Must specify either projectName or projectId"); - } else if (isEmpty(slug)) { - throw new Error("Must specify slug"); - } - - const state = stateArg ?? _globalState; - const result = await runCredentialScopedLoaderRequest( - state, - { orgName, apiKey, appUrl, fetch, forceLogin }, - async (requestState) => { - let response; - if (id) { - response = await requestState.apiConn().get_json(`v1/function/${id}`, { - ...versionOrEnvironment, - }); - if (response) { - response = { objects: [response] }; - } - } else { - response = await requestState.apiConn().get_json("v1/function", { - project_name: projectName, - project_id: projectId, - slug, - function_type: "parameters", - ...versionOrEnvironment, - }); - } - return response; - }, - ); - const { parametersCache } = result; - if (!result.ok) { - const e = result.error; - if (!isLoaderCacheFallbackError(e)) { - throw e; - } - if (version || environment) { - throw new Error(`Parameters not found with specified parameters: ${e}`); - } - - debugLogger - .forState(state) - .warn("Failed to load parameters, attempting to fall back to cache:", e); - let parameters; - if (id) { - parameters = await parametersCache.get({ id }); - if (!parameters) { - throw new Error( - `Parameters with id ${id} not found (not found on server or in local cache): ${e}`, - ); - } - } else { - parameters = await parametersCache.get({ - slug, - projectId, - projectName, - version: version ?? "latest", - }); - if (!parameters) { - throw new Error( - `Parameters ${slug} (version ${version ?? "latest"}) not found in ${[ - projectName ?? projectId, - ]} (not found on server or in local cache): ${e}`, - ); - } - } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return parameters as RemoteEvalParameters>; - } - const { response } = result; - - if (!("objects" in response) || response.objects.length === 0) { - if (id) { - throw new Error(`Parameters with id ${id} not found.`); - } else { - throw new Error( - `Parameters ${slug} not found in ${[projectName ?? projectId]}`, - ); - } - } else if (response.objects.length > 1) { - if (id) { - throw new Error( - `Multiple parameters found with id ${id}. This should never happen.`, - ); - } else { - throw new Error( - `Multiple parameters found with slug ${slug} in project ${ - projectName ?? projectId - }. This should never happen.`, - ); - } - } - - const metadata = parametersRowSchema.parse(response["objects"][0]); - const parameters = new RemoteEvalParameters(metadata); - try { - if (id) { - await parametersCache.set({ id }, parameters); - } else if (slug) { - await parametersCache.set( - { slug, projectId, projectName, version: version ?? "latest" }, - parameters, - ); - } - } catch (e) { - debugLogger.forState(state).warn("Failed to set parameters in cache:", e); - } - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return parameters as RemoteEvalParameters>; -} - /** * Options for logging in to Braintrust. */ @@ -9517,99 +9147,6 @@ export class Prompt< } } -export interface SerializedParameters { - metadata: ParametersRow; -} - -export let serializeParametersForCache: ( - parameters: RemoteEvalParameters, -) => SerializedParameters; - -export class RemoteEvalParameters< - HasId extends boolean = true, - HasVersion extends boolean = true, - T extends Record = Record, -> { - private readonly __braintrust_parameters_marker = true; - - constructor(private metadata: ParametersRow) { - void this.__braintrust_parameters_marker; - } - - public get id(): HasId extends true ? string : string | undefined { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return this.metadata.id as HasId extends true ? string : string | undefined; - } - - public get projectId(): string | undefined { - return this.metadata.project_id; - } - - public get name(): string { - return this.metadata.name; - } - - public get slug(): string { - return this.metadata.slug; - } - - public get version(): HasVersion extends true - ? TransactionId - : TransactionId | undefined { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return this.metadata[TRANSACTION_ID_FIELD] as HasVersion extends true - ? TransactionId - : TransactionId | undefined; - } - - public get schema(): Record { - return this.metadata.function_data.__schema; - } - - public get data(): T { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return (this.metadata.function_data.data ?? {}) as T; - } - - static { - serializeParametersForCache = (parameters) => ({ - metadata: parameters.metadata, - }); - } - - public validate(data: unknown): boolean { - if (typeof data !== "object" || data === null) { - return false; - } - const schemaProps = this.schema.properties; - if (typeof schemaProps !== "object" || schemaProps === null) { - return true; - } - for (const key of Object.keys(schemaProps)) { - if (!(key in data)) { - const required = Array.isArray(this.schema.required) - ? this.schema.required - : []; - if (required.includes(key)) { - return false; - } - } - } - return true; - } - - public static isParameters( - x: unknown, - ): x is RemoteEvalParameters> { - return ( - typeof x === "object" && - x !== null && - "__braintrust_parameters_marker" in x && - x.__braintrust_parameters_marker === true - ); - } -} - export type AnyDataset = Dataset; /** diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts deleted file mode 100644 index 69576dd9a..000000000 --- a/js/src/parameters.test.ts +++ /dev/null @@ -1,295 +0,0 @@ -import { expect, test, beforeAll } from "vitest"; -import { validateParameters } from "./eval-parameters"; -import { runEvaluator } from "./framework"; -import { RemoteEvalParameters } from "./logger"; -import { z } from "zod/v3"; -import { type ProgressReporter } from "./reporters/types"; -import { configureNode } from "./node/config"; - -beforeAll(() => { - configureNode(); -}); - -class NoopProgressReporter implements ProgressReporter { - public start() {} - public stop() {} - public increment() {} -} - -test("parameters are passed to task", async () => { - const out = await runEvaluator( - null, - { - projectName: "test-parameters", - evalName: "test", - data: [{ input: "hello" }], - task: async (input: string, { parameters }) => { - const output = `${parameters.prefix}${input}${parameters.suffix}`; - return output; - }, - scores: [], - classifiers: [], - parameters: { - prefix: z.string().default("start:"), - suffix: z.string().default(":end"), - }, - }, - new NoopProgressReporter(), - undefined, - { prefix: "start:", suffix: ":end" }, - true, - ); - - expect(out.results).toHaveLength(1); - expect(out.results[0].output).toBe("start:hello:end"); -}); - -test("prompt parameter is passed correctly", async () => { - const result = await runEvaluator( - null, - { - projectName: "test-prompt-parameter", - evalName: "test", - data: [{ input: "test input" }], - task: async (input: string, { parameters }) => { - // Verify the prompt parameter has the expected structure - if (!parameters.main || typeof parameters.main.build !== "function") { - throw new Error( - "Prompt parameter 'main' is missing or does not have a build function", - ); - } - return input; - }, - scores: [], - classifiers: [], - parameters: { - main: { - type: "prompt", - name: "Main prompt", - description: "Test prompt", - default: { - messages: [ - { - role: "user", - content: "{{input}}", - }, - ], - model: "gpt-4", - }, - }, - }, - }, - new NoopProgressReporter(), - undefined, - undefined, - true, - ); - - expect(result.results).toHaveLength(1); - expect(result.results[0].output).toBe("test input"); -}); - -test("remote prompt parameter is rehydrated correctly", async () => { - const parameters = new RemoteEvalParameters< - true, - true, - { - main: { - build: (args: { input: string }) => { - messages: Array<{ role: string; content: string }>; - model?: string; - }; - }; - } - >({ - id: "11111111-1111-4111-8111-111111111111", - _xact_id: "v1", - project_id: "22222222-2222-4222-8222-222222222222", - name: "Saved parameters", - slug: "saved-parameters", - function_type: "parameters", - function_data: { - type: "parameters", - data: { - main: { - prompt: { - type: "chat", - messages: [{ role: "user", content: "{{input}}" }], - }, - options: { - model: "gpt-5-mini", - }, - }, - }, - __schema: { - type: "object", - properties: { - main: { - type: "object", - "x-bt-type": "prompt", - }, - }, - additionalProperties: true, - }, - }, - }); - - const validated = await validateParameters({}, parameters); - - expect((validated.main as any).build({ input: "test input" })).toMatchObject({ - messages: [{ role: "user", content: "test input" }], - model: "gpt-5-mini", - }); -}); - -test("custom parameter values override defaults", async () => { - const result = await runEvaluator( - null, - { - projectName: "test-custom-parameters", - evalName: "test", - data: [{ input: "hello" }], - task: async (input: string, { parameters }) => { - const output = `${parameters.prefix}${input}${parameters.suffix}`; - return output; - }, - scores: [], - classifiers: [], - parameters: { - prefix: z.string().default("start:"), - suffix: z.string().default(":end"), - }, - }, - new NoopProgressReporter(), - undefined, - { - prefix: "custom:", - suffix: ":custom", - }, - true, - ); - - expect(result.results).toHaveLength(1); - expect(result.results[0].output).toBe("custom:hello:custom"); -}); - -test("array parameter is handled correctly", async () => { - const result = await runEvaluator( - null, - { - projectName: "test-array-parameter", - evalName: "test", - data: [{ input: "test" }], - task: async (input: string, { parameters }) => { - expect(Array.isArray(parameters.items)).toBe(true); - expect(parameters.items).toEqual(["item1", "item2"]); - return input; - }, - scores: [], - classifiers: [], - parameters: { - items: z.array(z.string()).default(["item1", "item2"]), - }, - }, - new NoopProgressReporter(), - undefined, - undefined, - true, - ); - - expect(result.results).toHaveLength(1); - expect(result.results[0].output).toBe("test"); -}); - -test("object parameter is handled correctly", async () => { - const result = await runEvaluator( - null, - { - projectName: "test-object-parameter", - evalName: "test", - data: [{ input: "test" }], - task: async (input: string, { parameters }) => { - expect(parameters.config).toEqual({ - name: "test", - value: 123, - }); - return input; - }, - scores: [], - classifiers: [], - parameters: { - config: z - .object({ - name: z.string(), - value: z.number(), - }) - .default({ - name: "test", - value: 123, - }), - }, - }, - new NoopProgressReporter(), - undefined, - undefined, - true, - ); - - expect(result.results).toHaveLength(1); - expect(result.results[0].output).toBe("test"); -}); - -test("model parameter defaults to configured value", async () => { - const result = await runEvaluator( - null, - { - projectName: "test-model-parameter-default", - evalName: "test", - data: [{ input: "test" }], - task: async (input: string, { parameters }) => { - expect(parameters.model).toBe("gpt-5-mini"); - return input; - }, - scores: [], - classifiers: [], - parameters: { - model: { - type: "model", - default: "gpt-5-mini", - }, - }, - }, - new NoopProgressReporter(), - undefined, - undefined, - true, - ); - - expect(result.results).toHaveLength(1); - expect(result.results[0].output).toBe("test"); -}); - -test("model parameter is required when default is missing", async () => { - await expect( - runEvaluator( - null, - { - projectName: "test-model-parameter-required", - evalName: "test", - data: [{ input: "test" }], - task: async (input: string) => input, - scores: [], - classifiers: [], - parameters: { - model: { - type: "model", - }, - }, - }, - new NoopProgressReporter(), - undefined, - undefined, - true, - ), - ).rejects.toThrow("Parameter 'model' is required"); -}); diff --git a/js/src/prompt-cache/cache-config.ts b/js/src/prompt-cache/cache-config.ts index 66dcc5e87..06f2559ba 100644 --- a/js/src/prompt-cache/cache-config.ts +++ b/js/src/prompt-cache/cache-config.ts @@ -2,6 +2,11 @@ import { debugLogger } from "../debug-logger"; import iso from "../isomorph"; import { canUseDiskCache, DiskCache } from "./disk-cache"; import { LRUCache } from "../lru-cache"; +import { + PromptCache, + type PromptMemoryCacheEntry, + type PromptDiskCacheEntry, +} from "./prompt-cache"; type CacheMode = "mixed" | "memory" | "disk" | "none"; @@ -28,7 +33,7 @@ function warnUnavailableDiskCache() { } warnedUnavailableDiskCacheMode = true; debugLogger.warn( - `Disk cache is not supported on this platform, so ${CACHE_LOCATION_ENV_VAR}="disk" disables prompt and parameters caching.`, + `Disk cache is not supported on this platform, so ${CACHE_LOCATION_ENV_VAR}="disk" disables prompt caching.`, ); } @@ -55,45 +60,34 @@ function parsePositiveIntegerEnv(envVar: string, defaultValue: number): number { return Number.isInteger(value) && value > 0 ? value : defaultValue; } -export function createCacheLayers({ - memoryMaxEnvVar, - diskCacheDirEnvVar, - diskMaxEnvVar, - getDefaultDiskCacheDir, -}: { - memoryMaxEnvVar: string; - diskCacheDirEnvVar: string; - diskMaxEnvVar: string; - getDefaultDiskCacheDir: () => string; -}): { - memoryCache?: LRUCache; - diskCache?: DiskCache; -} { +export function createPromptCache(): PromptCache { const mode = parseCacheMode(); const memoryCache = mode === "mixed" || mode === "memory" - ? new LRUCache({ + ? new LRUCache({ max: parsePositiveIntegerEnv( - memoryMaxEnvVar, + "BRAINTRUST_PROMPT_CACHE_MEMORY_MAX", DEFAULT_CACHE_MEMORY_MAX, ), }) : undefined; - let diskCache: DiskCache | undefined; + let diskCache: DiskCache | undefined; if (mode === "mixed" || mode === "disk") { if (canUseDiskCache()) { - diskCache = new DiskCache({ - cacheDir: iso.getEnv(diskCacheDirEnvVar) ?? getDefaultDiskCacheDir(), - max: parsePositiveIntegerEnv(diskMaxEnvVar, DEFAULT_CACHE_DISK_MAX), + diskCache = new DiskCache({ + cacheDir: + iso.getEnv("BRAINTRUST_PROMPT_CACHE_DIR") ?? + `${iso.getEnv("HOME") ?? iso.homedir!()}/.braintrust/prompt_cache`, + max: parsePositiveIntegerEnv( + "BRAINTRUST_PROMPT_CACHE_DISK_MAX", + DEFAULT_CACHE_DISK_MAX, + ), }); } else if (mode === "disk") { warnUnavailableDiskCache(); } } - if (diskCache) { - return { memoryCache, diskCache }; - } - return { memoryCache }; + return new PromptCache({ memoryCache, diskCache }); } diff --git a/js/src/prompt-cache/cache-mode.test.ts b/js/src/prompt-cache/cache-mode.test.ts index 0a585b0c2..e19dc7b89 100644 --- a/js/src/prompt-cache/cache-mode.test.ts +++ b/js/src/prompt-cache/cache-mode.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "os"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import iso from "../isomorph"; import { configureNode } from "../node/config"; -import { BraintrustState, Prompt, RemoteEvalParameters } from "../logger"; +import { BraintrustState, Prompt } from "../logger"; configureNode(); @@ -14,9 +14,6 @@ const CACHE_ENV_VARS = [ "BRAINTRUST_PROMPT_CACHE_DIR", "BRAINTRUST_PROMPT_CACHE_MEMORY_MAX", "BRAINTRUST_PROMPT_CACHE_DISK_MAX", - "BRAINTRUST_PARAMETERS_CACHE_DIR", - "BRAINTRUST_PARAMETERS_CACHE_MEMORY_MAX", - "BRAINTRUST_PARAMETERS_CACHE_DISK_MAX", ]; const promptKey = { @@ -25,12 +22,6 @@ const promptKey = { version: "v1", }; -const parametersKey = { - projectId: "22222222-2222-4222-8222-222222222222", - slug: "saved-parameters", - version: "v1", -}; - const testPrompt = new Prompt( { id: "33333333-3333-4333-8333-333333333333", @@ -43,27 +34,7 @@ const testPrompt = new Prompt( false, ); -const testParameters = new RemoteEvalParameters({ - id: "44444444-4444-4444-8444-444444444444", - _xact_id: "v1", - project_id: parametersKey.projectId, - name: "Saved parameters", - slug: parametersKey.slug, - description: null, - function_type: "parameters", - function_data: { - type: "parameters", - data: { prefix: "hello" }, - __schema: { - type: "object", - properties: { - prefix: { type: "string" }, - }, - }, - }, -}); - -describe("prompt and parameters cache modes", () => { +describe("prompt cache modes", () => { const originalEnv = new Map(); const originalGzip = iso.gzip; const testDirs: string[] = []; @@ -104,87 +75,61 @@ describe("prompt and parameters cache modes", () => { async function configureCacheDirs() { const promptDir = await makeTempDir(); - const parametersDir = await makeTempDir(); process.env.BRAINTRUST_PROMPT_CACHE_DIR = promptDir; - process.env.BRAINTRUST_PARAMETERS_CACHE_DIR = parametersDir; - return { promptDir, parametersDir }; + return { promptDir }; } it("mixed mode uses memory and disk caches", async () => { process.env.BRAINTRUST_CACHE_LOCATION = " MiXeD "; process.env.BRAINTRUST_PROMPT_CACHE_MEMORY_MAX = "1"; - process.env.BRAINTRUST_PARAMETERS_CACHE_MEMORY_MAX = "1"; await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); await state.promptCache.set({ ...promptKey, slug: "other" }, testPrompt); - await state.parametersCache.set(parametersKey, testParameters); - await state.parametersCache.set( - { ...parametersKey, slug: "other" }, - testParameters, - ); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toEqual(testPrompt); - expect(await nextState.parametersCache.get(parametersKey)).toEqual( - testParameters, - ); }); it("memory mode does not write to disk", async () => { process.env.BRAINTRUST_CACHE_LOCATION = "memory"; - const { promptDir, parametersDir } = await configureCacheDirs(); + const { promptDir } = await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); - await state.parametersCache.set(parametersKey, testParameters); expect(await state.promptCache.get(promptKey)).toEqual(testPrompt); - expect(await state.parametersCache.get(parametersKey)).toEqual( - testParameters, - ); expect(await fs.readdir(promptDir)).toEqual([]); - expect(await fs.readdir(parametersDir)).toEqual([]); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toBeUndefined(); - expect(await nextState.parametersCache.get(parametersKey)).toBeUndefined(); }); it("disk mode does not warm memory", async () => { process.env.BRAINTRUST_CACHE_LOCATION = "disk"; - const { promptDir, parametersDir } = await configureCacheDirs(); + const { promptDir } = await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); - await state.parametersCache.set(parametersKey, testParameters); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toEqual(testPrompt); - expect(await nextState.parametersCache.get(parametersKey)).toEqual( - testParameters, - ); await fs.rm(promptDir, { recursive: true, force: true }); - await fs.rm(parametersDir, { recursive: true, force: true }); expect(await nextState.promptCache.get(promptKey)).toBeUndefined(); - expect(await nextState.parametersCache.get(parametersKey)).toBeUndefined(); }); it("none mode disables all cache reads and writes", async () => { process.env.BRAINTRUST_CACHE_LOCATION = "none"; - const { promptDir, parametersDir } = await configureCacheDirs(); + const { promptDir } = await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); - await state.parametersCache.set(parametersKey, testParameters); expect(await state.promptCache.get(promptKey)).toBeUndefined(); - expect(await state.parametersCache.get(parametersKey)).toBeUndefined(); expect(await fs.readdir(promptDir)).toEqual([]); - expect(await fs.readdir(parametersDir)).toEqual([]); }); it("invalid cache mode warns once and falls back to mixed", async () => { @@ -195,13 +140,9 @@ describe("prompt and parameters cache modes", () => { const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); - await state.parametersCache.set(parametersKey, testParameters); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toEqual(testPrompt); - expect(await nextState.parametersCache.get(parametersKey)).toEqual( - testParameters, - ); expect(warnSpy).toHaveBeenCalledTimes(1); expect(warnSpy).toHaveBeenCalledWith( @@ -220,30 +161,14 @@ describe("prompt and parameters cache modes", () => { { ...promptKey, slug: `prompt-${i}` }, testPrompt, ); - await state.parametersCache.set( - { ...parametersKey, slug: `parameters-${i}` }, - testParameters, - ); } expect( await state.promptCache.get({ ...promptKey, slug: "prompt-0" }), ).toBeUndefined(); - expect( - await state.parametersCache.get({ - ...parametersKey, - slug: "parameters-0", - }), - ).toBeUndefined(); expect( await state.promptCache.get({ ...promptKey, slug: "prompt-1024" }), ).toEqual(testPrompt); - expect( - await state.parametersCache.get({ - ...parametersKey, - slug: "parameters-1024", - }), - ).toEqual(testParameters); }); it("disk mode disables caching and warns once when disk cache is unavailable", async () => { @@ -255,14 +180,12 @@ describe("prompt and parameters cache modes", () => { const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); expect(await state.promptCache.get(promptKey)).toBeUndefined(); - await state.parametersCache.set(parametersKey, testParameters); - expect(await state.parametersCache.get(parametersKey)).toBeUndefined(); new BraintrustState({}); expect(warnSpy).toHaveBeenCalledTimes(1); expect(warnSpy).toHaveBeenCalledWith( "[braintrust]", - 'Disk cache is not supported on this platform, so BRAINTRUST_CACHE_LOCATION="disk" disables prompt and parameters caching.', + 'Disk cache is not supported on this platform, so BRAINTRUST_CACHE_LOCATION="disk" disables prompt caching.', ); }); }); diff --git a/js/src/prompt-cache/parameters-cache.test.ts b/js/src/prompt-cache/parameters-cache.test.ts deleted file mode 100644 index 7fe320871..000000000 --- a/js/src/prompt-cache/parameters-cache.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import * as fs from "fs/promises"; -import * as path from "path"; -import { tmpdir } from "os"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { RemoteEvalParameters } from "../logger"; -import { configureNode } from "../node/config"; -import { DiskCache } from "./disk-cache"; -import { - ParametersCache, - type ParametersDiskCacheEntry, -} from "./parameters-cache"; - -describe("ParametersCache", () => { - configureNode(); - - let cacheDir: string; - - beforeEach(() => { - cacheDir = path.join( - tmpdir(), - `parameters-cache-test-${Date.now()}-${Math.random()}`, - ); - }); - - afterEach(async () => { - await fs.rm(cacheDir, { recursive: true, force: true }); - }); - - it("reconstructs parameters loaded from disk", async () => { - const key = { - projectId: "55555555-5555-4555-8555-555555555555", - slug: "saved-parameters", - version: "v1", - }; - const parameters = new RemoteEvalParameters({ - id: "44444444-4444-4444-8444-444444444444", - _xact_id: "v1", - project_id: key.projectId, - name: "Saved parameters", - slug: key.slug, - description: null, - function_type: "parameters", - function_data: { - type: "parameters", - data: { prefix: "hello" }, - __schema: { - type: "object", - properties: { prefix: { type: "string" } }, - required: ["prefix"], - }, - }, - }); - const diskCache = new DiskCache({ - cacheDir, - logWarnings: false, - }); - - await new ParametersCache({ diskCache }).set(key, parameters); - const result = await new ParametersCache({ diskCache }).get(key); - - expect(result).toBeInstanceOf(RemoteEvalParameters); - expect(result?.data).toEqual({ prefix: "hello" }); - expect(result?.schema).toEqual({ - type: "object", - properties: { prefix: { type: "string" } }, - required: ["prefix"], - }); - }); -}); diff --git a/js/src/prompt-cache/parameters-cache.ts b/js/src/prompt-cache/parameters-cache.ts deleted file mode 100644 index 530cfb28a..000000000 --- a/js/src/prompt-cache/parameters-cache.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { - RemoteEvalParameters, - serializeParametersForCache, - type SerializedParameters, -} from "../logger"; -import { LRUCache } from "../lru-cache"; -import { DiskCache } from "./disk-cache"; - -interface ParametersKey { - slug?: string; - version?: string; - projectId?: string; - projectName?: string; - id?: string; -} - -function createCacheKey(key: ParametersKey, namespace?: string): string { - let cacheKey: string; - if (key.id) { - cacheKey = `parameters:id:${key.id}`; - } else { - const prefix = key.projectId ?? key.projectName; - if (!prefix) { - throw new Error("Either projectId or projectName must be provided"); - } - if (!key.slug) { - throw new Error("Slug must be provided when not using ID"); - } - cacheKey = `parameters:${prefix}:${key.slug}:${key.version ?? "latest"}`; - } - return namespace === undefined - ? cacheKey - : `${namespace.length}:${namespace}:${cacheKey}`; -} - -export type ParametersMemoryCacheEntry = { - value: RemoteEvalParameters; - resolvedOrgIdentity?: string; -}; - -export type ParametersDiskCacheEntry = { - value: SerializedParameters; - resolvedOrgIdentity?: string; -}; - -export class ParametersCache { - private readonly memoryCache?: LRUCache; - private readonly diskCache?: DiskCache; - private readonly namespace?: string; - private readonly expectedResolvedOrgIdentity?: string; - - constructor(options: { - memoryCache?: LRUCache; - diskCache?: DiskCache; - namespace?: string; - expectedResolvedOrgIdentity?: string; - }) { - this.memoryCache = options.memoryCache; - this.diskCache = options.diskCache; - this.namespace = options.namespace; - this.expectedResolvedOrgIdentity = options.expectedResolvedOrgIdentity; - } - - withNamespace( - namespace: string, - expectedResolvedOrgIdentity?: string, - ): ParametersCache { - return new ParametersCache({ - memoryCache: this.memoryCache, - diskCache: this.diskCache, - namespace, - expectedResolvedOrgIdentity, - }); - } - - async get(key: ParametersKey): Promise { - const cacheKey = createCacheKey(key, this.namespace); - if (this.memoryCache) { - const memoryEntry = this.memoryCache.get(cacheKey); - if ( - memoryEntry !== undefined && - (this.expectedResolvedOrgIdentity === undefined || - memoryEntry.resolvedOrgIdentity === this.expectedResolvedOrgIdentity) - ) { - return memoryEntry.value; - } - } - - if (this.diskCache) { - const diskEntry = await this.diskCache.get(cacheKey); - if ( - !diskEntry || - (this.expectedResolvedOrgIdentity !== undefined && - diskEntry.resolvedOrgIdentity !== this.expectedResolvedOrgIdentity) - ) { - return undefined; - } - const diskParameters = new RemoteEvalParameters(diskEntry.value.metadata); - this.memoryCache?.set(cacheKey, { - value: diskParameters, - resolvedOrgIdentity: diskEntry.resolvedOrgIdentity, - }); - return diskParameters; - } - - return undefined; - } - - async set(key: ParametersKey, value: RemoteEvalParameters): Promise { - const cacheKey = createCacheKey(key, this.namespace); - const memoryEntry = { - value, - resolvedOrgIdentity: this.expectedResolvedOrgIdentity, - }; - this.memoryCache?.set(cacheKey, memoryEntry); - if (this.diskCache) { - await this.diskCache.set(cacheKey, { - value: serializeParametersForCache(value), - resolvedOrgIdentity: this.expectedResolvedOrgIdentity, - }); - } - } -} diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts deleted file mode 100644 index 94a433212..000000000 --- a/js/src/prompt-schemas.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { - ToolFunctionDefinition, - ChatCompletionMessageParam, - ModelParams, - PromptBlockData, - PromptData, -} from "./sdk-types"; - -// This roughly maps to promptBlockDataSchema, but is more ergonomic for the user. -export type PromptContents = - | { prompt: string } - | { messages: ChatCompletionMessageParam[] }; - -export type PromptDefinition = PromptContents & { - model: string; - params?: ModelParams; - templateFormat?: "mustache" | "nunjucks" | "none"; - environments?: string[]; -}; - -export type PromptDefinitionWithTools = PromptDefinition & { - tools?: ToolFunctionDefinition[]; -}; - -export function promptDefinitionToPromptData( - promptDefinition: PromptDefinition, - rawTools?: ToolFunctionDefinition[], -): PromptData { - const promptBlock: PromptBlockData = - "messages" in promptDefinition - ? { - type: "chat", - messages: promptDefinition.messages, - tools: - rawTools && rawTools.length > 0 - ? JSON.stringify(rawTools) - : undefined, - } - : { - type: "completion", - content: promptDefinition.prompt, - }; - - return { - prompt: promptBlock, - options: { - model: promptDefinition.model, - params: promptDefinition.params, - }, - ...(promptDefinition.templateFormat - ? { template_format: promptDefinition.templateFormat } - : {}), - }; -} diff --git a/js/src/prompt-types.test.ts b/js/src/prompt-types.test.ts index 51df00f1b..9d0c23239 100644 --- a/js/src/prompt-types.test.ts +++ b/js/src/prompt-types.test.ts @@ -8,10 +8,9 @@ import { type DefaultPromptArgs, type PromptRowWithId, } from "./logger"; -import type { CodeParameters, CodePrompt } from "./framework2"; +import type { CodePrompt } from "./framework2"; import type { AnyModelParamsType, - FunctionDataType, PromptDataType, PromptType, } from "./generated_plain_types"; @@ -85,9 +84,10 @@ test("compiled prompts preserve provider parameter types and compatibility", () test("prompt metadata preserves required and optional identifiers", () => { type Metadata = Omit< PromptType, - "log_id" | "org_id" | "project_id" | "id" | "_xact_id" + "log_id" | "org_id" | "project_id" | "id" | "_xact_id" | "function_type" > & { project_id?: string; + function_type?: Exclude; }; type RequiredRow = Metadata & { id: string; _xact_id: string }; type OptionalRow = Metadata & { id?: string; _xact_id?: string }; @@ -114,22 +114,10 @@ test("prompt metadata preserves required and optional identifiers", () => { test("function definitions expose only the variant each builder produces", () => { type PromptEvent = Awaited>; - type ParametersEvent = Awaited< - ReturnType - >; expectTypeOf().toEqualTypeOf<{ type: "prompt"; }>(); - expectTypeOf< - PromptEvent["prompt_data"] - >().branded.toEqualTypeOf(); - expectTypeOf< - ParametersEvent["function_data"]["type"] - >().toEqualTypeOf<"parameters">(); - expectTypeOf< - ParametersEvent["function_type"] - >().toEqualTypeOf<"parameters">(); - expectTypeOf().toMatchTypeOf< - Extract + expectTypeOf().branded.toEqualTypeOf< + Omit >(); }); diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts index 2870520fd..dcb170e49 100644 --- a/js/src/public-types.test.ts +++ b/js/src/public-types.test.ts @@ -36,12 +36,62 @@ test("exports only intentional extension types", () => { expectTypeOf().toBeObject(); expectTypeOf().toBeObject(); expectTypeOf(BraintrustLangChainCallbackHandler).toBeConstructibleWith(); - expectTypeOf< - EvalTask - >().toBeFunction(); + expectTypeOf>().toBeFunction(); expectTypeOf>().toBeFunction(); }); +test("does not expose deprecated tools or evaluation parameters", () => { + expect(publicExports).not.toHaveProperty("loadParameters"); + const project = publicExports.projects.create({ name: "test-project" }); + expect(project).not.toHaveProperty("tools"); + expect(project).not.toHaveProperty("parameters"); + + if (false) { + // @ts-expect-error Saved configurations are no longer supported. + publicExports.loadParameters; + // @ts-expect-error Saved tools are no longer supported. + project.tools; + // @ts-expect-error Configuration builders are no longer supported. + project.parameters; + + const evaluator = { + data: [{ input: 1 }], + task: (input: number) => input, + scores: [], + }; + publicExports.Eval("test", { + ...evaluator, + // @ts-expect-error Local parameter schemas are no longer supported. + parameters: {}, + }); + publicExports.Eval("test", evaluator, { + // @ts-expect-error Parameter overrides are no longer supported. + parameters: {}, + }); + publicExports.Eval("test", { + ...evaluator, + task: (input, hooks) => { + // @ts-expect-error Tasks no longer receive parameters. + hooks.parameters; + return input; + }, + }); + project.prompts.create({ + name: "test-prompt", + model: "test-model", + messages: [], + // @ts-expect-error Prompts accept inline provider tools, not saved tools. + tools: [{ type: "function", id: "saved-tool" }], + }); + publicExports.invoke({ + globalFunction: "test", + input: {}, + // @ts-expect-error Deprecated function kinds are no longer supported. + functionType: "tool", + }); + } +}); + test("does not expose runtime schemas or implementation helpers", () => { for (const name of [ "AttachmentReference", diff --git a/js/src/reporters/types.ts b/js/src/reporters/types.ts index a85be9bdb..7909dfbfd 100644 --- a/js/src/reporters/types.ts +++ b/js/src/reporters/types.ts @@ -24,7 +24,7 @@ export interface ReporterBody { // These any's are required because these function specifications don't know // or need to know the types of the input/output/etc for the evaluator. // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, + evaluator: EvaluatorDef, // eslint-disable-next-line @typescript-eslint/no-explicit-any result: EvalResultWithSummary, opts: ReporterOpts, diff --git a/js/src/sdk-contracts.test.ts b/js/src/sdk-contracts.test.ts index 00ec7cd07..4430cf133 100644 --- a/js/src/sdk-contracts.test.ts +++ b/js/src/sdk-contracts.test.ts @@ -6,20 +6,24 @@ import type * as SDK from "./sdk-types"; import type * as schemas from "./sdk-schemas"; // Backend definitions are compatibility fixtures, never production dependencies. +// Function contracts intentionally exclude the deprecated tool and parameters kinds. test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf< + Exclude + >(); expectTypeOf< Exclude >().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + Exclude + >(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -30,7 +34,9 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf< + Omit + >(); expectTypeOf< Exclude >().toEqualTypeOf(); @@ -44,7 +50,6 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -53,11 +58,16 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf | null>(); + expectTypeOf().toEqualTypeOf< + Exclude + >(); + expectTypeOf().toExtend(); + expectTypeOf().toExtend(); + expectTypeOf().toExtend(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -67,10 +77,10 @@ test("SDK-owned contracts remain compatible with the backend", () => { }); test("runtime validators accept and return compatible types", () => { - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); expectTypeOf>().toEqualTypeOf< @@ -79,10 +89,10 @@ test("runtime validators accept and return compatible types", () => { expectTypeOf>().toEqualTypeOf< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); expectTypeOf>().toEqualTypeOf< @@ -147,16 +157,16 @@ test("runtime validators accept and return compatible types", () => { expectTypeOf>().toEqualTypeOf< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); expectTypeOf>().toEqualTypeOf< @@ -171,10 +181,22 @@ test("runtime validators accept and return compatible types", () => { expectTypeOf>().toEqualTypeOf< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); }); + +test("SDK validators match the supported function contracts", () => { + expectTypeOf< + z.output + >().toExtend(); + expectTypeOf< + z.output + >().toExtend(); + expectTypeOf< + z.output + >().toEqualTypeOf(); +}); diff --git a/js/src/sdk-schemas.ts b/js/src/sdk-schemas.ts index 70ae00ae1..f1d447e41 100644 --- a/js/src/sdk-schemas.ts +++ b/js/src/sdk-schemas.ts @@ -53,13 +53,11 @@ const FunctionTypeEnum = z.enum([ "llm", "scorer", "task", - "tool", "custom_view", "preprocessor", "facet", "classifier", "tag", - "parameters", "sandbox", ]); @@ -365,7 +363,6 @@ export const PromptData = z options: PromptOptionsNullish, parser: PromptParserNullish, preprocessor: PreprocessorId, - tool_functions: z.union([z.array(SavedFunctionId), z.null()]), template_format: z.union([ z.enum(["mustache", "nunjucks", "none"]), z.null(), @@ -442,7 +439,6 @@ export const FunctionId = z.union([ const FunctionObjectType = z.enum([ "prompt", - "tool", "scorer", "task", "workflow", @@ -450,7 +446,6 @@ const FunctionObjectType = z.enum([ "preprocessor", "facet", "classifier", - "parameters", "sandbox", ]); diff --git a/js/src/sdk-types.ts b/js/src/sdk-types.ts index 1fdc3f6a9..fec881e46 100644 --- a/js/src/sdk-types.ts +++ b/js/src/sdk-types.ts @@ -97,25 +97,12 @@ export type FunctionTypeEnum = | "llm" | "scorer" | "task" - | "tool" | "custom_view" | "preprocessor" | "facet" | "classifier" | "tag" - | "parameters" | "sandbox"; -export type SavedFunctionId = - | { - type: "function"; - id: string; - version?: string | undefined; - } - | { - type: "global"; - name: string; - function_type: FunctionTypeEnum; - }; export type CallEvent = | { id?: string | undefined; @@ -276,14 +263,6 @@ export type RepoInfo = { commit_time?: string | null; git_diff?: string | null; } | null; -export type ExtendedSavedFunctionId = - | SavedFunctionId - | { - type: "slug"; - project_id: string; - slug: string; - }; -// Keep provider fields visible to editor completion while accepting custom options. export type ModelParams = | { use_cache?: boolean; @@ -400,7 +379,6 @@ export interface PromptData { options?: PromptOptionsNullish; parser?: PromptParserNullish; preprocessor?: PreprocessorId; - tool_functions?: Array | null; template_format?: ("mustache" | "nunjucks" | "none") | null; mcp?: {} | null; origin?: { @@ -450,7 +428,6 @@ export type FunctionId = }; export type FunctionObjectType = | "prompt" - | "tool" | "scorer" | "task" | "workflow" @@ -458,7 +435,6 @@ export type FunctionObjectType = | "preprocessor" | "facet" | "classifier" - | "parameters" | "sandbox"; export type FunctionOutputType = | "completion" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0ef57e08d..c5278b7df 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -288,9 +288,6 @@ importers: acorn-import-attributes: specifier: ^1.9.5 version: 1.9.5(acorn@8.16.0) - ajv: - specifier: ^8.20.0 - version: 8.20.0 astring: specifier: ^1.9.0 version: 1.9.0 From d6e49583e1da7af7bce57c9f7044d3288bca4fb0 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:08:32 +0000 Subject: [PATCH 2/4] Update PR #2609 --- .../durable-eval-webhook/scenario.ts | 14 +- js/src/durable-eval.test.ts | 179 +++++++++- js/src/durable-eval.ts | 247 ++++++++++--- js/src/eval-parameters.ts | 92 +++++ js/src/framework.test.ts | 16 + js/src/framework.ts | 106 ++++-- js/src/framework2.ts | 47 +-- js/src/parameters.test.ts | 336 ++++++++++++++++++ js/src/prompt-schemas.ts | 51 +++ js/src/public-types.test.ts | 25 +- js/src/reporters/types.ts | 2 +- 11 files changed, 971 insertions(+), 144 deletions(-) create mode 100644 js/src/eval-parameters.ts create mode 100644 js/src/parameters.test.ts create mode 100644 js/src/prompt-schemas.ts diff --git a/e2e/scenarios/durable-eval-webhook/scenario.ts b/e2e/scenarios/durable-eval-webhook/scenario.ts index 5e096aec4..f133ce36f 100644 --- a/e2e/scenarios/durable-eval-webhook/scenario.ts +++ b/e2e/scenarios/durable-eval-webhook/scenario.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { z } from "zod/v3"; import { BatchScorer, BatchTask, @@ -15,6 +16,7 @@ async function main() { const testRunId = getTestRunId(); const scenario = "durable-eval-webhook"; const store = new DurableEvalMemoryStore(); + const parameters = { factor: z.number().default(4) }; const jobs = new Map(); const webhookCompletion = { mode: "webhook" as const, @@ -24,12 +26,14 @@ async function main() { number, number, number, - { testRunId: string; kind: string } + { testRunId: string; kind: string }, + typeof parameters, + { id: string } >({ batchSize: 2, async submit(items) { for (const item of items) { - assert.equal("parameters" in item, false); + assert.equal(item.parameters.factor, 2); } const id = `task-${jobs.size + 1}`; jobs.set(id, items); @@ -40,10 +44,11 @@ async function main() { const items = (jobs.get(submissionData.id) ?? []) as Array<{ id: string; input: number; + parameters: { factor: number }; }>; return items.map((item) => ({ id: item.id, - output: item.input * 2, + output: item.input * item.parameters.factor, })); }, }); @@ -82,6 +87,7 @@ async function main() { scopedName("e2e-durable-eval-webhook-project", testRunId), { store, + parameters, experimentName: `${scenario}-${testRunId}`, data: [1, 2, 3].map((input) => ({ id: `case-${input}`, @@ -112,7 +118,7 @@ async function main() { }, ); - const waiting = await definition.start(); + const waiting = await definition.start({ parameters: { factor: 2 } }); if (waiting.status !== "waiting" || jobs.size !== 2) { throw new Error("Durable eval did not pause with two webhook batches"); } diff --git a/js/src/durable-eval.test.ts b/js/src/durable-eval.test.ts index f6a05813e..7a0d7f892 100644 --- a/js/src/durable-eval.test.ts +++ b/js/src/durable-eval.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test, vi } from "vitest"; +import { z } from "zod/v3"; import { configureNode } from "./node/config"; import { BatchScorer, @@ -177,6 +178,113 @@ describe("defineDurableEval", () => { expect(task).toHaveBeenCalledTimes(2); }); + test("validates inline parameters once for ordinary tasks", async () => { + const transform = vi.fn((value: number) => value + 1); + const result = await defineDurableEval("inline-parameters", { + store: new DurableEvalMemoryStore(), + data: [{ input: "hello", expected: "hello:2" }], + parameters: { + count: z.number().default(1).transform(transform), + prompt: { + type: "prompt", + default: { prompt: "{{input}}:{{count}}", model: "test-model" }, + }, + }, + task: (input, { parameters }) => + parameters.prompt.build( + { input, count: parameters.count }, + { flavor: "completion" }, + ).prompt, + scores: [ + function exact({ output, expected }) { + return output === expected ? 1 : 0; + }, + ], + }).start({ noSendLogs: true }); + expect(transform).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + status: "completed", + summary: { scores: { exact: { score: 1 } } }, + }); + }); + + test("batch tasks receive persisted inline parameter values", async () => { + const store = new DurableEvalMemoryStore(); + const schema = { + factor: z.number().default(4), + prompt: { + type: "prompt" as const, + default: { prompt: "{{input}}", model: "test-model" }, + }, + }; + let items: DurableBatchTaskItem[] = []; + const task = new BatchTask< + number, + number, + number, + void, + typeof schema, + null + >({ + async submit(submitted) { + items = submitted; + expect(items[0].parameters.factor).toBe(2); + expect( + items[0].parameters.prompt.build( + { input: "hello" }, + { flavor: "completion" }, + ).prompt, + ).toBe("hello"); + return null; + }, + completion: { mode: "webhook", getExternalId: () => "inline-job" }, + async collect() { + return items.map(({ id, input, parameters }) => ({ + id, + output: input * parameters.factor, + })); + }, + }); + const evaluator = { + store, + parameters: schema, + data: [{ id: "case-one", input: 3, expected: 6 }], + task, + scores: [ + function exact({ + output, + expected, + }: { + output: number; + expected?: number; + }) { + return output === expected ? 1 : 0; + }, + ], + }; + const { Prompt } = await import("./logger"); + const run = await defineDurableEval("batch-parameters", evaluator).start({ + noSendLogs: true, + parameters: { + factor: 2, + prompt: Prompt.fromPromptData("main", { + prompt: { type: "completion", content: "{{input}}" }, + options: { model: "test-model" }, + }), + }, + }); + expect(run.status).toBe("waiting"); + const resumed = await defineDurableEval( + "batch-parameters", + evaluator, + ).processBatchResult({ runId: run.runId, externalId: "inline-job" }); + expect(resumed.status).toBe("completed"); + expect(resumed).toMatchObject({ + status: "completed", + summary: { scores: { exact: { score: 1 } } }, + }); + }); + test("generates a new run id for every start", async () => { const durable = defineDurableEval("generated-runs", { store: new DurableEvalMemoryStore(), @@ -193,9 +301,20 @@ describe("defineDurableEval", () => { test("merges batch task metadata with case metadata", async () => { type Metadata = { fromCase?: boolean; fromTask?: boolean }; - let taskItems: DurableBatchTaskItem[] = []; + let taskItems: DurableBatchTaskItem< + number, + void, + Metadata, + Record + >[] = []; let scorerMetadata: Metadata | undefined; - const task = new BatchTask({ + const task = new BatchTask< + number, + number, + void, + Metadata, + Record + >({ async submit(items) { taskItems = items; return null; @@ -248,7 +367,14 @@ describe("defineDurableEval", () => { return { value, created: true }; }, }; - const task = new BatchTask({ + const task = new BatchTask< + number, + number, + void, + void, + Record, + { id: string } + >({ async submit() { return { id: "provider-job" }; }, @@ -290,7 +416,6 @@ describe("defineDurableEval", () => { caseIds: ["one:trial:0", "two:trial:0"], }); expect(run).not.toHaveProperty("schemaVersion"); - expect(run).not.toHaveProperty("parameters"); expect(run).not.toHaveProperty("projectName"); expect(run).not.toHaveProperty("evalName"); expect(run).not.toHaveProperty("cases"); @@ -308,7 +433,7 @@ describe("defineDurableEval", () => { test("polls each existing task and scorer sub-batch once", async () => { const taskJobs = new Map< string, - DurableBatchTaskItem[] + DurableBatchTaskItem>[] >(); const scoreJobs = new Map< string, @@ -317,7 +442,14 @@ describe("defineDurableEval", () => { const taskPoll = vi.fn(async () => ({ status: "complete" as const })); const scorePoll = vi.fn(async () => ({ status: "complete" as const })); - const task = new BatchTask({ + const task = new BatchTask< + number, + number, + number, + void, + Record, + { id: string } + >({ batchSize: 2, async submit(items) { const id = `task-${taskJobs.size + 1}`; @@ -421,7 +553,7 @@ describe("defineDurableEval", () => { const store = new DurableEvalMemoryStore(); const taskJobs = new Map< string, - DurableBatchTaskItem[] + DurableBatchTaskItem>[] >(); const scoreJobs = new Map< string, @@ -432,7 +564,14 @@ describe("defineDurableEval", () => { const taskBatchesCollecting = new Promise((resolve) => { releaseTaskCollect = resolve; }); - const task = new BatchTask({ + const task = new BatchTask< + number, + number, + number, + void, + Record, + { id: string } + >({ batchSize: 2, async submit(items) { const id = `task-provider-${taskJobs.size + 1}`; @@ -529,13 +668,25 @@ describe("defineDurableEval", () => { }); test("claims downstream work once across concurrent webhook deliveries", async () => { - let taskItems: DurableBatchTaskItem[] = []; + let taskItems: DurableBatchTaskItem< + number, + number, + void, + Record + >[] = []; let collectCount = 0; let releaseCollect!: () => void; const bothCollecting = new Promise((resolve) => { releaseCollect = resolve; }); - const task = new BatchTask({ + const task = new BatchTask< + number, + number, + number, + void, + Record, + { id: string } + >({ async submit(items) { taskItems = items; return { id: "task-provider" }; @@ -611,7 +762,13 @@ describe("defineDurableEval", () => { return { status: "complete" as const }; }, }; - const task = new BatchTask({ + const task = new BatchTask< + number, + number, + number, + void, + Record + >({ async submit(items) { jobs.set("task", items); return { id: "task" }; diff --git a/js/src/durable-eval.ts b/js/src/durable-eval.ts index ba2b49d0b..632849fd5 100644 --- a/js/src/durable-eval.ts +++ b/js/src/durable-eval.ts @@ -4,7 +4,11 @@ import { SpanTypeAttribute, uint8ArrayToBase64, } from "../util/index"; - +import { + type EvalParameters, + type InferParameters, + validateParameters, +} from "./eval-parameters"; import { _internalInitEvaluatorExperiment, _internalPrepareEvaluatorClassification, @@ -32,6 +36,9 @@ import { type Experiment, type ExperimentSummary, NOOP_SPAN, + Prompt, + serializePromptForCache, + type SerializedPrompt, type Span, _internalResumeSpan, _internalStartSpanWithInitialMerge, @@ -224,6 +231,7 @@ export interface DurableBatchTaskItem< Input, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, > { /** Stable identifier for this case and trial within the durable run. */ id: string; @@ -235,6 +243,8 @@ export interface DurableBatchTaskItem< metadata: Metadata; /** Tags associated with the evaluation case. */ tags: string[] | undefined; + /** Parameters supplied when the durable evaluation started. */ + parameters: InferParameters; /** Zero-based trial index for this case. */ trialIndex: number; } @@ -292,11 +302,12 @@ interface DurableBatchTask< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, SubmissionData extends JsonValue, > { readonly kind: typeof BATCH_TASK_KIND; readonly processor: DurableBatchProcessor< - DurableBatchTaskItem, + DurableBatchTaskItem, DurableBatchTaskResult, SubmissionData >; @@ -328,19 +339,21 @@ export class BatchTask< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, + Parameters extends EvalParameters = EvalParameters, SubmissionData extends JsonValue = JsonValue, > implements DurableBatchTask< Input, Output, Expected, Metadata, + Parameters, SubmissionData > { readonly kind: typeof BATCH_TASK_KIND = BATCH_TASK_KIND; constructor( readonly processor: DurableBatchProcessor< - DurableBatchTaskItem, + DurableBatchTaskItem, DurableBatchTaskResult, SubmissionData >, @@ -384,8 +397,9 @@ type DurableEvaluator< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, + Parameters extends EvalParameters = EvalParameters, > = Omit< - Evaluator, + Evaluator, "task" | "scores" | "timeout" | "signal" | "maxConcurrency" | "update" > & { store: DurableEvalStore; @@ -397,15 +411,25 @@ type DurableEvaluator< datum: EvalCase, ) => string | Promise; task: - | EvalTask - | DurableBatchTask; + | EvalTask + | DurableBatchTask< + Input, + Output, + Expected, + Metadata, + Parameters, + JsonValue + >; scores?: Array< | EvalScorer | DurableBatchScorer >; }; -interface DurableEvalStartOptions { +interface DurableEvalStartOptions< + Parameters extends EvalParameters = EvalParameters, +> { + parameters?: InferParameters; noSendLogs?: boolean; } @@ -439,14 +463,23 @@ interface DurableEvalRuntimeDefinition< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, + Parameters extends EvalParameters = EvalParameters, > { readonly projectName: string; readonly evalName: string; - readonly evaluator: DurableEvaluator; + readonly evaluator: DurableEvaluator< + Input, + Output, + Expected, + Metadata, + Parameters + >; } -interface DurableEvalDefinition { - start(options?: DurableEvalStartOptions): Promise; +interface DurableEvalDefinition { + start( + options?: DurableEvalStartOptions, + ): Promise; status(options: { runId: string }): Promise; poll(options: { runId: string }): Promise; processBatchResult(result: DurableBatchResult): Promise; @@ -497,6 +530,7 @@ type DurableRunState = { runId: string; experimentName: string; noSendLogs: boolean; + parameters: JsonValue; status: "running" | "completed"; summary?: ExperimentSummary; cases: DurableCaseRecord[]; @@ -731,15 +765,17 @@ export function defineDurableEval< Output, Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, + Parameters extends EvalParameters = EvalParameters, >( projectName: string, - evaluator: DurableEvaluator, -): DurableEvalDefinition { + evaluator: DurableEvaluator, +): DurableEvalDefinition { const definition: DurableEvalRuntimeDefinition< Input, Output, Expected, - Metadata + Metadata, + Parameters > = { projectName, evalName: evaluator.experimentName ?? projectName, @@ -759,14 +795,34 @@ async function startDurableEval< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, - options: DurableEvalStartOptions, + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, + options: DurableEvalStartOptions, ): Promise { const store = definition.evaluator.store; const runId = newId(); const key = runKey(definition.projectName, definition.evalName, runId); const { data } = callEvaluatorData(definition.evaluator.data); + const parameters = validateParameters( + options.parameters ?? {}, + definition.evaluator.parameters, + ); + const serializedParameters = assertJsonValue( + Object.fromEntries( + Object.entries(parameters).map(([name, value]) => [ + name, + Prompt.isPrompt(value) ? serializePromptForCache(value) : value, + ]), + ), + "eval parameters", + ); const experimentName = definition.evaluator.experimentName ?? `${definition.evalName}-${runId}`; const experiment = await _internalInitEvaluatorExperiment( @@ -775,7 +831,8 @@ async function startDurableEval< Input, Output, Expected, - Metadata + Metadata, + Parameters >, data, { @@ -795,13 +852,22 @@ async function startDurableEval< projectName: definition.projectName, evalName: definition.evalName, data, - } as unknown as EvaluatorDef, + // Parameters were already validated above, including Zod transforms. + parameters: undefined, + } as unknown as EvaluatorDef< + Input, + Output, + Expected, + Metadata, + Parameters + >, { start: () => undefined, stop: () => undefined, increment: () => undefined, }, undefined, + parameters, true, true, ); @@ -809,6 +875,7 @@ async function startDurableEval< runId, experimentName, noSendLogs: options.noSendLogs ?? false, + parameters: serializedParameters, status: "completed", summary: result.summary, cases: [], @@ -822,6 +889,7 @@ async function startDurableEval< runId, experimentName, noSendLogs: options.noSendLogs ?? false, + parameters: serializedParameters, status: "running", cases: await materializeCases(definition, data, experiment), batches: [], @@ -836,8 +904,15 @@ async function getDurableEvalStatus< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, options: { runId: string }, ): Promise { const store = definition.evaluator.store; @@ -855,8 +930,15 @@ async function processDurableBatchResult< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, result: DurableBatchResult, ): Promise { if (!result.batchId && !result.externalId) { @@ -898,8 +980,15 @@ async function pollDurableEval< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, options: { runId: string }, ): Promise { const store = definition.evaluator.store; @@ -956,7 +1045,7 @@ async function pollDurableEval< } async function openDurableExperiment( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, ) { const data: EvalCase[] = []; @@ -966,7 +1055,8 @@ async function openDurableExperiment( unknown, unknown, unknown, - BaseMetadata + BaseMetadata, + EvalParameters >, data, { @@ -982,8 +1072,15 @@ async function advanceDurableEval< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1038,7 +1135,7 @@ async function advanceDurableEval< } function currentStatus( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, ): DurableEvalResult { if (state.status === "completed") { @@ -1064,7 +1161,7 @@ function currentStatus( } async function startCaseRoot( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, experiment: Experiment | null, @@ -1090,11 +1187,11 @@ async function startCaseRoot( } async function logTaskResult( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, experiment: Experiment | null, - task?: EvalTask, + task?: EvalTask, ) { const datum = record.datum as EvalCase; const root = await startCaseRoot(definition, state, record, experiment); @@ -1102,7 +1199,13 @@ async function logTaskResult( if (task) { const result = await root.traced( (span) => - _internalRunEvaluatorTask(task, datum, record.trialIndex, span), + _internalRunEvaluatorTask( + task, + datum, + record.trialIndex, + taskParameters(definition, state), + span, + ), { name: "task", spanId: deterministicId(`${state.runId}:${record.id}:task`), @@ -1149,7 +1252,7 @@ async function logTaskResult( } async function logCompletedTasks( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1173,8 +1276,15 @@ async function runTaskStage< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1189,7 +1299,8 @@ async function runTaskStage< Input, Output, Expected, - Metadata + Metadata, + Parameters >; const changed: DurableCaseRecord[] = []; for (const record of state.cases) { @@ -1209,8 +1320,15 @@ async function runScoreStages< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1296,7 +1414,7 @@ function scorerArgs(record: DurableCaseRecord) { } function resumeCaseRoot( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, record: DurableCaseRecord, experiment: Experiment | null, ) { @@ -1311,7 +1429,7 @@ function resumeCaseRoot( } async function evaluateAndLogScore( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, name: string, @@ -1361,7 +1479,7 @@ async function evaluateAndLogScore( } async function evaluateAndLogClassification( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, record: DurableCaseRecord, name: string, @@ -1410,7 +1528,7 @@ async function evaluateAndLogClassification( } async function ensureBatches( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, store: DurableEvalStore, key: string, @@ -1441,8 +1559,11 @@ async function ensureBatches( ); if (!claim.created) continue; const context = { runId: state.runId, batchId }; + const parameters = kind === "task" ? taskParameters(definition, state) : {}; const items = records.map((record) => - kind === "task" ? taskBatchItem(record) : scorerBatchItem(record), + kind === "task" + ? taskBatchItem(record, parameters) + : scorerBatchItem(record), ); const submissionData = assertJsonValue( await processor.submit(items, context), @@ -1470,7 +1591,7 @@ async function ensureBatches( } async function collectBatch( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, batch: DurableBatchRecord, ) { @@ -1528,7 +1649,7 @@ async function collectBatch( } function processorForStage( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, kind: "task" | "score", scorerName?: string, ): DurableBatchProcessor { @@ -1556,9 +1677,16 @@ async function materializeCases< Output, Expected, Metadata extends BaseMetadata, + Parameters extends EvalParameters, >( - definition: DurableEvalRuntimeDefinition, - data: Evaluator["data"], + definition: DurableEvalRuntimeDefinition< + Input, + Output, + Expected, + Metadata, + Parameters + >, + data: Evaluator["data"], experiment: Experiment | null, ): Promise { const evaluator = definition.evaluator; @@ -1615,7 +1743,31 @@ async function materializeCases< return records; } -function taskBatchItem(record: DurableCaseRecord) { +function taskParameters( + definition: DurableEvalRuntimeDefinition, + state: DurableRunState, +): Record { + return Object.fromEntries( + Object.entries(state.parameters as Record).map( + ([name, value]) => { + const schema = definition.evaluator.parameters?.[name]; + if (schema && "type" in schema && schema.type === "prompt") { + const prompt = value as SerializedPrompt; + return [ + name, + new Prompt(prompt.metadata, prompt.defaults, prompt.noTrace), + ]; + } + return [name, value]; + }, + ), + ); +} + +function taskBatchItem( + record: DurableCaseRecord, + parameters: Record, +) { const datum = record.datum as EvalCase; return { id: record.id, @@ -1623,6 +1775,7 @@ function taskBatchItem(record: DurableCaseRecord) { expected: "expected" in datum ? datum.expected : undefined, metadata: record.metadata, tags: record.tags, + parameters, trialIndex: record.trialIndex, }; } @@ -1641,7 +1794,7 @@ function scorerBatchItem(record: DurableCaseRecord) { } async function finishExperiment( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, state: DurableRunState, experiment: Experiment | null, ) { @@ -1774,7 +1927,7 @@ type DurableBatchPlan = Omit< >; function plannedBatches( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, runId: string, caseIds: string[], ) { @@ -1813,7 +1966,7 @@ function plannedBatches( } async function readCaseRecord( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, store: DurableEvalStore, key: string, id: string, @@ -1904,7 +2057,7 @@ async function readCaseRecord( } async function readRunState( - definition: DurableEvalRuntimeDefinition, + definition: DurableEvalRuntimeDefinition, store: DurableEvalStore, key: string, ) { @@ -2096,7 +2249,7 @@ function resultItemId(value: unknown) { function isBatchTask( value: unknown, -): value is DurableBatchTask { +): value is DurableBatchTask { return ( typeof value === "object" && value !== null && diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts new file mode 100644 index 000000000..dde85d466 --- /dev/null +++ b/js/src/eval-parameters.ts @@ -0,0 +1,92 @@ +import { z } from "zod/v3"; +import { Prompt } from "./logger"; +import { + promptDefinitionToPromptData, + type PromptDefinitionWithTools, +} from "./prompt-schemas"; +import { PromptData as promptDataSchema } from "./sdk-schemas"; + +export type EvalParameters = Record< + string, + | { + type: "prompt"; + default?: PromptDefinitionWithTools; + description?: string; + } + | { + type: "model"; + default?: string; + description?: string; + } + | z.ZodTypeAny +>; + +type InferParameterValue = T extends { type: "prompt" } + ? Prompt + : T extends { type: "model" } + ? string + : T extends z.ZodType + ? z.infer + : never; + +export type InferParameters = { + [K in keyof T]: InferParameterValue; +}; + +export function validateParameters( + parameters: Record, + parameterSchema?: Parameters, +): InferParameters { + if (!parameterSchema) { + return parameters as InferParameters; + } + + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return Object.fromEntries( + Object.entries(parameterSchema).map(([name, schema]) => { + const value = parameters[name]; + try { + if ("type" in schema && schema.type === "prompt") { + if (Prompt.isPrompt(value)) { + return [name, value]; + } + let promptData; + if (value) { + promptData = promptDataSchema.parse(value); + } else if (schema.default) { + promptData = promptDefinitionToPromptData( + schema.default, + schema.default.tools, + ); + } else { + throw new Error(`Parameter '${name}' is required`); + } + return [name, Prompt.fromPromptData(name, promptData)]; + } + + if ("type" in schema && schema.type === "model") { + const model = value ?? schema.default; + if (model === undefined) { + throw new Error(`Parameter '${name}' is required`); + } + if (typeof model !== "string") { + throw new Error( + `Parameter '${name}' must be a string model identifier`, + ); + } + return [name, model]; + } + + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + const zodSchema = schema as z.ZodSchema; + return [name, zodSchema.parse(value)]; + } catch (e) { + // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. + console.error("Error validating parameter", name, e); + throw Error( + `Invalid parameter '${name}': ${e instanceof Error ? e.message : String(e)}`, + ); + } + }), + ) as InferParameters; +} diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 6727361ec..9efa9cf68 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -54,6 +54,7 @@ test("meta (write) is passed to task", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -90,6 +91,7 @@ test("metadata (read/write) is passed to task", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -128,6 +130,7 @@ test("expected (read/write) is passed to task", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -169,6 +172,7 @@ test("EvalCase id and tags are passed to scorers", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -561,6 +565,7 @@ describe("runEvaluator", () => { }, new NoopProgressReporter(), undefined, + undefined, true, ), ).rejects.toThrow("Encountered 1 unhandled task errors"); @@ -745,6 +750,7 @@ test("trialIndex is passed to task", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -785,6 +791,7 @@ test("trialIndex with multiple inputs", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -829,6 +836,7 @@ test("per-input trialCount overrides global trialCount", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -874,6 +882,7 @@ test("per-input trialCount works without global trialCount", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -1039,6 +1048,7 @@ test("runEvaluator forwards baseExperimentId to summary", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -1080,6 +1090,7 @@ test("runEvaluator forwards persisted baseExperimentName id to summary", async ( }, new NoopProgressReporter(), undefined, + undefined, true, ); @@ -1120,6 +1131,7 @@ test("tags can be appended and logged to root span", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); expect(result.results[0].tags).toEqual(expectedTags); @@ -1167,6 +1179,7 @@ test.each([ }, new NoopProgressReporter(), undefined, + undefined, true, ); expect(result.results[0].tags).toEqual(expectedTags); @@ -1205,6 +1218,7 @@ test("tags are persisted with a failing scorer", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); expect(result.results[0].tags).toEqual(expectedTags); @@ -1236,6 +1250,7 @@ test("tags remain empty when not set", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); expect(result.results[0].tags).toEqual(undefined); @@ -1269,6 +1284,7 @@ test("scorer spans have purpose='scorer' attribute", async () => { }, new NoopProgressReporter(), undefined, + undefined, true, ); diff --git a/js/src/framework.ts b/js/src/framework.ts index 2fe0f4847..7c53dba4f 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -51,6 +51,11 @@ import type { ReporterDef, } from "./reporters/types"; import { isEmpty, InternalAbortError } from "./util"; +import { + type EvalParameters, + type InferParameters, + validateParameters, +} from "./eval-parameters"; export type BaseExperiment< Input, @@ -98,9 +103,15 @@ export type EvalData< | Promise[]> | BaseExperiment); -export type EvalTask = ( +export type EvalTask< + Input, + Output, + Expected, + Metadata extends BaseMetadata, + Parameters extends EvalParameters, +> = ( input: Input, - hooks: EvalHooks, + hooks: EvalHooks, ) => Output | Promise; type TaskProgressEvent = Omit< @@ -108,7 +119,11 @@ type TaskProgressEvent = Omit< "id" | "origin" | "object_type" | "name" >; -export interface EvalHooks { +export interface EvalHooks< + Expected, + Metadata extends BaseMetadata, + Parameters extends EvalParameters, +> { /** * The metadata object for the current evaluation. You can mutate this object to add or remove metadata. */ @@ -121,6 +136,10 @@ export interface EvalHooks { * The task's span. */ span: Span; + /** + * The current parameters being used for this specific task execution. + */ + parameters: InferParameters; /** * Report progress that will show up in the playground. */ @@ -196,6 +215,7 @@ export interface Evaluator< Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, + Parameters extends EvalParameters = EvalParameters, > { /** * A function that returns a list of inputs, expected outputs, and metadata. @@ -205,7 +225,7 @@ export interface Evaluator< /** * A function that takes an input and returns an output. */ - task: EvalTask; + task: EvalTask; /** * A set of functions that take an input, output, and expected value and return a {@link Score}. @@ -220,6 +240,12 @@ export interface Evaluator< */ classifiers?: EvalClassifier[]; + /** + * Inline parameter schemas whose validated values are passed to task hooks. + * Use Zod schemas, prompt definitions, or model identifiers with optional defaults. + */ + parameters?: Parameters; + /** * An optional name for the experiment. */ @@ -360,10 +386,11 @@ export type EvaluatorDef< Output, Expected, Metadata extends BaseMetadata = DefaultMetadataType, + Parameters extends EvalParameters = EvalParameters, > = { projectName: string; evalName: string; -} & Evaluator; +} & Evaluator; type EvaluatorFile = { functions: CodeFunction< @@ -374,7 +401,13 @@ type EvaluatorFile = { prompts: CodePrompt[]; evaluators: { [evalName: string]: { - evaluator: EvaluatorDef; + evaluator: EvaluatorDef< + unknown, + unknown, + unknown, + BaseMetadata, + EvalParameters + >; reporter?: ReporterDef | string; }; }; @@ -394,7 +427,7 @@ function initExperiment( export async function _internalInitEvaluatorExperiment( projectName: string, - evaluator: Evaluator, + evaluator: Evaluator, data: EvalData, options: { disabled?: boolean; @@ -472,7 +505,7 @@ function isIterable(value: unknown): value is Iterable { export async function _internalResolveEvaluatorData( evaluator: Pick< - EvaluatorDef, + EvaluatorDef, "data" | "projectName" | "projectId" | "state" >, experiment: Experiment | null, @@ -544,7 +577,7 @@ globalThis._evals = { reporters: {}, }; -interface EvalOptions { +interface EvalOptions { /** * A `Reporter` which you can use to summarize progress after an Eval() runs. */ @@ -577,6 +610,10 @@ interface EvalOptions { * is somewhat outdated, and may be removed in the future. */ progress?: ProgressReporter; + /** + * The parameters to use for the evaluator. + */ + parameters?: InferParameters; /** * Whether to retain the per-example Eval results and return them from Eval(). * @@ -631,18 +668,23 @@ export async function Eval< Expected = void, Metadata extends BaseMetadata = DefaultMetadataType, EvalReport = boolean, + Parameters extends EvalParameters = EvalParameters, >( name: string, - evaluator: Evaluator, - reporterOrOpts?: ReporterDef | string | EvalOptions, + evaluator: Evaluator, + reporterOrOpts?: + | ReporterDef + | string + | EvalOptions, ): Promise> { - const options: EvalOptions = isEmpty(reporterOrOpts) - ? {} - : typeof reporterOrOpts === "string" - ? { reporter: reporterOrOpts } - : "name" in reporterOrOpts - ? { reporter: reporterOrOpts } - : reporterOrOpts; + let options: EvalOptions; + if (isEmpty(reporterOrOpts)) { + options = {}; + } else if (typeof reporterOrOpts === "string" || "name" in reporterOrOpts) { + options = { reporter: reporterOrOpts }; + } else { + options = reporterOrOpts; + } let evalName = makeEvalName(name, evaluator.experimentName); if (globalThis._evals.evaluators[evalName]) { @@ -655,7 +697,13 @@ export async function Eval< evalName, projectName: name, ...evaluator, - } as EvaluatorDef, + } as EvaluatorDef< + unknown, + unknown, + unknown, + BaseMetadata, + EvalParameters + >, reporter: options.reporter, }; @@ -726,6 +774,7 @@ export async function Eval< evalDef, progressReporter, options.stream, + options.parameters, shouldCollectResults, enableCache, ), @@ -737,6 +786,7 @@ export async function Eval< evalDef, progressReporter, options.stream, + options.parameters, shouldCollectResults, enableCache, ); @@ -793,9 +843,10 @@ export function classifierName( } export async function _internalRunEvaluatorTask( - task: EvalTask, + task: EvalTask, datum: EvalCase, trialIndex: number, + parameters: Record, span: Span, reportProgress: (event: TaskProgressEvent) => void = () => undefined, ): Promise<{ @@ -806,10 +857,11 @@ export async function _internalRunEvaluatorTask( const metadata: Record = { ...("metadata" in datum ? datum.metadata : {}), }; - const hooks: EvalHooks> = { + const hooks: EvalHooks, EvalParameters> = { metadata, expected: "expected" in datum ? datum.expected : undefined, span, + parameters, reportProgress, trialIndex, tags: [...(datum.tags ?? [])], @@ -1027,9 +1079,10 @@ export const defaultErrorScoreHandler: ErrorScoreHandler = ({ export async function runEvaluator( experiment: Experiment | null, // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, + evaluator: EvaluatorDef, progressReporter: ProgressReporter, stream: ((data: SSEProgressEventData) => void) | undefined, + parameters?: InferParameters, collectResults = true, enableCache = true, // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -1045,6 +1098,10 @@ export async function runEvaluator( (evaluator.state ?? _internalGetGlobalState())?.spanCache?.start(); } try { + const validatedParameters = validateParameters( + parameters ?? {}, + evaluator.parameters, + ); const dataIterable = await _internalResolveEvaluatorData( evaluator, experiment, @@ -1182,6 +1239,7 @@ export async function runEvaluator( evaluator.task, datum, trialIndex, + validatedParameters, span, (event) => { stream?.({ @@ -1584,7 +1642,7 @@ function ensureScoreAccumulator( export function buildLocalSummary( // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, + evaluator: EvaluatorDef, // eslint-disable-next-line @typescript-eslint/no-explicit-any results: EvalResult[], precomputedScores?: ScoreAccumulator, @@ -1661,7 +1719,7 @@ const defaultReporter: ReporterDef = { name: "Braintrust default reporter", async reportEval( // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, + evaluator: EvaluatorDef, // eslint-disable-next-line @typescript-eslint/no-explicit-any result: EvalResultWithSummary, { verbose, jsonl }: ReporterOpts, diff --git a/js/src/framework2.ts b/js/src/framework2.ts index 12eb8b897..702a90f35 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -7,8 +7,6 @@ import { Project as projectSchema } from "./sdk-schemas"; import type { FunctionTypeEnum as FunctionType, IfExists, - ChatCompletionMessageParam, - ModelParams, PromptData, ToolFunctionDefinition, } from "./sdk-types"; @@ -20,6 +18,10 @@ import { PromptRowWithId, } from "./logger"; import type { BaseFnOpts, GenericFunction } from "./framework-types"; +import { + promptDefinitionToPromptData, + type PromptDefinition, +} from "./prompt-schemas"; // Safe access to __filename (only exists in Node.js CJS) const currentFilename = @@ -372,16 +374,6 @@ export class CodePrompt { } } -type PromptDefinition = ( - | { prompt: string } - | { messages: ChatCompletionMessageParam[] } -) & { - model: string; - params?: ModelParams; - templateFormat?: "mustache" | "nunjucks" | "none"; - environments?: string[]; -}; - type PromptOpts< HasId extends boolean, HasVersion extends boolean, @@ -449,34 +441,3 @@ interface PromptFunctionEvent { metadata?: Record; environments?: { slug: string }[]; } - -function promptDefinitionToPromptData( - promptDefinition: PromptDefinition, - rawTools?: ToolFunctionDefinition[], -): PromptData { - const promptBlock: PromptData["prompt"] = - "messages" in promptDefinition - ? { - type: "chat", - messages: promptDefinition.messages, - tools: - rawTools && rawTools.length > 0 - ? JSON.stringify(rawTools) - : undefined, - } - : { - type: "completion", - content: promptDefinition.prompt, - }; - - return { - prompt: promptBlock, - options: { - model: promptDefinition.model, - params: promptDefinition.params, - }, - ...(promptDefinition.templateFormat - ? { template_format: promptDefinition.templateFormat } - : {}), - }; -} diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts new file mode 100644 index 000000000..ace8f114c --- /dev/null +++ b/js/src/parameters.test.ts @@ -0,0 +1,336 @@ +import { expect, expectTypeOf, test, beforeAll, vi } from "vitest"; +import { validateParameters } from "./eval-parameters"; +import { runEvaluator } from "./framework"; +import { Eval, type Prompt } from "./exports"; +import { z } from "zod/v3"; +import { type ProgressReporter } from "./reporters/types"; +import { configureNode } from "./node/config"; + +beforeAll(() => { + configureNode(); +}); + +class NoopProgressReporter implements ProgressReporter { + public start() {} + public stop() {} + public increment() {} +} + +test("parameters are passed to task", async () => { + const out = await runEvaluator( + null, + { + projectName: "test-parameters", + evalName: "test", + data: [{ input: "hello" }], + task: async (input: string, { parameters }) => { + const output = `${parameters.prefix}${input}${parameters.suffix}`; + return output; + }, + scores: [], + classifiers: [], + parameters: { + prefix: z.string().default("start:"), + suffix: z.string().default(":end"), + }, + }, + new NoopProgressReporter(), + undefined, + { prefix: "start:", suffix: ":end" }, + true, + ); + + expect(out.results).toHaveLength(1); + expect(out.results[0].output).toBe("start:hello:end"); +}); + +test("prompt parameter is passed correctly", async () => { + const result = await runEvaluator( + null, + { + projectName: "test-prompt-parameter", + evalName: "test", + data: [{ input: "test input" }], + task: async (input: string, { parameters }) => { + // Verify the prompt parameter has the expected structure + if (!parameters.main || typeof parameters.main.build !== "function") { + throw new Error( + "Prompt parameter 'main' is missing or does not have a build function", + ); + } + return input; + }, + scores: [], + classifiers: [], + parameters: { + main: { + type: "prompt", + name: "Main prompt", + description: "Test prompt", + default: { + messages: [ + { + role: "user", + content: "{{input}}", + }, + ], + model: "gpt-4", + }, + }, + }, + }, + new NoopProgressReporter(), + undefined, + undefined, + true, + ); + + expect(result.results).toHaveLength(1); + expect(result.results[0].output).toBe("test input"); +}); + +test("custom parameter values override defaults", async () => { + const result = await runEvaluator( + null, + { + projectName: "test-custom-parameters", + evalName: "test", + data: [{ input: "hello" }], + task: async (input: string, { parameters }) => { + const output = `${parameters.prefix}${input}${parameters.suffix}`; + return output; + }, + scores: [], + classifiers: [], + parameters: { + prefix: z.string().default("start:"), + suffix: z.string().default(":end"), + }, + }, + new NoopProgressReporter(), + undefined, + { + prefix: "custom:", + suffix: ":custom", + }, + true, + ); + + expect(result.results).toHaveLength(1); + expect(result.results[0].output).toBe("custom:hello:custom"); +}); + +test("array parameter is handled correctly", async () => { + const result = await runEvaluator( + null, + { + projectName: "test-array-parameter", + evalName: "test", + data: [{ input: "test" }], + task: async (input: string, { parameters }) => { + expect(Array.isArray(parameters.items)).toBe(true); + expect(parameters.items).toEqual(["item1", "item2"]); + return input; + }, + scores: [], + classifiers: [], + parameters: { + items: z.array(z.string()).default(["item1", "item2"]), + }, + }, + new NoopProgressReporter(), + undefined, + undefined, + true, + ); + + expect(result.results).toHaveLength(1); + expect(result.results[0].output).toBe("test"); +}); + +test("object parameter is handled correctly", async () => { + const result = await runEvaluator( + null, + { + projectName: "test-object-parameter", + evalName: "test", + data: [{ input: "test" }], + task: async (input: string, { parameters }) => { + expect(parameters.config).toEqual({ + name: "test", + value: 123, + }); + return input; + }, + scores: [], + classifiers: [], + parameters: { + config: z + .object({ + name: z.string(), + value: z.number(), + }) + .default({ + name: "test", + value: 123, + }), + }, + }, + new NoopProgressReporter(), + undefined, + undefined, + true, + ); + + expect(result.results).toHaveLength(1); + expect(result.results[0].output).toBe("test"); +}); + +test("model parameter defaults to configured value", async () => { + const result = await runEvaluator( + null, + { + projectName: "test-model-parameter-default", + evalName: "test", + data: [{ input: "test" }], + task: async (input: string, { parameters }) => { + expect(parameters.model).toBe("gpt-5-mini"); + return input; + }, + scores: [], + classifiers: [], + parameters: { + model: { + type: "model", + default: "gpt-5-mini", + }, + }, + }, + new NoopProgressReporter(), + undefined, + undefined, + true, + ); + + expect(result.results).toHaveLength(1); + expect(result.results[0].output).toBe("test"); +}); + +test("model parameter is required when default is missing", async () => { + await expect( + runEvaluator( + null, + { + projectName: "test-model-parameter-required", + evalName: "test", + data: [{ input: "test" }], + task: async (input: string) => input, + scores: [], + classifiers: [], + parameters: { + model: { + type: "model", + }, + }, + }, + new NoopProgressReporter(), + undefined, + undefined, + true, + ), + ).rejects.toThrow("Parameter 'model' is required"); +}); + +test("Eval infers inline parameter types and applies value overrides", async () => { + const result = await Eval( + "inline-parameters", + { + data: [{ input: "hello" }], + parameters: { + prefix: z.string().default("default:"), + count: z.number().default(1), + model: { type: "model", default: "test-model" }, + prompt: { + type: "prompt", + default: { prompt: "{{input}}", model: "test-model" }, + }, + }, + task: (input, { parameters }) => { + expectTypeOf(parameters.prefix).toEqualTypeOf(); + expectTypeOf(parameters.count).toEqualTypeOf(); + expectTypeOf(parameters.model).toEqualTypeOf(); + expectTypeOf(parameters.prompt).toEqualTypeOf< + Prompt + >(); + expect( + parameters.prompt.build({ input }, { flavor: "completion" }).prompt, + ).toBe("hello"); + return `${parameters.prefix}${input}`.repeat(parameters.count); + }, + scores: [], + }, + { noSendLogs: true }, + ); + expect(result.results[0].output).toBe("default:hello"); + + const overridden = await Eval( + "inline-overrides", + { + data: [{ input: "hello" }], + parameters: { prefix: z.string().default("default:") }, + task: (input, { parameters }) => `${parameters.prefix}${input}`, + scores: [], + }, + { noSendLogs: true, parameters: { prefix: "custom:" } }, + ); + expect(overridden.results[0].output).toBe("custom:hello"); +}); + +test("invalid inline parameters reject before the task runs", async () => { + const task = vi.fn((input: string) => input); + await expect( + Eval( + "invalid-parameters", + { + data: [{ input: "hello" }], + parameters: { count: z.number().positive() }, + task, + scores: [], + }, + { noSendLogs: true, parameters: { count: -1 } }, + ), + ).rejects.toThrow("Invalid parameter 'count'"); + expect(task).not.toHaveBeenCalled(); +}); + +test("prompt parameters preserve inline provider tools and accept prompt values", () => { + const schema = { + main: { + type: "prompt" as const, + default: { + model: "test-model", + messages: [{ role: "user" as const, content: "{{input}}" }], + tools: [{ type: "function" as const, function: { name: "weather" } }], + }, + }, + }; + const defaults = validateParameters({}, schema); + expect(defaults.main.build({ input: "hello" }).tools).toEqual( + schema.main.default.tools, + ); + expect(validateParameters({ main: defaults.main }, schema).main).toBe( + defaults.main, + ); + const override = validateParameters( + { + main: { + prompt: { type: "completion", content: "Hi {{input}}" }, + options: { model: "other-model" }, + }, + }, + schema, + ); + expect( + override.main.build({ input: "world" }, { flavor: "completion" }).prompt, + ).toBe("Hi world"); +}); diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts new file mode 100644 index 000000000..be27c9ed9 --- /dev/null +++ b/js/src/prompt-schemas.ts @@ -0,0 +1,51 @@ +import type { + ChatCompletionMessageParam, + ModelParams, + PromptData, + ToolFunctionDefinition, +} from "./sdk-types"; + +export type PromptDefinition = ( + | { prompt: string } + | { messages: ChatCompletionMessageParam[] } +) & { + model: string; + params?: ModelParams; + templateFormat?: "mustache" | "nunjucks" | "none"; + environments?: string[]; +}; + +export type PromptDefinitionWithTools = PromptDefinition & { + tools?: ToolFunctionDefinition[]; +}; + +export function promptDefinitionToPromptData( + promptDefinition: PromptDefinition, + rawTools?: ToolFunctionDefinition[], +): PromptData { + let promptBlock: PromptData["prompt"]; + if ("messages" in promptDefinition) { + promptBlock = { + type: "chat", + messages: promptDefinition.messages, + tools: + rawTools && rawTools.length > 0 ? JSON.stringify(rawTools) : undefined, + }; + } else { + promptBlock = { + type: "completion", + content: promptDefinition.prompt, + }; + } + + return { + prompt: promptBlock, + options: { + model: promptDefinition.model, + params: promptDefinition.params, + }, + ...(promptDefinition.templateFormat + ? { template_format: promptDefinition.templateFormat } + : {}), + }; +} diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts index dcb170e49..2e7d38e0a 100644 --- a/js/src/public-types.test.ts +++ b/js/src/public-types.test.ts @@ -1,4 +1,5 @@ import { expect, expectTypeOf, test } from "vitest"; +import { z } from "zod/v3"; import * as publicExports from "./exports"; import type { @@ -36,11 +37,13 @@ test("exports only intentional extension types", () => { expectTypeOf().toBeObject(); expectTypeOf().toBeObject(); expectTypeOf(BraintrustLangChainCallbackHandler).toBeConstructibleWith(); - expectTypeOf>().toBeFunction(); + expectTypeOf< + EvalTask + >().toBeFunction(); expectTypeOf>().toBeFunction(); }); -test("does not expose deprecated tools or evaluation parameters", () => { +test("does not expose deprecated tools or saved configurations", () => { expect(publicExports).not.toHaveProperty("loadParameters"); const project = publicExports.projects.create({ name: "test-project" }); expect(project).not.toHaveProperty("tools"); @@ -58,24 +61,18 @@ test("does not expose deprecated tools or evaluation parameters", () => { data: [{ input: 1 }], task: (input: number) => input, scores: [], + parameters: { count: z.number().default(1) }, }; - publicExports.Eval("test", { - ...evaluator, - // @ts-expect-error Local parameter schemas are no longer supported. - parameters: {}, - }); publicExports.Eval("test", evaluator, { - // @ts-expect-error Parameter overrides are no longer supported. - parameters: {}, + // @ts-expect-error Inline overrides must match the parameter schema. + parameters: { count: "one" }, }); publicExports.Eval("test", { ...evaluator, - task: (input, hooks) => { - // @ts-expect-error Tasks no longer receive parameters. - hooks.parameters; - return input; - }, + // @ts-expect-error Only inline schemas are supported, not remote configuration promises. + parameters: Promise.resolve(evaluator.parameters), }); + project.prompts.create({ name: "test-prompt", model: "test-model", diff --git a/js/src/reporters/types.ts b/js/src/reporters/types.ts index 7909dfbfd..a85be9bdb 100644 --- a/js/src/reporters/types.ts +++ b/js/src/reporters/types.ts @@ -24,7 +24,7 @@ export interface ReporterBody { // These any's are required because these function specifications don't know // or need to know the types of the input/output/etc for the evaluator. // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, + evaluator: EvaluatorDef, // eslint-disable-next-line @typescript-eslint/no-explicit-any result: EvalResultWithSummary, opts: ReporterOpts, From 9dfab8f4a434aca5f54196b839dcda4489aa291c Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:18:03 +0000 Subject: [PATCH 3/4] Update PR #2609 --- .changeset/remove-deprecated-tools-and-configurations.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/remove-deprecated-tools-and-configurations.md diff --git a/.changeset/remove-deprecated-tools-and-configurations.md b/.changeset/remove-deprecated-tools-and-configurations.md new file mode 100644 index 000000000..2dd31ed5b --- /dev/null +++ b/.changeset/remove-deprecated-tools-and-configurations.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +refactor(sdk)!: Remove deprecated tools and configurations From 131b3f2ceef6d2dadf5544676a2a192bac516aa1 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:32:56 +0000 Subject: [PATCH 4/4] add back remote parameters --- .../workflow-eval-webhook/scenario.ts | 19 +- js/package.json | 1 + js/src/eval-parameters.ts | 138 ++++- js/src/exports.ts | 1 + js/src/framework.test.ts | 119 ++++ js/src/framework.ts | 120 +++- js/src/framework2.test.ts | 33 +- js/src/framework2.ts | 236 +++++++- js/src/logger.test.ts | 175 +++++- js/src/logger.ts | 547 ++++++++++++++++-- js/src/parameters.test.ts | 161 ++---- js/src/prompt-cache/cache-config.ts | 44 +- js/src/prompt-cache/cache-mode.test.ts | 91 ++- js/src/prompt-cache/parameters-cache.test.ts | 69 +++ js/src/prompt-cache/parameters-cache.ts | 123 ++++ js/src/prompt-types.test.ts | 2 +- js/src/public-types.test.ts | 16 +- js/src/sdk-contracts.test.ts | 8 +- js/src/sdk-schemas.ts | 2 + js/src/sdk-types.ts | 2 + js/src/workflow-eval.test.ts | 113 ---- js/src/workflow-eval.ts | 52 +- pnpm-lock.yaml | 3 + 23 files changed, 1657 insertions(+), 418 deletions(-) create mode 100644 js/src/prompt-cache/parameters-cache.test.ts create mode 100644 js/src/prompt-cache/parameters-cache.ts diff --git a/e2e/scenarios/workflow-eval-webhook/scenario.ts b/e2e/scenarios/workflow-eval-webhook/scenario.ts index 0b9d98007..2ef67869d 100644 --- a/e2e/scenarios/workflow-eval-webhook/scenario.ts +++ b/e2e/scenarios/workflow-eval-webhook/scenario.ts @@ -1,5 +1,3 @@ -import assert from "node:assert/strict"; -import { z } from "zod/v3"; import { WorkflowScorer, WorkflowTask, @@ -16,15 +14,9 @@ async function main() { const testRunId = getTestRunId(); const scenario = "workflow-eval-webhook"; const store = new WorkflowEvalMemoryStore(); - const parameters = { factor: z.number().default(3) }; const jobs = new Map< string, - { - input: number; - output?: number; - expected: number; - parameters?: { factor: number }; - } + { input: number; output?: number; expected: number } >(); const webhookCompletion = { mode: "webhook" as const, @@ -35,19 +27,17 @@ async function main() { number, number, { testRunId: string; kind: string }, - typeof parameters, + Record, { id: string } >({ async submit(item) { - assert.equal(item.parameters.factor, 2); const id = `task-${jobs.size + 1}`; jobs.set(id, item); return { id }; }, completion: webhookCompletion, async collect(submissionData) { - const item = jobs.get(submissionData.id)!; - return { output: item.input * item.parameters!.factor }; + return { output: jobs.get(submissionData.id)!.input * 2 }; }, }); const scorer = new WorkflowScorer< @@ -78,7 +68,6 @@ async function main() { scopedName("e2e-workflow-eval-webhook-project", testRunId), { store, - parameters, maxConcurrency: 2, experimentName: `${scenario}-${testRunId}`, data: [1, 2, 3].map((input) => ({ @@ -113,7 +102,7 @@ async function main() { let localScoreCount = 0; let classifierCount = 0; - const waiting = await definition.start({ parameters: { factor: 2 } }); + const waiting = await definition.start(); if (waiting.status !== "waiting" || jobs.size !== 3) { throw new Error( "Workflow eval did not pause with three webhook submissions", diff --git a/js/package.json b/js/package.json index 83cf63111..3be3f3e3b 100644 --- a/js/package.json +++ b/js/package.json @@ -190,6 +190,7 @@ "@vercel/functions": "^1.0.2", "acorn": "^8.16.0", "acorn-import-attributes": "^1.9.5", + "ajv": "^8.20.0", "astring": "^1.9.0", "cjs-module-lexer": "^2.2.0", "dotenv": "^16.4.5", diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index 76640c034..60d75560d 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -1,5 +1,5 @@ -import { Prompt } from "./logger"; -import type { ZodSchema } from "./zod/utils"; +import Ajv from "ajv"; +import { Prompt, RemoteEvalParameters } from "./logger"; import { promptDefinitionToPromptData, type PromptDefinitionWithTools, @@ -20,51 +20,89 @@ export type EvalParameters = Record< } | ZodSchema >; +import type { ZodSchema } from "./zod/utils"; +// Type helper to infer the type of a parameter value type InferParameterValue = T extends { type: "prompt" } - ? Prompt + ? Prompt : T extends { type: "model" } ? string : T extends ZodSchema ? T["_output"] : never; +// Type helper to infer the full parameters type export type InferParameters = { [K in keyof T]: InferParameterValue; }; -export function validateParameters( +export async function validateParameters< + Parameters extends EvalParameters = EvalParameters, + T extends Record = InferParameters, +>( parameters: Record, - parameterSchema?: Parameters, -): InferParameters { - if (!parameterSchema) { - return parameters as InferParameters; + parameterSchema: Parameters | Promise | unknown, +): Promise { + let resolvedSchema = parameterSchema; + if (resolvedSchema instanceof Promise) { + resolvedSchema = await resolvedSchema; + } + + // If no schema is provided, return parameters as-is + if (resolvedSchema === undefined || resolvedSchema === null) { + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return parameters as T; + } + + if (RemoteEvalParameters.isParameters(resolvedSchema)) { + const mergedParameters = + parameters && Object.keys(parameters).length > 0 + ? { + ...resolvedSchema.data, + ...parameters, + } + : resolvedSchema.data; + + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return validateParametersWithJsonSchema( + mergedParameters, + resolvedSchema.schema, + ) as T; } + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return validateParametersWithZod( + parameters, + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + resolvedSchema as Parameters, + ) as T; +} + +function validateParametersWithZod< + Parameters extends EvalParameters = EvalParameters, +>( + parameters: Record, + parameterSchema: Parameters, +): InferParameters { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return Object.fromEntries( Object.entries(parameterSchema).map(([name, schema]) => { const value = parameters[name]; try { if ("type" in schema && schema.type === "prompt") { - if (Prompt.isPrompt(value)) { - return [name, value]; - } - let promptData; - if (value) { - promptData = promptDataSchema.parse(value); - } else if (schema.default) { - promptData = promptDefinitionToPromptData( - schema.default, - schema.default.tools, - ); - } else { + const promptData = value + ? promptDataSchema.parse(value) + : schema.default + ? promptDefinitionToPromptData( + schema.default, + schema.default.tools, + ) + : undefined; + if (!promptData) { throw new Error(`Parameter '${name}' is required`); } return [name, Prompt.fromPromptData(name, promptData)]; - } - - if ("type" in schema && schema.type === "model") { + } else if ("type" in schema && schema.type === "model") { const model = value ?? schema.default; if (model === undefined) { throw new Error(`Parameter '${name}' is required`); @@ -75,11 +113,9 @@ export function validateParameters( ); } return [name, model]; + } else { + return [name, schema.parse(value)]; } - - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const zodSchema = schema as ZodSchema; - return [name, zodSchema.parse(value)]; } catch (e) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. console.error("Error validating parameter", name, e); @@ -90,3 +126,51 @@ export function validateParameters( }), ) as InferParameters; } + +function validateParametersWithJsonSchema>( + parameters: Record, + schema: Record, +): T { + const ajv = new Ajv({ coerceTypes: true, useDefaults: true, strict: false }); + const validate = ajv.compile(schema); + + if (!validate(parameters)) { + const errorMessages = validate.errors + ?.map((err) => { + const path = err.instancePath || "root"; + return `${path}: ${err.message}`; + }) + .join(", "); + throw Error(`Invalid parameters: ${errorMessages}`); + } + + return rehydrateRemoteParameters(parameters, schema) as T; +} + +function rehydrateRemoteParameters( + parameters: Record, + schema: Record, +): Record { + const schemaProperties = schema.properties; + if (typeof schemaProperties !== "object" || schemaProperties === null) { + return parameters; + } + + return Object.fromEntries( + Object.entries(parameters).map(([name, value]) => { + const propertySchema = Reflect.get(schemaProperties, name); + if (typeof propertySchema !== "object" || propertySchema === null) { + return [name, value]; + } + + if (Reflect.get(propertySchema, "x-bt-type") === "prompt") { + return [ + name, + Prompt.fromPromptData(name, promptDataSchema.parse(value)), + ]; + } + + return [name, value]; + }), + ); +} diff --git a/js/src/exports.ts b/js/src/exports.ts index a8d2e62a2..9abb729d7 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -37,6 +37,7 @@ export { injectTraceContext, initDataset, initLogger, + loadParameters, loadPrompt, logError, login, diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index ccf4dd5d0..56149b5a7 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -2119,6 +2119,125 @@ describe("framework2 metadata support", () => { }); }); + describe("CodeParameters defaults", () => { + test("toFunctionDefinition initializes data with schema defaults", async () => { + const project = projects.create({ name: "test-project" }); + project.parameters.create({ + name: "test-parameters", + schema: { + title: z.string().default("default-title"), + numSamples: z.number().default(10), + enabled: z.boolean().default(true), + tags: z.array(z.string()).default(["default-tag"]), + config: z + .object({ + retryCount: z.number(), + strategy: z.string(), + }) + .default({ + retryCount: 3, + strategy: "balanced", + }), + datasetName: z.string(), + main: { + type: "prompt", + default: { + messages: [{ role: "user", content: "{{input}}" }], + model: "gpt-4", + }, + }, + model: { + type: "model", + default: "gpt-5-mini", + }, + }, + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const parameters = (project as any)._publishableParameters; + expect(parameters).toHaveLength(1); + + const mockProjectMap = { + resolve: vi.fn().mockResolvedValue("project-123"), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const funcDef = await parameters[0].toFunctionDefinition(mockProjectMap); + + expect(funcDef.function_data.type).toBe("parameters"); + expect(funcDef.function_data.data).toMatchObject({ + title: "default-title", + numSamples: 10, + enabled: true, + tags: ["default-tag"], + config: { + retryCount: 3, + strategy: "balanced", + }, + main: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "{{input}}" }], + }, + options: { + model: "gpt-4", + }, + }, + model: "gpt-5-mini", + }); + expect(funcDef.function_data.__schema.properties.model).toMatchObject({ + type: "string", + "x-bt-type": "model", + default: "gpt-5-mini", + }); + expect(funcDef.function_data.data).not.toHaveProperty("datasetName"); + }); + + test("toFunctionDefinition does not initialize data when schema has no defaults", async () => { + const project = projects.create({ name: "test-project" }); + project.parameters.create({ + name: "test-parameters-no-defaults", + schema: { + title: z.string(), + numSamples: z.number(), + enabled: z.boolean(), + tags: z.array(z.string()), + config: z.object({ + retryCount: z.number(), + strategy: z.string(), + }), + main: { + type: "prompt", + }, + model: { + type: "model", + }, + }, + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const parameters = (project as any)._publishableParameters; + expect(parameters).toHaveLength(1); + + const mockProjectMap = { + resolve: vi.fn().mockResolvedValue("project-123"), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const funcDef = await parameters[0].toFunctionDefinition(mockProjectMap); + + expect(funcDef.function_data.type).toBe("parameters"); + expect(funcDef.function_data.data).toEqual({}); + expect(funcDef.function_data.data).not.toHaveProperty("title"); + expect(funcDef.function_data.data).not.toHaveProperty("numSamples"); + expect(funcDef.function_data.data).not.toHaveProperty("enabled"); + expect(funcDef.function_data.data).not.toHaveProperty("tags"); + expect(funcDef.function_data.data).not.toHaveProperty("config"); + expect(funcDef.function_data.data).not.toHaveProperty("main"); + expect(funcDef.function_data.data).not.toHaveProperty("model"); + }); + }); + describe("Scorer metadata", () => { test("code scorer stores metadata correctly", () => { const project = projects.create({ name: "test-project" }); diff --git a/js/src/framework.ts b/js/src/framework.ts index 235b4974a..9a3d09710 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -21,7 +21,7 @@ import { v5 as uuidv5 } from "uuid"; import iso from "./isomorph"; import { debugLogger } from "./debug-logger"; import { GenericFunction } from "./framework-types"; -import { CodeFunction, CodePrompt } from "./framework2"; +import { CodeFunction, CodePrompt, CodeParameters } from "./framework2"; import { Trace, LocalTrace } from "./trace"; import { BaseMetadata, @@ -33,6 +33,8 @@ import { ExperimentSummary, FullInitOptions, NOOP_SPAN, + type ParametersRef, + RemoteEvalParameters, Span, StartSpanArgs, init as _initExperiment, @@ -54,8 +56,8 @@ import type { } from "./reporters/types"; import { isEmpty, InternalAbortError } from "./util"; import { - type EvalParameters, - type InferParameters, + EvalParameters, + InferParameters, validateParameters, } from "./eval-parameters"; @@ -140,6 +142,7 @@ export interface EvalHooks< span: Span; /** * The current parameters being used for this specific task execution. + * Array parameters are converted to single values. */ parameters: InferParameters; /** @@ -249,10 +252,18 @@ export interface Evaluator< classifiers?: EvalClassifier[]; /** - * Inline parameter schemas whose validated values are passed to task hooks. - * Use Zod schemas, prompt definitions, or model identifiers with optional defaults. + * A set of parameters that will be passed to the evaluator. + * Can be: + * - A raw EvalParameters schema (Zod schemas) + * - A Parameters instance from loadParameters() + * - A Promise from loadParameters() */ - parameters?: Parameters; + parameters?: + | Parameters + | RemoteEvalParameters> + | Promise< + RemoteEvalParameters> + >; /** * An optional name for the experiment. @@ -418,6 +429,7 @@ type EvaluatorFile = { GenericFunction >[]; prompts: CodePrompt[]; + parameters?: CodeParameters[]; evaluators: { [evalName: string]: { evaluator: EvaluatorDef< @@ -444,6 +456,34 @@ function initExperiment( }); } +async function getExperimentParametersRef( + parameters: + | EvalParameters + | RemoteEvalParameters + | Promise> + | undefined, +): Promise { + if (!parameters) { + return undefined; + } + + const resolvedParameters = + parameters instanceof Promise ? await parameters : parameters; + + if (!RemoteEvalParameters.isParameters(resolvedParameters)) { + return undefined; + } + + if (resolvedParameters.id === undefined) { + return undefined; + } + + return { + id: resolvedParameters.id, + version: resolvedParameters.version, + }; +} + export async function _internalInitEvaluatorExperiment( projectName: string, evaluator: Evaluator, @@ -456,6 +496,7 @@ export async function _internalInitEvaluatorExperiment( ): Promise { if (options.disabled) return null; const { baseExperiment } = callEvaluatorData(data); + const parameters = await getExperimentParametersRef(evaluator.parameters); return initExperiment(evaluator.state, { ...(evaluator.projectId ? { projectId: evaluator.projectId } @@ -471,6 +512,7 @@ export async function _internalInitEvaluatorExperiment( gitMetadataSettings: evaluator.gitMetadataSettings, repoInfo: evaluator.repoInfo, dataset: Dataset.isDataset(data) ? data : undefined, + parameters, }); } @@ -592,6 +634,7 @@ declare global { globalThis._evals = { functions: [], prompts: [], + parameters: [], evaluators: {}, reporters: {}, }; @@ -696,14 +739,13 @@ export async function Eval< | string | EvalOptions, ): Promise> { - let options: EvalOptions; - if (isEmpty(reporterOrOpts)) { - options = {}; - } else if (typeof reporterOrOpts === "string" || "name" in reporterOrOpts) { - options = { reporter: reporterOrOpts }; - } else { - options = reporterOrOpts; - } + const options: EvalOptions = isEmpty(reporterOrOpts) + ? {} + : typeof reporterOrOpts === "string" + ? { reporter: reporterOrOpts } + : "name" in reporterOrOpts + ? { reporter: reporterOrOpts } + : reporterOrOpts; let evalName = makeEvalName(name, evaluator.experimentName); if (globalThis._evals.evaluators[evalName]) { @@ -1096,16 +1138,6 @@ function logScoringFailures( return Object.keys(errorMap); } -export const defaultErrorScoreHandler: ErrorScoreHandler = ({ - rootSpan, - data: _, - unhandledScores, -}) => { - const scores = Object.fromEntries(unhandledScores.map((s) => [s, 0])); - rootSpan.log({ scores }); - return scores; -}; - export async function runEvaluator( experiment: Experiment | null, // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -1122,13 +1154,45 @@ export async function runEvaluator( "Evaluator must include at least one of `scores` or `classifiers`", ); } + return await runEvaluatorInternal( + experiment, + evaluator, + progressReporter, + stream, + parameters, + collectResults, + enableCache, + ); +} + +export const defaultErrorScoreHandler: ErrorScoreHandler = ({ + rootSpan, + data: _, + unhandledScores, +}) => { + const scores = Object.fromEntries(unhandledScores.map((s) => [s, 0])); + rootSpan.log({ scores }); + return scores; +}; + +async function runEvaluatorInternal( + experiment: Experiment | null, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + evaluator: EvaluatorDef, + progressReporter: ProgressReporter, + stream: ((data: SSEProgressEventData) => void) | undefined, + parameters: InferParameters | undefined, + collectResults: boolean, + enableCache: boolean, + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): Promise> { // Start span cache for this eval (it's disabled by default to avoid temp files outside of evals) // Only start if enableCache is true (default) if (enableCache) { (evaluator.state ?? _internalGetGlobalState())?.spanCache?.start(); } try { - const validatedParameters = validateParameters( + parameters = await validateParameters( parameters ?? {}, evaluator.parameters, ); @@ -1280,7 +1344,7 @@ export async function runEvaluator( evaluator.task, datum, trialIndex, - validatedParameters, + parameters ?? {}, span, (event) => { stream?.({ @@ -1668,7 +1732,7 @@ function ensureScoreAccumulator( export function buildLocalSummary( // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, + evaluator: EvaluatorDef, // eslint-disable-next-line @typescript-eslint/no-explicit-any results: EvalResult[], precomputedScores?: ScoreAccumulator, @@ -1745,7 +1809,7 @@ const defaultReporter: ReporterDef = { name: "Braintrust default reporter", async reportEval( // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, + evaluator: EvaluatorDef, // eslint-disable-next-line @typescript-eslint/no-explicit-any result: EvalResultWithSummary, { verbose, jsonl }: ReporterOpts, diff --git a/js/src/framework2.test.ts b/js/src/framework2.test.ts index 2310fb208..2503d743d 100644 --- a/js/src/framework2.test.ts +++ b/js/src/framework2.test.ts @@ -1,7 +1,12 @@ import { afterEach, beforeAll, expect, expectTypeOf, test, vi } from "vitest"; import * as z3 from "zod/v3"; import * as z4 from "zod/v4"; -import { CodeFunction, projects } from "./framework2"; +import { + CodeFunction, + projects, + CodeParameters, + ProjectNameIdMap, +} from "./framework2"; import { _internalGetGlobalState } from "./logger"; import { configureNode } from "./node/config"; import { zodToJsonSchema } from "./zod/utils"; @@ -107,3 +112,29 @@ test.each([ }); expect(zodToJsonSchema(fn.returns!)).toMatchObject({ type: "number" }); }); + +test("parameter builders serialize mixed Zod v3 and v4 schemas", async () => { + const project = projects.create({ id: "test-zod-parameters" }); + const schema = project.parameters.create({ + name: "mixed", + schema: { + prefix: z3.string().default("start:"), + suffix: z4.string().default(":end"), + }, + }); + expectTypeOf(schema.suffix).toEqualTypeOf>(); + const definition = await new CodeParameters(project, { + name: "mixed", + slug: "mixed", + schema, + }).toFunctionDefinition(new ProjectNameIdMap()); + expect(definition.function_data).toMatchObject({ + data: { prefix: "start:", suffix: ":end" }, + __schema: { + properties: { + prefix: { type: "string", default: "start:" }, + suffix: { type: "string", default: ":end" }, + }, + }, + }); +}); diff --git a/js/src/framework2.ts b/js/src/framework2.ts index de134ae2e..25c19a683 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -22,7 +22,14 @@ import { promptDefinitionToPromptData, type PromptDefinition, } from "./prompt-schemas"; -import type { ZodSchema } from "./zod/utils"; +import { zodToJsonSchema, type ZodSchema } from "./zod/utils"; +import type { EvalParameters } from "./eval-parameters"; +type ParametersSchema = { + type: "object"; + properties: Record>; + required?: string[]; + additionalProperties?: boolean; +}; // Safe access to __filename (only exists in Node.js CJS) const currentFilename = @@ -50,6 +57,7 @@ class Project { public readonly id?: string; public readonly projectGroupName?: string; public prompts: PromptBuilder; + public parameters: ParametersBuilder; public scorers: ScorerBuilder; public classifiers: ClassifierBuilder; @@ -62,6 +70,7 @@ class Project { GenericFunction >[] = []; private _publishablePrompts: CodePrompt[] = []; + private _publishableParameters: CodeParameters[] = []; constructor(args: CreateProjectOpts) { _initializeSpanContext(); @@ -69,6 +78,7 @@ class Project { this.id = "id" in args ? args.id : undefined; this.projectGroupName = "name" in args ? args.projectGroupName : undefined; this.prompts = new PromptBuilder(this); + this.parameters = new ParametersBuilder(this); this.scorers = new ScorerBuilder(this); this.classifiers = new ClassifierBuilder(this); } @@ -80,6 +90,16 @@ class Project { } } + public addParameters(parameters: CodeParameters) { + this._publishableParameters.push(parameters); + if (globalThis._lazy_load) { + if (globalThis._evals.parameters == null) + globalThis._evals.parameters = []; + + globalThis._evals.parameters.push(parameters); + } + } + public addCodeFunction( fn: CodeFunction< // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -443,9 +463,153 @@ class PromptBuilder { } } -interface PromptFunctionEvent { - prompt_data: PromptData; - function_data: { type: "prompt" }; +interface ParametersOpts { + name: string; + slug?: string; + description?: string; + schema: S; + ifExists?: IfExists; + metadata?: Record; +} + +export class CodeParameters { + public readonly project: Project; + public readonly name: string; + public readonly slug: string; + public readonly description?: string; + public readonly schema: EvalParameters; + public readonly ifExists?: IfExists; + public readonly metadata?: Record; + + constructor( + project: Project, + opts: { + name: string; + slug: string; + description?: string; + schema: EvalParameters; + ifExists?: IfExists; + metadata?: Record; + }, + ) { + this.project = project; + this.name = opts.name; + this.slug = opts.slug; + this.description = opts.description; + this.schema = opts.schema; + this.ifExists = opts.ifExists; + this.metadata = opts.metadata; + } + + async toFunctionDefinition( + projectNameToId: ProjectNameIdMap, + ): Promise { + const schema = serializeEvalParameterstoParametersSchema(this.schema); + return { + project_id: await projectNameToId.resolve(this.project), + name: this.name, + slug: this.slug, + description: this.description ?? "", + function_type: "parameters", + function_data: { + type: "parameters", + data: getDefaultDataFromParametersSchema(schema), + __schema: schema, + }, + if_exists: this.ifExists, + metadata: this.metadata, + }; + } +} + +class ParametersBuilder { + constructor(private readonly project: Project) {} + + public create(opts: ParametersOpts): S { + const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true }); + + const codeParameters = new CodeParameters(this.project, { + name: opts.name, + slug, + description: opts.description, + schema: opts.schema, + ifExists: opts.ifExists, + metadata: opts.metadata, + }); + + this.project.addParameters(codeParameters); + + return opts.schema; + } +} + +function serializeEvalParameterstoParametersSchema( + parameters: EvalParameters, +): ParametersSchema { + const properties: Record> = {}; + const required: string[] = []; + + for (const [name, value] of Object.entries(parameters)) { + if ("type" in value && value.type === "prompt") { + const defaultPromptData = value.default + ? promptDefinitionToPromptData(value.default) + : undefined; + + properties[name] = { + type: "object", + "x-bt-type": "prompt", + ...(value.description ? { description: value.description } : {}), + ...(defaultPromptData ? { default: defaultPromptData } : {}), + }; + + if (!defaultPromptData) { + required.push(name); + } + } else if ("type" in value && value.type === "model") { + properties[name] = { + type: "string", + "x-bt-type": "model", + ...(value.description ? { description: value.description } : {}), + ...("default" in value ? { default: value.default } : {}), + }; + + if (!("default" in value)) { + required.push(name); + } + } else { + const jsonSchema = zodToJsonSchema(value); + + properties[name] = jsonSchema; + + if (!("default" in jsonSchema)) { + required.push(name); + } + } + } + + return { + type: "object", + properties, + ...(required.length > 0 ? { required } : {}), + additionalProperties: true, + }; +} + +function getDefaultDataFromParametersSchema( + schema: ParametersSchema, +): Record { + return Object.fromEntries( + Object.entries(schema.properties).flatMap(([name, value]) => { + if (!("default" in value)) { + return []; + } + + return [[name, value.default]]; + }), + ); +} + +interface FunctionEventBase { project_id: string; slug: string; name: string; @@ -456,3 +620,67 @@ interface PromptFunctionEvent { metadata?: Record; environments?: { slug: string }[]; } + +interface PromptFunctionEvent extends FunctionEventBase { + prompt_data: PromptData; + function_data: { type: "prompt" }; +} + +interface ParametersFunctionEvent extends FunctionEventBase { + function_type: "parameters"; + function_data: { + type: "parameters"; + data: Record; + __schema: ParametersSchema; + }; +} + +export class ProjectNameIdMap { + private nameToId: Record = {}; + private idToName: Record = {}; + + async getId(projectName: string, projectGroupName?: string): Promise { + if (!(projectName in this.nameToId)) { + const response = await _internalGetGlobalState() + .appConn() + .post_json("api/project/register", { + project_name: projectName, + ...(projectGroupName ? { project_group_name: projectGroupName } : {}), + }); + + const result = z + .object({ + project: projectSchema, + }) + .parse(response); + + const projectId = result.project.id; + + this.nameToId[projectName] = projectId; + this.idToName[projectId] = projectName; + } + return this.nameToId[projectName]; + } + + async getName(projectId: string): Promise { + if (!(projectId in this.idToName)) { + const response = await _internalGetGlobalState() + .appConn() + .post_json("api/project/get", { + id: projectId, + }); + const result = z.array(projectSchema).nonempty().parse(response); + const projectName = result[0].name; + this.idToName[projectId] = projectName; + this.nameToId[projectName] = projectId; + } + return this.idToName[projectId]; + } + + async resolve(project: Project): Promise { + if (project.id) { + return project.id; + } + return this.getId(project.name!, project.projectGroupName); + } +} diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index 90cad7041..3bc66d532 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -17,9 +17,11 @@ import { initLogger, extractTraceContextFromHeaders, Prompt, + RemoteEvalParameters, BraintrustState, FailedHTTPResponse, loadPrompt, + loadParameters, wrapTraced, traced, currentSpan, @@ -46,6 +48,10 @@ import { type PromptDiskCacheEntry, type PromptMemoryCacheEntry, } from "./prompt-cache/prompt-cache"; +import { + ParametersCache, + type ParametersMemoryCacheEntry, +} from "./prompt-cache/parameters-cache"; import { DiskCache } from "./prompt-cache/disk-cache"; import { LRUCache } from "./lru-cache"; @@ -2198,7 +2204,7 @@ test("init surfaces dataset environment lookup errors instead of falling back to vi.restoreAllMocks(); }); -describe("prompt loaders", () => { +describe("prompt and parameters loaders", () => { let state: BraintrustState; let getJson: ReturnType; const promptRow = { @@ -2236,6 +2242,44 @@ describe("prompt loaders", () => { options: { model: string }; }; }; + const parametersRow = { + id: "44444444-4444-4444-8444-444444444444", + _xact_id: "v1", + project_id: "55555555-5555-4555-8555-555555555555", + name: "Saved parameters", + slug: "saved-parameters", + description: null, + function_type: "parameters", + function_data: { + type: "parameters", + data: { prefix: "hello" }, + __schema: { + type: "object", + properties: { + prefix: { type: "string", default: "hello" }, + }, + additionalProperties: true, + }, + }, + } satisfies { + id: string; + _xact_id: string; + project_id: string; + name: string; + slug: string; + description: null; + function_type: "parameters"; + function_data: { + type: "parameters"; + data: { prefix: string }; + __schema: { + type: string; + properties: { prefix: { type: string; default: string } }; + additionalProperties: boolean; + }; + }; + }; + function jsonResponse(data: unknown): Response { return new Response(JSON.stringify(data), { headers: { "Content-Type": "application/json" }, @@ -2950,6 +2994,135 @@ describe("prompt loaders", () => { "Organization prompt-org-name not found. Must be one of other-org-name", ); }); + + test("loadParameters uses request-local credentials and isolated caches", async () => { + state.parametersCache = new ParametersCache({ + memoryCache: new LRUCache({ + max: 10, + }), + }); + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + loginResponse({ + id: "parameters-org-id", + name: "test-org-name", + apiUrl: "https://parameters-api.test", + }), + ) + .mockResolvedValueOnce(jsonResponse({ objects: [parametersRow] })) + .mockResolvedValueOnce( + loginResponse({ + id: "parameters-org-id", + name: "test-org-name", + apiUrl: "https://parameters-api.test", + }), + ) + .mockResolvedValueOnce(serverErrorResponse()); + + await loadParameters({ + projectName: "test-project", + slug: "saved-parameters", + apiKey: "first-parameters-api-key", + fetch: fetchMock, + state, + }); + + await expect( + loadParameters({ + projectName: "test-project", + slug: "saved-parameters", + apiKey: "second-parameters-api-key", + fetch: fetchMock, + state, + }), + ).rejects.toThrow("not found on server or in local cache"); + expect(state.loginToken).toBe("___TEST_API_KEY__THIS_IS_NOT_REAL___"); + }); + + test("loadParameters never falls back on ambient authentication rejection", async () => { + getJson + .mockResolvedValueOnce({ objects: [parametersRow] }) + .mockRejectedValueOnce( + new FailedHTTPResponse(403, "Forbidden", "Revoked API key"), + ); + + await loadParameters({ + projectName: "test-project", + slug: "saved-parameters", + state, + }); + + await expect( + loadParameters({ + projectName: "test-project", + slug: "saved-parameters", + state, + }), + ).rejects.toThrow("403: Forbidden"); + }); + + test("resolved parameter caches validate the organization identity", async () => { + const cache = new ParametersCache({ + memoryCache: new LRUCache({ + max: 10, + }), + }); + const parameters = new RemoteEvalParameters(parametersRow); + const key = { + projectName: "test-project", + slug: "saved-parameters", + version: "latest", + }; + + await cache.withNamespace("credential", "first-org").set(key, parameters); + + expect( + await cache.withNamespace("credential", "first-org").get(key), + ).toBe(parameters); + expect(await cache.withNamespace("credential").get(key)).toBe(parameters); + expect( + await cache.withNamespace("credential", "second-org").get(key), + ).toBeUndefined(); + }); + }); + + test("loadParameters prefers version over environment for project lookup", async () => { + getJson.mockResolvedValue({ objects: [parametersRow] }); + + await loadParameters({ + projectName: "test-project", + slug: "saved-parameters", + version: "v1", + environment: "production", + state, + }); + + expect(getJson).toHaveBeenCalledWith( + "v1/function", + expect.objectContaining({ + project_name: "test-project", + slug: "saved-parameters", + version: "v1", + function_type: "parameters", + }), + ); + expect(getJson.mock.calls[0][1]).not.toHaveProperty("environment"); + }); + + test("loadParameters prefers version over environment for id lookup", async () => { + getJson.mockResolvedValue(parametersRow); + + await loadParameters({ + id: parametersRow.id, + version: "v1", + environment: "production", + state, + }); + + expect(getJson).toHaveBeenCalledWith(`v1/function/${parametersRow.id}`, { + version: "v1", + }); }); }); diff --git a/js/src/logger.ts b/js/src/logger.ts index d4c7a16eb..8ebaddaa4 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -145,6 +145,39 @@ export type DatasetRestoreResult = { rows_deleted: number; }; +const parametersRowSchema = z.object({ + id: z.string().uuid(), + _xact_id: z.string(), + project_id: z.string().uuid(), + name: z.string(), + slug: z.string(), + description: z.union([z.string(), z.null()]).optional(), + function_type: z.literal("parameters"), + function_data: z.object({ + type: z.literal("parameters"), + data: z.record(z.unknown()).optional(), + __schema: z.record(z.unknown()), + }), + metadata: z + .union([z.object({}).partial().passthrough(), z.null()]) + .optional(), +}); +type ParametersRow = { + id: string; + _xact_id: string; + project_id: string; + name: string; + slug: string; + description?: string | null; + function_type: "parameters"; + function_data: { + type: "parameters"; + data?: Record; + __schema: Record; + }; + metadata?: Record | null; +}; + import { waitUntil } from "@vercel/functions"; import { parseTemplateFormat, @@ -159,9 +192,17 @@ import { devNullWritableStream, } from "./functions/stream"; import iso, { IsoAsyncLocalStorage } from "./isomorph"; -import { createPromptCache } from "./prompt-cache/cache-config"; -import type { PromptCache } from "./prompt-cache/prompt-cache"; - +import { createCacheLayers } from "./prompt-cache/cache-config"; +import { + PromptCache, + type PromptDiskCacheEntry, + type PromptMemoryCacheEntry, +} from "./prompt-cache/prompt-cache"; +import { + ParametersCache, + type ParametersDiskCacheEntry, + type ParametersMemoryCacheEntry, +} from "./prompt-cache/parameters-cache"; import { LRUCache } from "./lru-cache"; import { addAzureBlobHeaders, @@ -175,6 +216,7 @@ import { import { lintTemplate as _lintMustacheTemplate } from "./template/mustache-utils"; import { prettifyXact } from "../util/index"; import { SpanCache, CachedSpan } from "./span-cache"; +import type { EvalParameters, InferParameters } from "./eval-parameters"; import { detectSpanOriginEnvironment, getSpanInstrumentationName, @@ -694,6 +736,11 @@ type ResolvedLoaderLoginOptions = Required< existingState?: BraintrustState; }; +type LoaderCacheViews = { + promptCache: PromptCache; + parametersCache: ParametersCache; +}; + type LoaderRequestState = Pick; // proxyUrl may point at the universal proxy (`{apiUrl}/v1/proxy`) for @@ -737,6 +784,7 @@ export class BraintrustState { private _proxyConn: HTTPConnection | null = null; public promptCache: PromptCache; + public parametersCache: ParametersCache; public spanCache: SpanCache; /** @internal */ public readonly projectMetadataCache = new LRUCache< @@ -800,7 +848,34 @@ export class BraintrustState { this.resetLoginInfo(); - this.promptCache = createPromptCache(); + const { memoryCache, diskCache } = createCacheLayers< + PromptMemoryCacheEntry, + PromptDiskCacheEntry + >({ + memoryMaxEnvVar: "BRAINTRUST_PROMPT_CACHE_MEMORY_MAX", + diskCacheDirEnvVar: "BRAINTRUST_PROMPT_CACHE_DIR", + diskMaxEnvVar: "BRAINTRUST_PROMPT_CACHE_DISK_MAX", + getDefaultDiskCacheDir: () => + `${iso.getEnv("HOME") ?? iso.homedir!()}/.braintrust/prompt_cache`, + }); + this.promptCache = new PromptCache({ memoryCache, diskCache }); + + const { + memoryCache: parametersMemoryCache, + diskCache: parametersDiskCache, + } = createCacheLayers( + { + memoryMaxEnvVar: "BRAINTRUST_PARAMETERS_CACHE_MEMORY_MAX", + diskCacheDirEnvVar: "BRAINTRUST_PARAMETERS_CACHE_DIR", + diskMaxEnvVar: "BRAINTRUST_PARAMETERS_CACHE_DISK_MAX", + getDefaultDiskCacheDir: () => + `${iso.getEnv("HOME") ?? iso.homedir!()}/.braintrust/parameters_cache`, + }, + ); + this.parametersCache = new ParametersCache({ + memoryCache: parametersMemoryCache, + diskCache: parametersDiskCache, + }); this.spanCache = new SpanCache({ disabled: loginParams.disableSpanCache }); this.spanOriginEnvironment = detectSpanOriginEnvironment(); @@ -916,10 +991,10 @@ export class BraintrustState { } /** @internal */ - public _internalGetPromptCache( + public _internalGetLoaderCacheViews( loginOptions: ResolvedLoaderLoginOptions, requestState?: LoaderRequestState, - ): PromptCache { + ): LoaderCacheViews { const expectedResolvedOrgIdentity = requestState?.orgId && requestState.appUrl ? JSON.stringify([ @@ -929,10 +1004,16 @@ export class BraintrustState { ]) : undefined; - return this.promptCache.withNamespace( - loginOptions.credentialCacheNamespace, - expectedResolvedOrgIdentity, - ); + return { + promptCache: this.promptCache.withNamespace( + loginOptions.credentialCacheNamespace, + expectedResolvedOrgIdentity, + ), + parametersCache: this.parametersCache.withNamespace( + loginOptions.credentialCacheNamespace, + expectedResolvedOrgIdentity, + ), + }; } /** @internal */ @@ -4064,11 +4145,17 @@ export type DatasetRef = { id: string; } & DatasetSelection; +export interface ParametersRef { + id: string; + version?: string; +} + export type InitOptions = FullLoginOptions & { experiment?: string; description?: string; dataset?: AnyDataset | DatasetRef; _internal_btql?: Record; + parameters?: ParametersRef | RemoteEvalParameters; update?: boolean; baseExperiment?: string; isPublic?: boolean; @@ -4148,6 +4235,7 @@ export function init( description, dataset, _internal_btql, + parameters, baseExperiment, isPublic, open, @@ -4295,6 +4383,18 @@ export function init( }; } + if (parameters !== undefined) { + if (RemoteEvalParameters.isParameters(parameters)) { + args["parameters_id"] = parameters.id; + args["parameters_version"] = parameters.version; + } else { + args["parameters_id"] = parameters.id; + if (parameters.version !== undefined) { + args["parameters_version"] = parameters.version; + } + } + } + if (isPublic !== undefined) { args["public"] = isPublic; } @@ -4970,6 +5070,99 @@ export type LoadPromptOptions = FullLoginOptions & { state?: BraintrustState; }; +type LoadParametersBaseOptions = FullLoginOptions & { + state?: BraintrustState; +}; + +export type LoadParametersByIdOptions = LoadParametersBaseOptions & { + id: string; + version?: string; +}; + +export type LoadParametersByIdWithEnvOptions = LoadParametersBaseOptions & { + id: string; + environment: string; + version?: string; +}; + +export type LoadParametersByProjectNameOptions = LoadParametersBaseOptions & { + projectName: string; + slug: string; + version?: string; +}; + +export type LoadParametersByProjectNameWithEnvOptions = + LoadParametersBaseOptions & { + projectName: string; + slug: string; + environment: string; + version?: string; + }; + +export type LoadParametersByProjectIdOptions = LoadParametersBaseOptions & { + projectId: string; + slug: string; + version?: string; +}; + +export type LoadParametersByProjectIdWithEnvOptions = + LoadParametersBaseOptions & { + projectId: string; + slug: string; + environment: string; + version?: string; + }; + +export type LoadParametersOptions = + | LoadParametersByIdOptions + | LoadParametersByIdWithEnvOptions + | LoadParametersByProjectNameOptions + | LoadParametersByProjectNameWithEnvOptions + | LoadParametersByProjectIdOptions + | LoadParametersByProjectIdWithEnvOptions; + +type LoadParametersImplementationOptions = LoadParametersBaseOptions & { + projectName?: string; + projectId?: string; + slug?: string; + version?: string; + id?: string; + environment?: string; +}; + +type LoaderRequestResult = + | ({ ok: true; response: T } & LoaderCacheViews) + | ({ ok: false; error: unknown } & LoaderCacheViews); + +async function runCredentialScopedLoaderRequest( + state: BraintrustState, + loginOptions: Pick< + FullLoginOptions, + "apiKey" | "appUrl" | "orgName" | "fetch" | "forceLogin" + >, + request: (requestState: LoaderRequestState) => Promise, +): Promise> { + const resolvedLoginOptions = + await state._internalResolveLoaderLoginOptions(loginOptions); + let cacheViews = state._internalGetLoaderCacheViews(resolvedLoginOptions); + + try { + const requestState = + await state._internalGetLoaderState(resolvedLoginOptions); + cacheViews = state._internalGetLoaderCacheViews( + resolvedLoginOptions, + requestState, + ); + return { + ok: true, + response: await request(requestState), + ...cacheViews, + }; + } catch (error) { + return { ok: false, error, ...cacheViews }; + } +} + /** * Load a prompt from the specified project. * @@ -5028,39 +5221,34 @@ export async function loadPrompt({ } const state = stateArg ?? _globalState; - const resolvedLoginOptions = await state._internalResolveLoaderLoginOptions({ - orgName, - apiKey, - appUrl, - fetch, - forceLogin, - }); - let promptCache = state._internalGetPromptCache(resolvedLoginOptions); - let response; - try { - const requestState = - await state._internalGetLoaderState(resolvedLoginOptions); - promptCache = state._internalGetPromptCache( - resolvedLoginOptions, - requestState, - ); - if (id) { - // Wrap the single prompt response to match the list API format. - response = await requestState - .apiConn() - .get_json(`v1/prompt/${id}`, versionOrEnvironment); - if (response) { - response = { objects: [response] }; + const result = await runCredentialScopedLoaderRequest( + state, + { orgName, apiKey, appUrl, fetch, forceLogin }, + async (requestState) => { + let response; + if (id) { + // Load prompt by ID using the /v1/prompt/{id} endpoint + response = await requestState + .apiConn() + .get_json(`v1/prompt/${id}`, versionOrEnvironment); + // Wrap single prompt response in objects array to match list API format + if (response) { + response = { objects: [response] }; + } + } else { + response = await requestState.apiConn().get_json("v1/prompt", { + project_name: projectName, + project_id: projectId, + slug, + ...versionOrEnvironment, + }); } - } else { - response = await requestState.apiConn().get_json("v1/prompt", { - project_name: projectName, - project_id: projectId, - slug, - ...versionOrEnvironment, - }); - } - } catch (e) { + return response; + }, + ); + const { promptCache } = result; + if (!result.ok) { + const e = result.error; if (!isLoaderCacheFallbackError(e)) { throw e; } @@ -5097,6 +5285,8 @@ export async function loadPrompt({ } return prompt; } + const { response } = result; + if (!("objects" in response) || response.objects.length === 0) { if (id) { throw new Error(`Prompt with id ${id} not found.`); @@ -5136,6 +5326,186 @@ export async function loadPrompt({ return prompt; } +/** + * Load parameters from the specified project. + * + * @param options Options for configuring loadParameters(). + * @param options.projectName The name of the project to load the parameters from. Must specify at least one of `projectName` or `projectId`. + * @param options.projectId The id of the project to load the parameters from. This takes precedence over `projectName` if specified. + * @param options.slug The slug of the parameters to load. + * @param options.version An optional version of the parameters (to read). If not specified, the latest version will be used. + * @param options.environment Fetch the version of the parameters assigned to the specified environment (e.g. "production", "staging"). If both `version` and `environment` are provided, `version` takes precedence. + * @param options.id The id of specific parameters to load. If specified, this takes precedence over all other parameters (project and slug). + * @param options.appUrl The URL of the Braintrust App. Defaults to https://www.braintrust.dev. + * @param options.apiKey The API key to use for this request, independently of any existing global login. If the parameter is not specified, will use an existing login or try the `BRAINTRUST_API_KEY` environment variable. In Node.js, if that is unset, will try the nearest `.env.braintrust` file in the current working directory or parent directories. + * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. + * @returns The parameters object. + * @throws If the parameters are not found. + * @throws If multiple parameters are found with the same slug in the same project (this should never happen). + * + * @example + * ```typescript + * const params = await loadParameters({ + * projectName: "My Project", + * slug: "my-parameters", + * }); + * ``` + */ +export function loadParameters( + options: LoadParametersByProjectNameOptions, +): Promise>>; +export function loadParameters( + options: LoadParametersByProjectNameWithEnvOptions, +): Promise>>; +export function loadParameters( + options: LoadParametersByProjectIdOptions, +): Promise>>; +export function loadParameters( + options: LoadParametersByProjectIdWithEnvOptions, +): Promise>>; +export function loadParameters( + options: LoadParametersByIdOptions, +): Promise>>; +export function loadParameters( + options: LoadParametersByIdWithEnvOptions, +): Promise>>; +export async function loadParameters< + S extends EvalParameters = EvalParameters, +>({ + projectName, + projectId, + slug, + version, + environment, + id, + appUrl, + apiKey, + orgName, + fetch, + forceLogin, + state: stateArg, +}: LoadParametersImplementationOptions): Promise< + RemoteEvalParameters> +> { + const versionOrEnvironment = version + ? { version } + : environment + ? { environment } + : {}; + if (id) { + // When loading by ID, we don't need project or slug + } else if (isEmpty(projectName) && isEmpty(projectId)) { + throw new Error("Must specify either projectName or projectId"); + } else if (isEmpty(slug)) { + throw new Error("Must specify slug"); + } + + const state = stateArg ?? _globalState; + const result = await runCredentialScopedLoaderRequest( + state, + { orgName, apiKey, appUrl, fetch, forceLogin }, + async (requestState) => { + let response; + if (id) { + response = await requestState.apiConn().get_json(`v1/function/${id}`, { + ...versionOrEnvironment, + }); + if (response) { + response = { objects: [response] }; + } + } else { + response = await requestState.apiConn().get_json("v1/function", { + project_name: projectName, + project_id: projectId, + slug, + function_type: "parameters", + ...versionOrEnvironment, + }); + } + return response; + }, + ); + const { parametersCache } = result; + if (!result.ok) { + const e = result.error; + if (!isLoaderCacheFallbackError(e)) { + throw e; + } + if (version || environment) { + throw new Error(`Parameters not found with specified parameters: ${e}`); + } + + debugLogger + .forState(state) + .warn("Failed to load parameters, attempting to fall back to cache:", e); + let parameters; + if (id) { + parameters = await parametersCache.get({ id }); + if (!parameters) { + throw new Error( + `Parameters with id ${id} not found (not found on server or in local cache): ${e}`, + ); + } + } else { + parameters = await parametersCache.get({ + slug, + projectId, + projectName, + version: version ?? "latest", + }); + if (!parameters) { + throw new Error( + `Parameters ${slug} (version ${version ?? "latest"}) not found in ${[ + projectName ?? projectId, + ]} (not found on server or in local cache): ${e}`, + ); + } + } + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return parameters as RemoteEvalParameters>; + } + const { response } = result; + + if (!("objects" in response) || response.objects.length === 0) { + if (id) { + throw new Error(`Parameters with id ${id} not found.`); + } else { + throw new Error( + `Parameters ${slug} not found in ${[projectName ?? projectId]}`, + ); + } + } else if (response.objects.length > 1) { + if (id) { + throw new Error( + `Multiple parameters found with id ${id}. This should never happen.`, + ); + } else { + throw new Error( + `Multiple parameters found with slug ${slug} in project ${ + projectName ?? projectId + }. This should never happen.`, + ); + } + } + + const metadata = parametersRowSchema.parse(response["objects"][0]); + const parameters = new RemoteEvalParameters(metadata); + try { + if (id) { + await parametersCache.set({ id }, parameters); + } else if (slug) { + await parametersCache.set( + { slug, projectId, projectName, version: version ?? "latest" }, + parameters, + ); + } + } catch (e) { + debugLogger.forState(state).warn("Failed to set parameters in cache:", e); + } + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return parameters as RemoteEvalParameters>; +} + /** * Options for logging in to Braintrust. */ @@ -9392,6 +9762,99 @@ export class Prompt< } } +export interface SerializedParameters { + metadata: ParametersRow; +} + +export let serializeParametersForCache: ( + parameters: RemoteEvalParameters, +) => SerializedParameters; + +export class RemoteEvalParameters< + HasId extends boolean = true, + HasVersion extends boolean = true, + T extends Record = Record, +> { + private readonly __braintrust_parameters_marker = true; + + constructor(private metadata: ParametersRow) { + void this.__braintrust_parameters_marker; + } + + public get id(): HasId extends true ? string : string | undefined { + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return this.metadata.id as HasId extends true ? string : string | undefined; + } + + public get projectId(): string | undefined { + return this.metadata.project_id; + } + + public get name(): string { + return this.metadata.name; + } + + public get slug(): string { + return this.metadata.slug; + } + + public get version(): HasVersion extends true + ? TransactionId + : TransactionId | undefined { + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return this.metadata[TRANSACTION_ID_FIELD] as HasVersion extends true + ? TransactionId + : TransactionId | undefined; + } + + public get schema(): Record { + return this.metadata.function_data.__schema; + } + + public get data(): T { + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + return (this.metadata.function_data.data ?? {}) as T; + } + + static { + serializeParametersForCache = (parameters) => ({ + metadata: parameters.metadata, + }); + } + + public validate(data: unknown): boolean { + if (typeof data !== "object" || data === null) { + return false; + } + const schemaProps = this.schema.properties; + if (typeof schemaProps !== "object" || schemaProps === null) { + return true; + } + for (const key of Object.keys(schemaProps)) { + if (!(key in data)) { + const required = Array.isArray(this.schema.required) + ? this.schema.required + : []; + if (required.includes(key)) { + return false; + } + } + } + return true; + } + + public static isParameters( + x: unknown, + ): x is RemoteEvalParameters> { + return ( + typeof x === "object" && + x !== null && + "__braintrust_parameters_marker" in x && + x.__braintrust_parameters_marker === true + ); + } +} + export type AnyDataset = Dataset; /** diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts index 3c9a10260..79cca4da8 100644 --- a/js/src/parameters.test.ts +++ b/js/src/parameters.test.ts @@ -1,7 +1,7 @@ -import { expect, expectTypeOf, test, beforeAll, vi } from "vitest"; +import { expect, expectTypeOf, test, beforeAll } from "vitest"; import { type InferParameters, validateParameters } from "./eval-parameters"; -import { runEvaluator } from "./framework"; -import { Eval, type Prompt } from "./exports"; +import { Eval, runEvaluator } from "./framework"; +import { RemoteEvalParameters } from "./logger"; import { z } from "zod/v3"; import * as z4 from "zod/v4"; import { type ProgressReporter } from "./reporters/types"; @@ -34,9 +34,9 @@ test("mixed Zod v3 and v4 parameters validate and infer parsed outputs", async ( optional: boolean | undefined; model: string; }>(); - expect( + await expect( validateParameters({ v3Length: "abc", v4Length: "abcd" }, schema), - ).toEqual({ + ).resolves.toEqual({ prefix: "start:", suffix: ":end", v3Length: 3, @@ -44,12 +44,12 @@ test("mixed Zod v3 and v4 parameters validate and infer parsed outputs", async ( optional: undefined, model: "test-model", }); - expect(() => + await expect( validateParameters( { suffix: 42, v3Length: "abc", v4Length: "abcd" }, schema, ), - ).toThrow("Invalid parameter 'suffix'"); + ).rejects.toThrow("Invalid parameter 'suffix'"); }); test("mixed Zod v3 and v4 parameters are inferred and passed to tasks", async () => { @@ -149,6 +149,59 @@ test("prompt parameter is passed correctly", async () => { expect(result.results[0].output).toBe("test input"); }); +test("remote prompt parameter is rehydrated correctly", async () => { + const parameters = new RemoteEvalParameters< + true, + true, + { + main: { + build: (args: { input: string }) => { + messages: Array<{ role: string; content: string }>; + model?: string; + }; + }; + } + >({ + id: "11111111-1111-4111-8111-111111111111", + _xact_id: "v1", + project_id: "22222222-2222-4222-8222-222222222222", + name: "Saved parameters", + slug: "saved-parameters", + function_type: "parameters", + function_data: { + type: "parameters", + data: { + main: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "{{input}}" }], + }, + options: { + model: "gpt-5-mini", + }, + }, + }, + __schema: { + type: "object", + properties: { + main: { + type: "object", + "x-bt-type": "prompt", + }, + }, + additionalProperties: true, + }, + }, + }); + + const validated = await validateParameters({}, parameters); + + expect((validated.main as any).build({ input: "test input" })).toMatchObject({ + messages: [{ role: "user", content: "test input" }], + model: "gpt-5-mini", + }); +}); + test("custom parameter values override defaults", async () => { const result = await runEvaluator( null, @@ -300,97 +353,3 @@ test("model parameter is required when default is missing", async () => { ), ).rejects.toThrow("Parameter 'model' is required"); }); - -test("Eval infers inline parameter types and applies value overrides", async () => { - const result = await Eval( - "inline-parameters", - { - data: [{ input: "hello" }], - parameters: { - prefix: z.string().default("default:"), - count: z.number().default(1), - model: { type: "model", default: "test-model" }, - prompt: { - type: "prompt", - default: { prompt: "{{input}}", model: "test-model" }, - }, - }, - task: (input, { parameters }) => { - expectTypeOf(parameters.prefix).toEqualTypeOf(); - expectTypeOf(parameters.count).toEqualTypeOf(); - expectTypeOf(parameters.model).toEqualTypeOf(); - expectTypeOf(parameters.prompt).toEqualTypeOf< - Prompt - >(); - expect( - parameters.prompt.build({ input }, { flavor: "completion" }).prompt, - ).toBe("hello"); - return `${parameters.prefix}${input}`.repeat(parameters.count); - }, - scores: [], - }, - { noSendLogs: true }, - ); - expect(result.results[0].output).toBe("default:hello"); - - const overridden = await Eval( - "inline-overrides", - { - data: [{ input: "hello" }], - parameters: { prefix: z.string().default("default:") }, - task: (input, { parameters }) => `${parameters.prefix}${input}`, - scores: [], - }, - { noSendLogs: true, parameters: { prefix: "custom:" } }, - ); - expect(overridden.results[0].output).toBe("custom:hello"); -}); - -test("invalid inline parameters reject before the task runs", async () => { - const task = vi.fn((input: string) => input); - await expect( - Eval( - "invalid-parameters", - { - data: [{ input: "hello" }], - parameters: { count: z.number().positive() }, - task, - scores: [], - }, - { noSendLogs: true, parameters: { count: -1 } }, - ), - ).rejects.toThrow("Invalid parameter 'count'"); - expect(task).not.toHaveBeenCalled(); -}); - -test("prompt parameters preserve inline provider tools and accept prompt values", () => { - const schema = { - main: { - type: "prompt" as const, - default: { - model: "test-model", - messages: [{ role: "user" as const, content: "{{input}}" }], - tools: [{ type: "function" as const, function: { name: "weather" } }], - }, - }, - }; - const defaults = validateParameters({}, schema); - expect(defaults.main.build({ input: "hello" }).tools).toEqual( - schema.main.default.tools, - ); - expect(validateParameters({ main: defaults.main }, schema).main).toBe( - defaults.main, - ); - const override = validateParameters( - { - main: { - prompt: { type: "completion", content: "Hi {{input}}" }, - options: { model: "other-model" }, - }, - }, - schema, - ); - expect( - override.main.build({ input: "world" }, { flavor: "completion" }).prompt, - ).toBe("Hi world"); -}); diff --git a/js/src/prompt-cache/cache-config.ts b/js/src/prompt-cache/cache-config.ts index 06f2559ba..66dcc5e87 100644 --- a/js/src/prompt-cache/cache-config.ts +++ b/js/src/prompt-cache/cache-config.ts @@ -2,11 +2,6 @@ import { debugLogger } from "../debug-logger"; import iso from "../isomorph"; import { canUseDiskCache, DiskCache } from "./disk-cache"; import { LRUCache } from "../lru-cache"; -import { - PromptCache, - type PromptMemoryCacheEntry, - type PromptDiskCacheEntry, -} from "./prompt-cache"; type CacheMode = "mixed" | "memory" | "disk" | "none"; @@ -33,7 +28,7 @@ function warnUnavailableDiskCache() { } warnedUnavailableDiskCacheMode = true; debugLogger.warn( - `Disk cache is not supported on this platform, so ${CACHE_LOCATION_ENV_VAR}="disk" disables prompt caching.`, + `Disk cache is not supported on this platform, so ${CACHE_LOCATION_ENV_VAR}="disk" disables prompt and parameters caching.`, ); } @@ -60,34 +55,45 @@ function parsePositiveIntegerEnv(envVar: string, defaultValue: number): number { return Number.isInteger(value) && value > 0 ? value : defaultValue; } -export function createPromptCache(): PromptCache { +export function createCacheLayers({ + memoryMaxEnvVar, + diskCacheDirEnvVar, + diskMaxEnvVar, + getDefaultDiskCacheDir, +}: { + memoryMaxEnvVar: string; + diskCacheDirEnvVar: string; + diskMaxEnvVar: string; + getDefaultDiskCacheDir: () => string; +}): { + memoryCache?: LRUCache; + diskCache?: DiskCache; +} { const mode = parseCacheMode(); const memoryCache = mode === "mixed" || mode === "memory" - ? new LRUCache({ + ? new LRUCache({ max: parsePositiveIntegerEnv( - "BRAINTRUST_PROMPT_CACHE_MEMORY_MAX", + memoryMaxEnvVar, DEFAULT_CACHE_MEMORY_MAX, ), }) : undefined; - let diskCache: DiskCache | undefined; + let diskCache: DiskCache | undefined; if (mode === "mixed" || mode === "disk") { if (canUseDiskCache()) { - diskCache = new DiskCache({ - cacheDir: - iso.getEnv("BRAINTRUST_PROMPT_CACHE_DIR") ?? - `${iso.getEnv("HOME") ?? iso.homedir!()}/.braintrust/prompt_cache`, - max: parsePositiveIntegerEnv( - "BRAINTRUST_PROMPT_CACHE_DISK_MAX", - DEFAULT_CACHE_DISK_MAX, - ), + diskCache = new DiskCache({ + cacheDir: iso.getEnv(diskCacheDirEnvVar) ?? getDefaultDiskCacheDir(), + max: parsePositiveIntegerEnv(diskMaxEnvVar, DEFAULT_CACHE_DISK_MAX), }); } else if (mode === "disk") { warnUnavailableDiskCache(); } } - return new PromptCache({ memoryCache, diskCache }); + if (diskCache) { + return { memoryCache, diskCache }; + } + return { memoryCache }; } diff --git a/js/src/prompt-cache/cache-mode.test.ts b/js/src/prompt-cache/cache-mode.test.ts index e19dc7b89..0a585b0c2 100644 --- a/js/src/prompt-cache/cache-mode.test.ts +++ b/js/src/prompt-cache/cache-mode.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "os"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import iso from "../isomorph"; import { configureNode } from "../node/config"; -import { BraintrustState, Prompt } from "../logger"; +import { BraintrustState, Prompt, RemoteEvalParameters } from "../logger"; configureNode(); @@ -14,6 +14,9 @@ const CACHE_ENV_VARS = [ "BRAINTRUST_PROMPT_CACHE_DIR", "BRAINTRUST_PROMPT_CACHE_MEMORY_MAX", "BRAINTRUST_PROMPT_CACHE_DISK_MAX", + "BRAINTRUST_PARAMETERS_CACHE_DIR", + "BRAINTRUST_PARAMETERS_CACHE_MEMORY_MAX", + "BRAINTRUST_PARAMETERS_CACHE_DISK_MAX", ]; const promptKey = { @@ -22,6 +25,12 @@ const promptKey = { version: "v1", }; +const parametersKey = { + projectId: "22222222-2222-4222-8222-222222222222", + slug: "saved-parameters", + version: "v1", +}; + const testPrompt = new Prompt( { id: "33333333-3333-4333-8333-333333333333", @@ -34,7 +43,27 @@ const testPrompt = new Prompt( false, ); -describe("prompt cache modes", () => { +const testParameters = new RemoteEvalParameters({ + id: "44444444-4444-4444-8444-444444444444", + _xact_id: "v1", + project_id: parametersKey.projectId, + name: "Saved parameters", + slug: parametersKey.slug, + description: null, + function_type: "parameters", + function_data: { + type: "parameters", + data: { prefix: "hello" }, + __schema: { + type: "object", + properties: { + prefix: { type: "string" }, + }, + }, + }, +}); + +describe("prompt and parameters cache modes", () => { const originalEnv = new Map(); const originalGzip = iso.gzip; const testDirs: string[] = []; @@ -75,61 +104,87 @@ describe("prompt cache modes", () => { async function configureCacheDirs() { const promptDir = await makeTempDir(); + const parametersDir = await makeTempDir(); process.env.BRAINTRUST_PROMPT_CACHE_DIR = promptDir; - return { promptDir }; + process.env.BRAINTRUST_PARAMETERS_CACHE_DIR = parametersDir; + return { promptDir, parametersDir }; } it("mixed mode uses memory and disk caches", async () => { process.env.BRAINTRUST_CACHE_LOCATION = " MiXeD "; process.env.BRAINTRUST_PROMPT_CACHE_MEMORY_MAX = "1"; + process.env.BRAINTRUST_PARAMETERS_CACHE_MEMORY_MAX = "1"; await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); await state.promptCache.set({ ...promptKey, slug: "other" }, testPrompt); + await state.parametersCache.set(parametersKey, testParameters); + await state.parametersCache.set( + { ...parametersKey, slug: "other" }, + testParameters, + ); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toEqual(testPrompt); + expect(await nextState.parametersCache.get(parametersKey)).toEqual( + testParameters, + ); }); it("memory mode does not write to disk", async () => { process.env.BRAINTRUST_CACHE_LOCATION = "memory"; - const { promptDir } = await configureCacheDirs(); + const { promptDir, parametersDir } = await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); + await state.parametersCache.set(parametersKey, testParameters); expect(await state.promptCache.get(promptKey)).toEqual(testPrompt); + expect(await state.parametersCache.get(parametersKey)).toEqual( + testParameters, + ); expect(await fs.readdir(promptDir)).toEqual([]); + expect(await fs.readdir(parametersDir)).toEqual([]); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toBeUndefined(); + expect(await nextState.parametersCache.get(parametersKey)).toBeUndefined(); }); it("disk mode does not warm memory", async () => { process.env.BRAINTRUST_CACHE_LOCATION = "disk"; - const { promptDir } = await configureCacheDirs(); + const { promptDir, parametersDir } = await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); + await state.parametersCache.set(parametersKey, testParameters); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toEqual(testPrompt); + expect(await nextState.parametersCache.get(parametersKey)).toEqual( + testParameters, + ); await fs.rm(promptDir, { recursive: true, force: true }); + await fs.rm(parametersDir, { recursive: true, force: true }); expect(await nextState.promptCache.get(promptKey)).toBeUndefined(); + expect(await nextState.parametersCache.get(parametersKey)).toBeUndefined(); }); it("none mode disables all cache reads and writes", async () => { process.env.BRAINTRUST_CACHE_LOCATION = "none"; - const { promptDir } = await configureCacheDirs(); + const { promptDir, parametersDir } = await configureCacheDirs(); const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); + await state.parametersCache.set(parametersKey, testParameters); expect(await state.promptCache.get(promptKey)).toBeUndefined(); + expect(await state.parametersCache.get(parametersKey)).toBeUndefined(); expect(await fs.readdir(promptDir)).toEqual([]); + expect(await fs.readdir(parametersDir)).toEqual([]); }); it("invalid cache mode warns once and falls back to mixed", async () => { @@ -140,9 +195,13 @@ describe("prompt cache modes", () => { const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); + await state.parametersCache.set(parametersKey, testParameters); const nextState = new BraintrustState({}); expect(await nextState.promptCache.get(promptKey)).toEqual(testPrompt); + expect(await nextState.parametersCache.get(parametersKey)).toEqual( + testParameters, + ); expect(warnSpy).toHaveBeenCalledTimes(1); expect(warnSpy).toHaveBeenCalledWith( @@ -161,14 +220,30 @@ describe("prompt cache modes", () => { { ...promptKey, slug: `prompt-${i}` }, testPrompt, ); + await state.parametersCache.set( + { ...parametersKey, slug: `parameters-${i}` }, + testParameters, + ); } expect( await state.promptCache.get({ ...promptKey, slug: "prompt-0" }), ).toBeUndefined(); + expect( + await state.parametersCache.get({ + ...parametersKey, + slug: "parameters-0", + }), + ).toBeUndefined(); expect( await state.promptCache.get({ ...promptKey, slug: "prompt-1024" }), ).toEqual(testPrompt); + expect( + await state.parametersCache.get({ + ...parametersKey, + slug: "parameters-1024", + }), + ).toEqual(testParameters); }); it("disk mode disables caching and warns once when disk cache is unavailable", async () => { @@ -180,12 +255,14 @@ describe("prompt cache modes", () => { const state = new BraintrustState({}); await state.promptCache.set(promptKey, testPrompt); expect(await state.promptCache.get(promptKey)).toBeUndefined(); + await state.parametersCache.set(parametersKey, testParameters); + expect(await state.parametersCache.get(parametersKey)).toBeUndefined(); new BraintrustState({}); expect(warnSpy).toHaveBeenCalledTimes(1); expect(warnSpy).toHaveBeenCalledWith( "[braintrust]", - 'Disk cache is not supported on this platform, so BRAINTRUST_CACHE_LOCATION="disk" disables prompt caching.', + 'Disk cache is not supported on this platform, so BRAINTRUST_CACHE_LOCATION="disk" disables prompt and parameters caching.', ); }); }); diff --git a/js/src/prompt-cache/parameters-cache.test.ts b/js/src/prompt-cache/parameters-cache.test.ts new file mode 100644 index 000000000..7fe320871 --- /dev/null +++ b/js/src/prompt-cache/parameters-cache.test.ts @@ -0,0 +1,69 @@ +import * as fs from "fs/promises"; +import * as path from "path"; +import { tmpdir } from "os"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { RemoteEvalParameters } from "../logger"; +import { configureNode } from "../node/config"; +import { DiskCache } from "./disk-cache"; +import { + ParametersCache, + type ParametersDiskCacheEntry, +} from "./parameters-cache"; + +describe("ParametersCache", () => { + configureNode(); + + let cacheDir: string; + + beforeEach(() => { + cacheDir = path.join( + tmpdir(), + `parameters-cache-test-${Date.now()}-${Math.random()}`, + ); + }); + + afterEach(async () => { + await fs.rm(cacheDir, { recursive: true, force: true }); + }); + + it("reconstructs parameters loaded from disk", async () => { + const key = { + projectId: "55555555-5555-4555-8555-555555555555", + slug: "saved-parameters", + version: "v1", + }; + const parameters = new RemoteEvalParameters({ + id: "44444444-4444-4444-8444-444444444444", + _xact_id: "v1", + project_id: key.projectId, + name: "Saved parameters", + slug: key.slug, + description: null, + function_type: "parameters", + function_data: { + type: "parameters", + data: { prefix: "hello" }, + __schema: { + type: "object", + properties: { prefix: { type: "string" } }, + required: ["prefix"], + }, + }, + }); + const diskCache = new DiskCache({ + cacheDir, + logWarnings: false, + }); + + await new ParametersCache({ diskCache }).set(key, parameters); + const result = await new ParametersCache({ diskCache }).get(key); + + expect(result).toBeInstanceOf(RemoteEvalParameters); + expect(result?.data).toEqual({ prefix: "hello" }); + expect(result?.schema).toEqual({ + type: "object", + properties: { prefix: { type: "string" } }, + required: ["prefix"], + }); + }); +}); diff --git a/js/src/prompt-cache/parameters-cache.ts b/js/src/prompt-cache/parameters-cache.ts new file mode 100644 index 000000000..530cfb28a --- /dev/null +++ b/js/src/prompt-cache/parameters-cache.ts @@ -0,0 +1,123 @@ +import { + RemoteEvalParameters, + serializeParametersForCache, + type SerializedParameters, +} from "../logger"; +import { LRUCache } from "../lru-cache"; +import { DiskCache } from "./disk-cache"; + +interface ParametersKey { + slug?: string; + version?: string; + projectId?: string; + projectName?: string; + id?: string; +} + +function createCacheKey(key: ParametersKey, namespace?: string): string { + let cacheKey: string; + if (key.id) { + cacheKey = `parameters:id:${key.id}`; + } else { + const prefix = key.projectId ?? key.projectName; + if (!prefix) { + throw new Error("Either projectId or projectName must be provided"); + } + if (!key.slug) { + throw new Error("Slug must be provided when not using ID"); + } + cacheKey = `parameters:${prefix}:${key.slug}:${key.version ?? "latest"}`; + } + return namespace === undefined + ? cacheKey + : `${namespace.length}:${namespace}:${cacheKey}`; +} + +export type ParametersMemoryCacheEntry = { + value: RemoteEvalParameters; + resolvedOrgIdentity?: string; +}; + +export type ParametersDiskCacheEntry = { + value: SerializedParameters; + resolvedOrgIdentity?: string; +}; + +export class ParametersCache { + private readonly memoryCache?: LRUCache; + private readonly diskCache?: DiskCache; + private readonly namespace?: string; + private readonly expectedResolvedOrgIdentity?: string; + + constructor(options: { + memoryCache?: LRUCache; + diskCache?: DiskCache; + namespace?: string; + expectedResolvedOrgIdentity?: string; + }) { + this.memoryCache = options.memoryCache; + this.diskCache = options.diskCache; + this.namespace = options.namespace; + this.expectedResolvedOrgIdentity = options.expectedResolvedOrgIdentity; + } + + withNamespace( + namespace: string, + expectedResolvedOrgIdentity?: string, + ): ParametersCache { + return new ParametersCache({ + memoryCache: this.memoryCache, + diskCache: this.diskCache, + namespace, + expectedResolvedOrgIdentity, + }); + } + + async get(key: ParametersKey): Promise { + const cacheKey = createCacheKey(key, this.namespace); + if (this.memoryCache) { + const memoryEntry = this.memoryCache.get(cacheKey); + if ( + memoryEntry !== undefined && + (this.expectedResolvedOrgIdentity === undefined || + memoryEntry.resolvedOrgIdentity === this.expectedResolvedOrgIdentity) + ) { + return memoryEntry.value; + } + } + + if (this.diskCache) { + const diskEntry = await this.diskCache.get(cacheKey); + if ( + !diskEntry || + (this.expectedResolvedOrgIdentity !== undefined && + diskEntry.resolvedOrgIdentity !== this.expectedResolvedOrgIdentity) + ) { + return undefined; + } + const diskParameters = new RemoteEvalParameters(diskEntry.value.metadata); + this.memoryCache?.set(cacheKey, { + value: diskParameters, + resolvedOrgIdentity: diskEntry.resolvedOrgIdentity, + }); + return diskParameters; + } + + return undefined; + } + + async set(key: ParametersKey, value: RemoteEvalParameters): Promise { + const cacheKey = createCacheKey(key, this.namespace); + const memoryEntry = { + value, + resolvedOrgIdentity: this.expectedResolvedOrgIdentity, + }; + this.memoryCache?.set(cacheKey, memoryEntry); + if (this.diskCache) { + await this.diskCache.set(cacheKey, { + value: serializeParametersForCache(value), + resolvedOrgIdentity: this.expectedResolvedOrgIdentity, + }); + } + } +} diff --git a/js/src/prompt-types.test.ts b/js/src/prompt-types.test.ts index 9d0c23239..813c69b50 100644 --- a/js/src/prompt-types.test.ts +++ b/js/src/prompt-types.test.ts @@ -87,7 +87,7 @@ test("prompt metadata preserves required and optional identifiers", () => { "log_id" | "org_id" | "project_id" | "id" | "_xact_id" | "function_type" > & { project_id?: string; - function_type?: Exclude; + function_type?: Exclude; }; type RequiredRow = Metadata & { id: string; _xact_id: string }; type OptionalRow = Metadata & { id?: string; _xact_id?: string }; diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts index 86442bfe6..16c0c1737 100644 --- a/js/src/public-types.test.ts +++ b/js/src/public-types.test.ts @@ -43,19 +43,15 @@ test("exports only intentional extension types", () => { expectTypeOf>().toBeFunction(); }); -test("does not expose deprecated tools or saved configurations", () => { - expect(publicExports).not.toHaveProperty("loadParameters"); +test("keeps parameters while removing deprecated tools", () => { + expect(publicExports.loadParameters).toBeTypeOf("function"); const project = publicExports.projects.create({ name: "test-project" }); expect(project).not.toHaveProperty("tools"); - expect(project).not.toHaveProperty("parameters"); + expect(project.parameters.create).toBeTypeOf("function"); if (false) { - // @ts-expect-error Saved configurations are no longer supported. - publicExports.loadParameters; // @ts-expect-error Saved tools are no longer supported. project.tools; - // @ts-expect-error Configuration builders are no longer supported. - project.parameters; const evaluator = { data: [{ input: 1 }], @@ -69,8 +65,10 @@ test("does not expose deprecated tools or saved configurations", () => { }); publicExports.Eval("test", { ...evaluator, - // @ts-expect-error Only inline schemas are supported, not remote configuration promises. - parameters: Promise.resolve(evaluator.parameters), + parameters: publicExports.loadParameters({ + projectName: "test", + slug: "settings", + }), }); project.prompts.create({ diff --git a/js/src/sdk-contracts.test.ts b/js/src/sdk-contracts.test.ts index 4430cf133..27e4ca8f3 100644 --- a/js/src/sdk-contracts.test.ts +++ b/js/src/sdk-contracts.test.ts @@ -6,7 +6,7 @@ import type * as SDK from "./sdk-types"; import type * as schemas from "./sdk-schemas"; // Backend definitions are compatibility fixtures, never production dependencies. -// Function contracts intentionally exclude the deprecated tool and parameters kinds. +// Function contracts intentionally exclude the deprecated tool kind. test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -14,7 +14,7 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toExtend(); expectTypeOf().toEqualTypeOf< - Exclude + Exclude >(); expectTypeOf< Exclude @@ -22,7 +22,7 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf< - Exclude + Exclude >(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -63,7 +63,7 @@ test("SDK-owned contracts remain compatible with the backend", () => { "tool_functions" > | null>(); expectTypeOf().toEqualTypeOf< - Exclude + Exclude >(); expectTypeOf().toExtend(); expectTypeOf().toExtend(); diff --git a/js/src/sdk-schemas.ts b/js/src/sdk-schemas.ts index ad20e5f37..7e5ad2cce 100644 --- a/js/src/sdk-schemas.ts +++ b/js/src/sdk-schemas.ts @@ -58,6 +58,7 @@ const FunctionTypeEnum = z.enum([ "facet", "classifier", "tag", + "parameters", "sandbox", ]); @@ -456,6 +457,7 @@ const FunctionObjectType = z.enum([ "preprocessor", "facet", "classifier", + "parameters", "sandbox", ]); diff --git a/js/src/sdk-types.ts b/js/src/sdk-types.ts index 641174985..77c6cfc81 100644 --- a/js/src/sdk-types.ts +++ b/js/src/sdk-types.ts @@ -103,6 +103,7 @@ export type FunctionTypeEnum = | "facet" | "classifier" | "tag" + | "parameters" | "sandbox"; export type CallEvent = | { @@ -445,6 +446,7 @@ export type FunctionObjectType = | "preprocessor" | "facet" | "classifier" + | "parameters" | "sandbox"; export type FunctionOutputType = | "completion" diff --git a/js/src/workflow-eval.test.ts b/js/src/workflow-eval.test.ts index 205d94efc..52e4cf691 100644 --- a/js/src/workflow-eval.test.ts +++ b/js/src/workflow-eval.test.ts @@ -1,5 +1,4 @@ import { describe, expect, expectTypeOf, test, vi } from "vitest"; -import { z } from "zod/v3"; import { configureNode } from "./node/config"; import { WorkflowBatchScorer, @@ -1530,115 +1529,3 @@ function workflowEval( classifier, }; } - -describe("workflow eval inline parameters", () => { - test("validates inline parameters once for ordinary tasks", async () => { - const transform = vi.fn((value: number) => value + 1); - const result = await defineWorkflowEval("inline-parameters", { - store: new WorkflowEvalMemoryStore(), - data: [{ input: "hello", expected: "hello:2" }], - parameters: { - count: z.number().default(1).transform(transform), - prompt: { - type: "prompt", - default: { prompt: "{{input}}:{{count}}", model: "test-model" }, - }, - }, - task: (input, { parameters }) => - parameters.prompt.build( - { input, count: parameters.count }, - { flavor: "completion" }, - ).prompt, - scores: [ - function exact({ output, expected }) { - return output === expected ? 1 : 0; - }, - ], - }).start({ noSendLogs: true }); - expect(transform).toHaveBeenCalledTimes(1); - expect(result).toMatchObject({ - status: "completed", - summary: { scores: { exact: { score: 1 } } }, - }); - }); - - test("batch tasks receive persisted inline parameter values", async () => { - const store = new WorkflowEvalMemoryStore(); - const schema = { - factor: z.number().default(4), - prompt: { - type: "prompt" as const, - default: { prompt: "{{input}}", model: "test-model" }, - }, - }; - let items: WorkflowBatchItem< - WorkflowTaskItem - >[] = []; - const task = new WorkflowBatchTask< - number, - number, - number, - void, - typeof schema, - null - >({ - batching: { maxSize: 2 }, - async submit(submitted) { - items = submitted; - expect(items[0].parameters.factor).toBe(2); - expect( - items[0].parameters.prompt.build( - { input: "hello" }, - { flavor: "completion" }, - ).prompt, - ).toBe("hello"); - return null; - }, - completion: { mode: "webhook", getExternalId: () => "inline-job" }, - async collect() { - return items.map(({ customId, input, parameters }) => ({ - customId, - result: { output: input * parameters.factor }, - })); - }, - }); - const evaluator = { - store, - parameters: schema, - data: [{ id: "case-one", input: 3, expected: 6 }], - task, - scores: [ - function exact({ - output, - expected, - }: { - output: number; - expected?: number; - }) { - return output === expected ? 1 : 0; - }, - ], - }; - const { Prompt } = await import("./logger"); - const run = await defineWorkflowEval("batch-parameters", evaluator).start({ - noSendLogs: true, - parameters: { - factor: 2, - prompt: Prompt.fromPromptData("main", { - prompt: { type: "completion", content: "{{input}}" }, - options: { model: "test-model" }, - }), - }, - }); - expect(run.status).toBe("waiting"); - const resumed = await defineWorkflowEval( - "batch-parameters", - evaluator, - ).processSubmissionResult({ runId: run.runId, externalId: "inline-job" }); - expect(resumed.status).toBe("completed"); - expect(resumed).toMatchObject({ - status: "completed", - summary: { scores: { exact: { score: 1 } } }, - }); - }); -}); diff --git a/js/src/workflow-eval.ts b/js/src/workflow-eval.ts index a094308e5..8e5ee9571 100644 --- a/js/src/workflow-eval.ts +++ b/js/src/workflow-eval.ts @@ -37,9 +37,6 @@ import { type Experiment, type ExperimentSummary, NOOP_SPAN, - Prompt, - serializePromptForCache, - type SerializedPrompt, type Span, _internalResumeSpan, _internalStartSpanWithInitialMerge, @@ -987,19 +984,10 @@ async function startWorkflowEval< const runId = newId(); const key = runKey(definition.projectName, definition.evalName, runId); const { data } = callEvaluatorData(definition.evaluator.data); - const parameters = validateParameters( + const parameters = await validateParameters( options.parameters ?? {}, definition.evaluator.parameters, ); - const serializedParameters = assertJsonValue( - Object.fromEntries( - Object.entries(parameters).map(([name, value]) => [ - name, - Prompt.isPrompt(value) ? serializePromptForCache(value) : value, - ]), - ), - "eval parameters", - ); const experimentName = definition.evaluator.experimentName ?? `${definition.evalName}-${runId}`; const experiment = await _internalInitEvaluatorExperiment( @@ -1029,8 +1017,6 @@ async function startWorkflowEval< projectName: definition.projectName, evalName: definition.evalName, data, - // Parameters were already validated above, including Zod transforms. - parameters: undefined, } as unknown as EvaluatorDef< Input, Output, @@ -1052,7 +1038,7 @@ async function startWorkflowEval< runId, experimentName, noSendLogs: options.noSendLogs ?? false, - parameters: serializedParameters, + parameters: assertJsonValue(parameters, "eval parameters"), status: "completed", summary: result.summary, caseCount: 0, @@ -1068,7 +1054,7 @@ async function startWorkflowEval< runId, experimentName, noSendLogs: options.noSendLogs ?? false, - parameters: serializedParameters, + parameters: assertJsonValue(parameters, "eval parameters"), status: "running", caseCount: cases.length, cases, @@ -1489,7 +1475,7 @@ async function logTaskResult( task, datum, record.trialIndex, - taskParameters(definition, state), + state.parameters as Record, span, ), { @@ -2022,11 +2008,9 @@ async function submitAndRecord( submission.kind, submission.scorerName, ); - const parameters = - submission.kind === "task" ? taskParameters(definition, state) : {}; const items = records.map((record) => submission.kind === "task" - ? taskSubmissionItem(record, parameters) + ? taskSubmissionItem(record, state.parameters) : scorerSubmissionItem(record), ); const context = { runId: state.runId, submissionId: submission.id }; @@ -2319,31 +2303,7 @@ async function materializeCases< return records; } -function taskParameters( - definition: WorkflowEvalRuntimeDefinition, - state: WorkflowRunState, -): Record { - return Object.fromEntries( - Object.entries(state.parameters as Record).map( - ([name, value]) => { - const schema = definition.evaluator.parameters?.[name]; - if (schema && "type" in schema && schema.type === "prompt") { - const prompt = value as SerializedPrompt; - return [ - name, - new Prompt(prompt.metadata, prompt.defaults, prompt.noTrace), - ]; - } - return [name, value]; - }, - ), - ); -} - -function taskSubmissionItem( - record: WorkflowCaseRecord, - parameters: Record, -) { +function taskSubmissionItem(record: WorkflowCaseRecord, parameters: JsonValue) { const datum = record.datum as EvalCase; return { id: record.id, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1bd7ad536..b9444e1a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -289,6 +289,9 @@ importers: acorn-import-attributes: specifier: ^1.9.5 version: 1.9.5(acorn@8.16.0) + ajv: + specifier: ^8.20.0 + version: 8.20.0 astring: specifier: ^1.9.0 version: 1.9.0