diff --git a/.changeset/remove-deprecated-tools-and-configurations.md b/.changeset/remove-deprecated-tools-and-configurations.md new file mode 100644 index 000000000..2dd31ed5b --- /dev/null +++ b/.changeset/remove-deprecated-tools-and-configurations.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +refactor(sdk)!: Remove deprecated tools and configurations diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 7e1d1e410..56149b5a7 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -18,6 +18,7 @@ import { } from "./framework"; import { _exportsForTestingOnly, + _internalGetGlobalState, BraintrustState, initLogger, injectTraceContext, @@ -1698,59 +1699,173 @@ import { z } from "zod/v3"; import { projects, CodePrompt } from "./framework2"; describe("framework2 metadata support", () => { + test("publishing resolves the project once for prompts and scorers", async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + const project = projects.create({ name: "test-project" }); + const projectId = "00000000-0000-4000-8000-000000000001"; + project.prompts.create({ + name: "completion", + model: "test-model", + prompt: "Hello {{name}}", + templateFormat: "nunjucks", + params: { temperature: 0.5 }, + }); + project.scorers.create({ + name: "quality", + model: "test-model", + messages: [{ role: "user", content: "Rate {{output}}" }], + templateFormat: "mustache", + useCot: true, + choiceScores: { good: 1, bad: 0 }, + }); + const register = vi + .spyOn(_internalGetGlobalState().appConn(), "post_json") + .mockResolvedValue({ + project: { id: projectId, org_id: projectId, name: "test-project" }, + }); + const insert = vi + .spyOn(_internalGetGlobalState().apiConn(), "post_json") + .mockResolvedValue({}); + try { + await project.publish(); + expect(register).toHaveBeenCalledExactlyOnceWith("api/project/register", { + project_name: "test-project", + }); + expect(insert).toHaveBeenCalledExactlyOnceWith("insert-functions", { + functions: [ + expect.objectContaining({ + project_id: projectId, + prompt_data: { + prompt: { type: "completion", content: "Hello {{name}}" }, + options: { model: "test-model", params: { temperature: 0.5 } }, + template_format: "nunjucks", + }, + }), + expect.objectContaining({ + project_id: projectId, + function_type: "scorer", + prompt_data: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "Rate {{output}}" }], + }, + options: { model: "test-model" }, + template_format: "mustache", + parser: { + type: "llm_classifier", + use_cot: true, + choice_scores: { good: 1, bad: 0 }, + }, + }, + }), + ], + }); + } finally { + register.mockRestore(); + insert.mockRestore(); + } + }); + + test("inline provider tools are built and published with prompts", async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + const project = projects.create({ id: "project-123" }); + const tools = [ + { + type: "function" as const, + function: { + name: "weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + }, + }, + }, + ]; + const prompt = project.prompts.create({ + name: "weather-prompt", + model: "test-model", + messages: [{ role: "user", content: "Weather in {{city}}?" }], + tools, + }); + expect(prompt.build({ city: "Paris" }).tools).toEqual(tools); + const insert = vi + .spyOn(_internalGetGlobalState().apiConn(), "post_json") + .mockResolvedValue({}); + try { + await project.publish(); + expect(insert).toHaveBeenCalledWith("insert-functions", { + functions: [ + expect.objectContaining({ + prompt_data: expect.objectContaining({ + prompt: expect.objectContaining({ tools: JSON.stringify(tools) }), + }), + }), + ], + }); + const definition = insert.mock.calls[0][1] as { + functions: { prompt_data: object }[]; + }; + expect(definition.functions[0].prompt_data).not.toHaveProperty( + "tool_functions", + ); + } finally { + insert.mockRestore(); + } + }); + describe("CodeFunction metadata", () => { - test("tool stores metadata correctly", () => { + test("classifier stores metadata correctly", () => { const project = projects.create({ name: "test-project" }); const metadata = { version: "1.0", author: "test" }; - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), metadata, }); - expect(tool.metadata).toEqual(metadata); - expect(tool.name).toBe("test-tool"); - expect(tool.slug).toBe("test-tool"); + expect(classifier.metadata).toEqual(metadata); + expect(classifier.name).toBe("test-classifier"); + expect(classifier.slug).toBe("test-classifier"); }); - test("tool works without metadata", () => { + test("classifier works without metadata", () => { const project = projects.create({ name: "test-project" }); - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), }); - expect(tool.metadata).toBeUndefined(); + expect(classifier.metadata).toBeUndefined(); }); - test("tool stores tags correctly", () => { + test("classifier stores tags correctly", () => { const project = projects.create({ name: "test-project" }); const tags = ["ci", "sdk"]; - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), tags, }); - expect(tool.tags).toEqual(tags); + expect(classifier.tags).toEqual(tags); }); - test("tool works without tags", () => { + test("classifier works without tags", () => { const project = projects.create({ name: "test-project" }); - const tool = project.tools.create({ - handler: (x: number) => x * 2, - name: "test-tool", - parameters: z.object({ x: z.number() }), + const classifier = project.classifiers.create({ + handler: ({ output }: { output: number }) => output * 2, + name: "test-classifier", + parameters: z.object({ output: z.number() }), }); - expect(tool.tags).toBeUndefined(); + expect(classifier.tags).toBeUndefined(); }); test("classifier registers as a code function", () => { @@ -1844,7 +1959,7 @@ describe("framework2 metadata support", () => { expect(prompts[0].metadata).toBeUndefined(); }); - test("toFunctionDefinition includes metadata when present", async () => { + test("toFunctionDefinition includes metadata when present", () => { const project = projects.create({ name: "test-project" }); const metadata = { version: "2.0", tag: "production" }; @@ -1854,28 +1969,21 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any { name: "test-prompt", slug: "test-prompt", metadata, - } as any, + }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.metadata).toEqual(metadata); expect(funcDef.name).toBe("test-prompt"); expect(funcDef.project_id).toBe("project-123"); }); - test("toFunctionDefinition excludes metadata when undefined", async () => { + test("toFunctionDefinition excludes metadata when undefined", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1884,22 +1992,15 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt" } as any, + { name: "test-prompt", slug: "test-prompt" }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.metadata).toBeUndefined(); }); - test("toFunctionDefinition includes environments for single environment", async () => { + test("toFunctionDefinition includes environments for single environment", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1908,26 +2009,19 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any { name: "test-prompt", slug: "test-prompt", environments: ["production"], - } as any, + }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.environments).toEqual([{ slug: "production" }]); }); - test("toFunctionDefinition includes environments for multiple environments", async () => { + test("toFunctionDefinition includes environments for multiple environments", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1936,21 +2030,14 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any { name: "test-prompt", slug: "test-prompt", environments: ["staging", "production"], - } as any, + }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.environments).toEqual([ { slug: "staging" }, @@ -1958,7 +2045,7 @@ describe("framework2 metadata support", () => { ]); }); - test("toFunctionDefinition excludes environments when undefined", async () => { + test("toFunctionDefinition excludes environments when undefined", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -1967,17 +2054,10 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt" } as any, + { name: "test-prompt", slug: "test-prompt" }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.environments).toBeUndefined(); }); @@ -2001,7 +2081,7 @@ describe("framework2 metadata support", () => { expect(prompts[0].tags).toEqual(tags); }); - test("toFunctionDefinition includes tags when present", async () => { + test("toFunctionDefinition includes tags when present", () => { const project = projects.create({ name: "test-project" }); const tags = ["ci", "production"]; @@ -2011,24 +2091,17 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt", tags } as any, + { name: "test-prompt", slug: "test-prompt", tags }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.tags).toEqual(tags); expect(funcDef.name).toBe("test-prompt"); expect(funcDef.project_id).toBe("project-123"); }); - test("toFunctionDefinition excludes tags when undefined", async () => { + test("toFunctionDefinition excludes tags when undefined", () => { const project = projects.create({ name: "test-project" }); const codePrompt = new CodePrompt( @@ -2037,17 +2110,10 @@ describe("framework2 metadata support", () => { prompt: { type: "completion", content: "Hello {{name}}" }, options: { model: "gpt-4" }, }, - [], - // eslint-disable-next-line @typescript-eslint/no-explicit-any - { name: "test-prompt", slug: "test-prompt" } as any, + { name: "test-prompt", slug: "test-prompt" }, ); - const mockProjectMap = { - resolve: vi.fn().mockResolvedValue("project-123"), - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any; - - const funcDef = await codePrompt.toFunctionDefinition(mockProjectMap); + const funcDef = codePrompt.toFunctionDefinition("project-123"); expect(funcDef.tags).toBeUndefined(); }); diff --git a/js/src/framework2.test.ts b/js/src/framework2.test.ts index 1097f2a26..2503d743d 100644 --- a/js/src/framework2.test.ts +++ b/js/src/framework2.test.ts @@ -3,9 +3,9 @@ import * as z3 from "zod/v3"; import * as z4 from "zod/v4"; import { CodeFunction, - ProjectNameIdMap, projects, CodeParameters, + ProjectNameIdMap, } from "./framework2"; import { _internalGetGlobalState } from "./logger"; import { configureNode } from "./node/config"; @@ -20,6 +20,10 @@ afterEach(() => { }); function mockProjectRegister() { + vi.spyOn(_internalGetGlobalState(), "login").mockResolvedValue(undefined); + vi.spyOn(_internalGetGlobalState(), "apiConn").mockReturnValue({ + post_json: vi.fn(async () => ({})), + } as never); const postJson = vi.fn(async () => ({ project: { id: "00000000-0000-0000-0000-000000000001", @@ -40,9 +44,12 @@ test("project resolution creates the project in its project group", async () => projectGroupName: "my-group", }); - expect(await new ProjectNameIdMap().resolve(project)).toBe( - "00000000-0000-0000-0000-000000000001", - ); + project.prompts.create({ + name: "greeting", + model: "test-model", + prompt: "Hello", + }); + await project.publish(); expect(postJson).toHaveBeenCalledWith("api/project/register", { project_name: "project", project_group_name: "my-group", @@ -52,7 +59,13 @@ test("project resolution creates the project in its project group", async () => test("project resolution omits project_group_name when unspecified", async () => { const postJson = mockProjectRegister(); - await new ProjectNameIdMap().resolve(projects.create({ name: "project" })); + const project = projects.create({ name: "project" }); + project.prompts.create({ + name: "greeting", + model: "test-model", + prompt: "Hello", + }); + await project.publish(); expect(postJson).toHaveBeenCalledWith("api/project/register", { project_name: "project", }); @@ -79,7 +92,7 @@ test.each([ const fn = new CodeFunction(project, { name: "length", slug: "length", - type: "tool", + type: "scorer", parameters, returns, handler: ({ text }) => { @@ -98,18 +111,6 @@ test.each([ required: ["text"], }); expect(zodToJsonSchema(fn.returns!)).toMatchObject({ type: "number" }); - - const tool = project.tools.create({ - name: "length-tool", - parameters, - returns, - handler: ({ text }) => { - expectTypeOf(text).toEqualTypeOf(); - return text.length; - }, - }); - expectTypeOf(tool.handler({ text: "hello" })).toEqualTypeOf(); - expect(tool.handler({ text: "hello" })).toBe(5); }); test("parameter builders serialize mixed Zod v3 and v4 schemas", async () => { diff --git a/js/src/framework2.ts b/js/src/framework2.ts index 0db3c1a64..25c19a683 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -7,11 +7,8 @@ import { Project as projectSchema } from "./sdk-schemas"; import type { FunctionTypeEnum as FunctionType, IfExists, - SavedFunctionId, - PromptBlockData, PromptData, ToolFunctionDefinition, - ExtendedSavedFunctionId, } from "./sdk-types"; import { loadPrettyXact, TransactionId } from "../util/index"; import { @@ -21,12 +18,12 @@ import { PromptRowWithId, } from "./logger"; import type { BaseFnOpts, GenericFunction } from "./framework-types"; -import type { EvalParameters } from "./eval-parameters"; import { promptDefinitionToPromptData, type PromptDefinition, } from "./prompt-schemas"; import { zodToJsonSchema, type ZodSchema } from "./zod/utils"; +import type { EvalParameters } from "./eval-parameters"; type ParametersSchema = { type: "object"; properties: Record>; @@ -59,7 +56,6 @@ class Project { public readonly name?: string; public readonly id?: string; public readonly projectGroupName?: string; - public tools: ToolBuilder; public prompts: PromptBuilder; public parameters: ParametersBuilder; public scorers: ScorerBuilder; @@ -81,7 +77,6 @@ class Project { this.name = "name" in args ? args.name : undefined; this.id = "id" in args ? args.id : undefined; this.projectGroupName = "name" in args ? args.projectGroupName : undefined; - this.tools = new ToolBuilder(this); this.prompts = new PromptBuilder(this); this.parameters = new ParametersBuilder(this); this.scorers = new ScorerBuilder(this); @@ -128,7 +123,6 @@ class Project { return; } await login(); - const projectMap = new ProjectNameIdMap(); const functionDefinitions: PromptFunctionEvent[] = []; if (this._publishableCodeFunctions.length > 0) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. @@ -137,10 +131,20 @@ class Project { ); } if (this._publishablePrompts.length > 0) { + const projectId = + this.id ?? + z.object({ project: projectSchema }).parse( + await _internalGetGlobalState() + .appConn() + .post_json("api/project/register", { + project_name: this.name, + ...(this.projectGroupName + ? { project_group_name: this.projectGroupName } + : {}), + }), + ).project.id; for (const prompt of this._publishablePrompts) { - const functionDefinition = - await prompt.toFunctionDefinition(projectMap); - functionDefinitions.push(functionDefinition); + functionDefinitions.push(prompt.toFunctionDefinition(projectId)); } } @@ -150,76 +154,6 @@ class Project { } } -class ToolBuilder { - private taskCounter = 0; - constructor(private readonly project: Project) {} - - public create< - // eslint-disable-next-line @typescript-eslint/no-explicit-any - TParams extends { _output: any; _input: any; _def: any }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - TReturns extends { _output: any; _input: any; _def: any }, - THandler extends GenericFunction, - >( - opts: Partial & { - handler: THandler; - parameters: TParams; - returns: TReturns; - }, - ): CodeFunction; - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - public create>( - opts: Partial & { - handler: THandler; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - parameters?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - returns?: any; - }, // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): CodeFunction; - - // This type definition is just a catch all so that the implementation can be - // less specific than the two more specific declarations above. - public create( - opts: Partial & { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - handler: GenericFunction; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - parameters?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - returns?: any; - }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): CodeFunction { - this.taskCounter++; - opts = opts ?? {}; - - const { handler, name, slug, parameters, returns, ...rest } = opts; - let resolvedName = name ?? handler.name; - - if (resolvedName.trim().length === 0) { - resolvedName = `Tool ${iso.basename(currentFilename)} ${this.taskCounter}`; - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const tool: CodeFunction = new CodeFunction(this.project, { - handler, - name: resolvedName, - slug: slug ?? slugify(resolvedName, { lower: true, strict: true }), - type: "tool", - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - parameters: parameters as any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/consistent-type-assertions - returns: returns as any, - ...rest, - }); - - this.project.addCodeFunction(tool); - return tool; - } -} - class ScorerBuilder { private taskCounter = 0; constructor(private readonly project: Project) {} @@ -259,35 +193,17 @@ class ScorerBuilder { }); this.project.addCodeFunction(scorer); } else { - const promptBlock: PromptBlockData = - "messages" in opts - ? { - type: "chat", - messages: opts.messages, - } - : { - type: "completion", - content: opts.prompt, - }; const promptData: PromptData = { - prompt: promptBlock, - options: { - model: opts.model, - params: opts.params, - }, + ...promptDefinitionToPromptData(opts), parser: { type: "llm_classifier", use_cot: opts.useCot, choice_scores: opts.choiceScores, }, - ...(opts.templateFormat - ? { template_format: opts.templateFormat } - : {}), }; const codePrompt = new CodePrompt( this.project, promptData, - [], { ...opts, name: resolvedName, @@ -313,7 +229,7 @@ class ClassifierBuilder { Exact>, Returns >, - >(opts: ClassifierOpts) { + >(opts: CodeOpts>, Returns, Fn>) { this.taskCounter++; let resolvedName = opts.name ?? opts.handler.name; @@ -346,14 +262,15 @@ type CodeOpts< handler: Fn; parameters?: ZodSchema; returns?: ZodSchema; - metadata?: Record; }; type ScorerPromptOpts = Partial & - PromptOpts & { + PromptDefinition & { + name: string; + id?: string; + version?: TransactionId; useCot: boolean; choiceScores: Record; - metadata?: Record; }; // A more correct ScorerArgs than that in core/js/src/score.ts. @@ -371,7 +288,7 @@ type Exact = T extends Shape : never : never; -type ScorerOptsUnion< +type ScorerOpts< Output, Input, Params, @@ -381,26 +298,6 @@ type ScorerOptsUnion< | CodeOpts>, Returns, Fn> | ScorerPromptOpts; -type ScorerOpts< - Output, - Input, - Params, - Returns, - Fn extends GenericFunction>, Returns>, -> = ScorerOptsUnion & { - metadata?: Record; -}; - -type ClassifierOpts< - Output, - Input, - Params, - Returns, - Fn extends GenericFunction>, Returns>, -> = CodeOpts>, Returns, Fn> & { - metadata?: Record; -}; - export class CodeFunction< Input, Output, @@ -453,17 +350,6 @@ export class CodeFunction< } } -type GenericCodeFunction = CodeFunction< - // This has to be marked as any because we want to support arrays of - // functions that return different things. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - GenericFunction ->; - export class CodePrompt { public readonly project: Project; public readonly name: string; @@ -473,7 +359,6 @@ export class CodePrompt { public readonly description?: string; public readonly id?: string; public readonly functionType?: FunctionType; - public readonly toolFunctions: (SavedFunctionId | GenericCodeFunction)[]; public readonly tags?: string[]; public readonly metadata?: Record; public readonly environmentSlugs?: string[]; @@ -481,10 +366,11 @@ export class CodePrompt { constructor( project: Project, prompt: PromptData, - toolFunctions: (SavedFunctionId | GenericCodeFunction)[], - opts: Omit, "name" | "slug"> & { + opts: Partial & { name: string; slug: string; + id?: string; + environments?: string[]; }, functionType?: FunctionType, ) { @@ -492,7 +378,6 @@ export class CodePrompt { this.name = opts.name; this.slug = opts.slug; this.prompt = prompt; - this.toolFunctions = toolFunctions; this.ifExists = opts.ifExists; this.description = opts.description; this.id = opts.id; @@ -502,35 +387,9 @@ export class CodePrompt { this.environmentSlugs = opts.environments; } - async toFunctionDefinition( - projectNameToId: ProjectNameIdMap, - ): Promise { - const prompt_data = { - ...this.prompt, - }; - if (this.toolFunctions.length > 0) { - const resolvableToolFunctions: ExtendedSavedFunctionId[] = - await Promise.all( - this.toolFunctions.map(async (fn) => { - if ("slug" in fn) { - return { - type: "slug", - project_id: await projectNameToId.resolve(fn.project), - slug: fn.slug, - }; - } else { - return fn; - } - }), - ); - - // This is a hack because these will be resolved on the server side. - prompt_data.tool_functions = - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - resolvableToolFunctions as SavedFunctionId[]; - } + toFunctionDefinition(projectId: string): PromptFunctionEvent { return { - project_id: await projectNameToId.resolve(this.project), + project_id: projectId, name: this.name, slug: this.slug, description: this.description ?? "", @@ -538,7 +397,7 @@ export class CodePrompt { type: "prompt", }, function_type: this.functionType, - prompt_data, + prompt_data: this.prompt, if_exists: this.ifExists, tags: this.tags, metadata: this.metadata, @@ -550,34 +409,17 @@ export class CodePrompt { } } -interface PromptId { - id: string; -} - -interface PromptVersion { - version: TransactionId; -} - -interface PromptTools { - tools: (GenericCodeFunction | SavedFunctionId | ToolFunctionDefinition)[]; -} - -interface PromptNoTrace { - noTrace: boolean; -} - type PromptOpts< HasId extends boolean, HasVersion extends boolean, - HasTools extends boolean = true, - HasNoTrace extends boolean = true, -> = (Partial> & { name: string }) & - (HasId extends true ? PromptId : Partial) & - (HasVersion extends true ? PromptVersion : Partial) & - // eslint-disable-next-line @typescript-eslint/no-empty-object-type - (HasTools extends true ? Partial : {}) & - // eslint-disable-next-line @typescript-eslint/no-empty-object-type - (HasNoTrace extends true ? Partial : {}) & +> = Partial & { + name: string; + tools?: ToolFunctionDefinition[]; + noTrace?: boolean; +} & (HasId extends true ? { id: string } : { id?: string }) & + (HasVersion extends true + ? { version: TransactionId } + : { version?: TransactionId }) & PromptDefinition; class PromptBuilder { @@ -587,22 +429,12 @@ class PromptBuilder { HasId extends boolean = false, HasVersion extends boolean = false, >(opts: PromptOpts): Prompt { - const toolFunctions: (SavedFunctionId | GenericCodeFunction)[] = []; - const rawTools: ToolFunctionDefinition[] = []; - - for (const tool of opts.tools ?? []) { - if (tool instanceof CodeFunction) { - toolFunctions.push(tool); - } else if ("type" in tool && !("function" in tool)) { - toolFunctions.push(tool); - } else { - rawTools.push(tool); - } - } - const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true }); - const promptData: PromptData = promptDefinitionToPromptData(opts, rawTools); + const promptData: PromptData = promptDefinitionToPromptData( + opts, + opts.tools, + ); // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const promptRow: PromptRowWithId = { @@ -621,7 +453,7 @@ class PromptBuilder { opts.noTrace ?? false, ); - const codePrompt = new CodePrompt(this.project, promptData, toolFunctions, { + const codePrompt = new CodePrompt(this.project, promptData, { ...opts, slug, }); diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index 94a433212..be27c9ed9 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -1,17 +1,14 @@ import type { - ToolFunctionDefinition, ChatCompletionMessageParam, ModelParams, - PromptBlockData, PromptData, + ToolFunctionDefinition, } from "./sdk-types"; -// This roughly maps to promptBlockDataSchema, but is more ergonomic for the user. -export type PromptContents = +export type PromptDefinition = ( | { prompt: string } - | { messages: ChatCompletionMessageParam[] }; - -export type PromptDefinition = PromptContents & { + | { messages: ChatCompletionMessageParam[] } +) & { model: string; params?: ModelParams; templateFormat?: "mustache" | "nunjucks" | "none"; @@ -26,20 +23,20 @@ export function promptDefinitionToPromptData( promptDefinition: PromptDefinition, rawTools?: ToolFunctionDefinition[], ): PromptData { - const promptBlock: PromptBlockData = - "messages" in promptDefinition - ? { - type: "chat", - messages: promptDefinition.messages, - tools: - rawTools && rawTools.length > 0 - ? JSON.stringify(rawTools) - : undefined, - } - : { - type: "completion", - content: promptDefinition.prompt, - }; + let promptBlock: PromptData["prompt"]; + if ("messages" in promptDefinition) { + promptBlock = { + type: "chat", + messages: promptDefinition.messages, + tools: + rawTools && rawTools.length > 0 ? JSON.stringify(rawTools) : undefined, + }; + } else { + promptBlock = { + type: "completion", + content: promptDefinition.prompt, + }; + } return { prompt: promptBlock, diff --git a/js/src/prompt-types.test.ts b/js/src/prompt-types.test.ts index 51df00f1b..813c69b50 100644 --- a/js/src/prompt-types.test.ts +++ b/js/src/prompt-types.test.ts @@ -8,10 +8,9 @@ import { type DefaultPromptArgs, type PromptRowWithId, } from "./logger"; -import type { CodeParameters, CodePrompt } from "./framework2"; +import type { CodePrompt } from "./framework2"; import type { AnyModelParamsType, - FunctionDataType, PromptDataType, PromptType, } from "./generated_plain_types"; @@ -85,9 +84,10 @@ test("compiled prompts preserve provider parameter types and compatibility", () test("prompt metadata preserves required and optional identifiers", () => { type Metadata = Omit< PromptType, - "log_id" | "org_id" | "project_id" | "id" | "_xact_id" + "log_id" | "org_id" | "project_id" | "id" | "_xact_id" | "function_type" > & { project_id?: string; + function_type?: Exclude; }; type RequiredRow = Metadata & { id: string; _xact_id: string }; type OptionalRow = Metadata & { id?: string; _xact_id?: string }; @@ -114,22 +114,10 @@ test("prompt metadata preserves required and optional identifiers", () => { test("function definitions expose only the variant each builder produces", () => { type PromptEvent = Awaited>; - type ParametersEvent = Awaited< - ReturnType - >; expectTypeOf().toEqualTypeOf<{ type: "prompt"; }>(); - expectTypeOf< - PromptEvent["prompt_data"] - >().branded.toEqualTypeOf(); - expectTypeOf< - ParametersEvent["function_data"]["type"] - >().toEqualTypeOf<"parameters">(); - expectTypeOf< - ParametersEvent["function_type"] - >().toEqualTypeOf<"parameters">(); - expectTypeOf().toMatchTypeOf< - Extract + expectTypeOf().branded.toEqualTypeOf< + Omit >(); }); diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts index d8f5bd89a..16c0c1737 100644 --- a/js/src/public-types.test.ts +++ b/js/src/public-types.test.ts @@ -1,4 +1,5 @@ import { expect, expectTypeOf, test } from "vitest"; +import { z } from "zod/v3"; import * as publicExports from "./exports"; import type { @@ -42,6 +43,50 @@ test("exports only intentional extension types", () => { expectTypeOf>().toBeFunction(); }); +test("keeps parameters while removing deprecated tools", () => { + expect(publicExports.loadParameters).toBeTypeOf("function"); + const project = publicExports.projects.create({ name: "test-project" }); + expect(project).not.toHaveProperty("tools"); + expect(project.parameters.create).toBeTypeOf("function"); + + if (false) { + // @ts-expect-error Saved tools are no longer supported. + project.tools; + + const evaluator = { + data: [{ input: 1 }], + task: (input: number) => input, + scores: [], + parameters: { count: z.number().default(1) }, + }; + publicExports.Eval("test", evaluator, { + // @ts-expect-error Inline overrides must match the parameter schema. + parameters: { count: "one" }, + }); + publicExports.Eval("test", { + ...evaluator, + parameters: publicExports.loadParameters({ + projectName: "test", + slug: "settings", + }), + }); + + project.prompts.create({ + name: "test-prompt", + model: "test-model", + messages: [], + // @ts-expect-error Prompts accept inline provider tools, not saved tools. + tools: [{ type: "function", id: "saved-tool" }], + }); + publicExports.invoke({ + globalFunction: "test", + input: {}, + // @ts-expect-error Deprecated function kinds are no longer supported. + functionType: "tool", + }); + } +}); + test("does not expose runtime schemas or implementation helpers", () => { for (const name of [ "AttachmentReference", diff --git a/js/src/sdk-contracts.test.ts b/js/src/sdk-contracts.test.ts index 00ec7cd07..27e4ca8f3 100644 --- a/js/src/sdk-contracts.test.ts +++ b/js/src/sdk-contracts.test.ts @@ -6,20 +6,24 @@ import type * as SDK from "./sdk-types"; import type * as schemas from "./sdk-schemas"; // Backend definitions are compatibility fixtures, never production dependencies. +// Function contracts intentionally exclude the deprecated tool kind. test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toExtend(); + expectTypeOf().toEqualTypeOf< + Exclude + >(); expectTypeOf< Exclude >().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + Exclude + >(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -30,7 +34,9 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf< + Omit + >(); expectTypeOf< Exclude >().toEqualTypeOf(); @@ -44,7 +50,6 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -53,11 +58,16 @@ test("SDK-owned contracts remain compatible with the backend", () => { expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); - expectTypeOf().branded.toEqualTypeOf(); + expectTypeOf().branded.toEqualTypeOf | null>(); + expectTypeOf().toEqualTypeOf< + Exclude + >(); + expectTypeOf().toExtend(); + expectTypeOf().toExtend(); + expectTypeOf().toExtend(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -67,10 +77,10 @@ test("SDK-owned contracts remain compatible with the backend", () => { }); test("runtime validators accept and return compatible types", () => { - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); expectTypeOf>().toEqualTypeOf< @@ -79,10 +89,10 @@ test("runtime validators accept and return compatible types", () => { expectTypeOf>().toEqualTypeOf< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); expectTypeOf>().toEqualTypeOf< @@ -147,16 +157,16 @@ test("runtime validators accept and return compatible types", () => { expectTypeOf>().toEqualTypeOf< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); expectTypeOf>().toEqualTypeOf< @@ -171,10 +181,22 @@ test("runtime validators accept and return compatible types", () => { expectTypeOf>().toEqualTypeOf< z.output >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.input >(); - expectTypeOf>().toEqualTypeOf< + expectTypeOf>().toExtend< z.output >(); }); + +test("SDK validators match the supported function contracts", () => { + expectTypeOf< + z.output + >().toExtend(); + expectTypeOf< + z.output + >().toExtend(); + expectTypeOf< + z.output + >().toEqualTypeOf(); +}); diff --git a/js/src/sdk-schemas.ts b/js/src/sdk-schemas.ts index a1c8c2ef6..7e5ad2cce 100644 --- a/js/src/sdk-schemas.ts +++ b/js/src/sdk-schemas.ts @@ -53,7 +53,6 @@ const FunctionTypeEnum = z.enum([ "llm", "scorer", "task", - "tool", "custom_view", "preprocessor", "facet", @@ -375,7 +374,6 @@ export const PromptData = z options: PromptOptionsNullish, parser: PromptParserNullish, preprocessor: PreprocessorId, - tool_functions: z.union([z.array(SavedFunctionId), z.null()]), template_format: z.union([ z.enum(["mustache", "nunjucks", "none"]), z.null(), @@ -452,7 +450,6 @@ export const FunctionId = z.union([ const FunctionObjectType = z.enum([ "prompt", - "tool", "scorer", "task", "workflow", diff --git a/js/src/sdk-types.ts b/js/src/sdk-types.ts index bc67f57ed..77c6cfc81 100644 --- a/js/src/sdk-types.ts +++ b/js/src/sdk-types.ts @@ -98,7 +98,6 @@ export type FunctionTypeEnum = | "llm" | "scorer" | "task" - | "tool" | "custom_view" | "preprocessor" | "facet" @@ -106,17 +105,6 @@ export type FunctionTypeEnum = | "tag" | "parameters" | "sandbox"; -export type SavedFunctionId = - | { - type: "function"; - id: string; - version?: string | undefined; - } - | { - type: "global"; - name: string; - function_type: FunctionTypeEnum; - }; export type CallEvent = | { id?: string | undefined; @@ -285,14 +273,6 @@ export type RepoInfo = { commit_time?: string | null; git_diff?: string | null; } | null; -export type ExtendedSavedFunctionId = - | SavedFunctionId - | { - type: "slug"; - project_id: string; - slug: string; - }; -// Keep provider fields visible to editor completion while accepting custom options. export type ModelParams = | { use_cache?: boolean; @@ -410,7 +390,6 @@ export interface PromptData { options?: PromptOptionsNullish; parser?: PromptParserNullish; preprocessor?: PreprocessorId; - tool_functions?: Array | null; template_format?: ("mustache" | "nunjucks" | "none") | null; mcp?: {} | null; origin?: { @@ -460,7 +439,6 @@ export type FunctionId = }; export type FunctionObjectType = | "prompt" - | "tool" | "scorer" | "task" | "workflow"