Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 24 additions & 29 deletions agent-service/src/agent/prompts.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,7 @@ function makeOperatorSchema(operatorType: string, description: string): Operator
return {
operatorType,
operatorVersion: "1.0",
jsonSchema: {
properties: {
condition: { type: "string" },
},
required: ["condition"],
},
jsonSchema: { properties: { condition: { type: "string" } }, required: ["condition"] },
additionalMetadata: {
userFriendlyName: operatorType,
operatorGroupName: "Test",
Expand All @@ -56,16 +51,26 @@ function makeMetadataStore(): WorkflowSystemMetadata {
}

describe("buildSystemPrompt", () => {
test("renders only explicitly allowed operators with their descriptions and compact schemas", () => {
test("names each allowed operator type in one line, and holds no schema", () => {
const prompt = buildSystemPrompt(makeMetadataStore(), ["Filter"]);

expect(prompt).toContain("## Filter");
expect(prompt).toContain("Description: Keeps rows that match a condition.");
expect(prompt).toMatch(/"condition":\s*\{\s*"type":\s*"string"/);
expect(prompt).not.toContain("## PythonUDFV2");
expect(prompt).not.toContain("## RUDF");
expect(prompt).not.toContain("## Python UDF Guide");
expect(prompt).not.toContain("## R UDF Guide");
expect(prompt).toContain("- Filter: Keeps rows that match a condition.");
expect(prompt).toContain("`get_operator_schema`");
expect(prompt).not.toContain("PythonUDFV2:");
expect(prompt).not.toContain('"condition"');
expect(prompt).not.toContain("```json");
});

test("names every known type when no allowlist is supplied", () => {
const prompt = buildSystemPrompt(makeMetadataStore());

expect(prompt).toContain("- Filter: Keeps rows that match a condition.");
expect(prompt).toContain("- PythonUDFV2: Runs user-defined Python code.");
expect(prompt).toContain("- RUDF: Runs user-defined R code.");
});

test("says so when no allowed type is known", () => {
expect(buildSystemPrompt(makeMetadataStore(), ["MissingOperator"])).toContain("No operator types are available.");
});

test("adds guidance only for the UDF language in a restricted allowlist", () => {
Expand All @@ -79,21 +84,11 @@ describe("buildSystemPrompt", () => {
expect(rPrompt).not.toContain("## Python UDF Guide");
});

test("uses all metadata operators and both UDF guides when no allowlist is supplied", () => {
const prompt = buildSystemPrompt(makeMetadataStore());

expect(prompt).toContain("## Filter");
expect(prompt).toContain("## PythonUDFV2");
expect(prompt).toContain("## RUDF");
expect(prompt).toContain("## Python UDF Guide");
expect(prompt).toContain("## R UDF Guide");
});

test("reports that no operators are available when a restricted type is absent from metadata", () => {
const prompt = buildSystemPrompt(makeMetadataStore(), ["MissingOperator"]);
test("adds both UDF guides when no allowlist is supplied, and neither for other types", () => {
const metadataStore = makeMetadataStore();

expect(prompt).toContain("No operators available.");
expect(prompt).not.toContain("## Python UDF Guide");
expect(prompt).not.toContain("## R UDF Guide");
expect(buildSystemPrompt(metadataStore)).toContain("## Python UDF Guide");
expect(buildSystemPrompt(metadataStore)).toContain("## R UDF Guide");
expect(buildSystemPrompt(metadataStore, ["Filter"])).not.toContain("UDF Guide");
});
});
40 changes: 11 additions & 29 deletions agent-service/src/agent/prompts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
* under the License.
*/

import { WorkflowSystemMetadata } from "./util/workflow-system-metadata";
import { usableOperatorTypes } from "./tools/operator-tools";
import type { WorkflowSystemMetadata } from "./util/workflow-system-metadata";

const PYTHON_UDF_OPERATOR_TYPES = ["PythonUDFV2"];
const R_UDF_OPERATOR_TYPES = ["RUDF"];
Expand Down Expand Up @@ -249,40 +250,21 @@ Result:
- **Handle messy data files**: Load data files directly in a single operator. Real-world data files are often malformed — they may have wrong delimiters, missing or misplaced headers, metadata/comment rows, or multiple tables in one file. After loading, inspect the result. If column names look auto-generated (e.g., \`Unnamed: 0\`) or a data value appears as a header, adjust the loading parameters (e.g., \`header=\`, \`skiprows=\`, \`sep=\`) by modifying the data loading operator.
- **Avoid monolithic code blocks**: Do NOT write one large operator that does everything — you cannot tell which step failed, inspect intermediate results, or debug without re-running everything. Instead, decompose into separate operators each doing ONE thing (e.g., filter → join → aggregate → filter → join → final filter). Each can be executed and verified independently.

## Available Operators
## Operators

You have the following operators available:
These are the operator types you can use. Before you add operators of types you have not used yet, read their properties with \`get_operator_schema\`, once for all of them.

{{OPERATOR_SCHEMA}}
{{OPERATORS}}
`;

function buildAllowedOperatorSchemas(
metadataStore: WorkflowSystemMetadata,
allowedOperatorTypes: string[] = []
): string {
const schemas: string[] = [];

const operatorTypes =
allowedOperatorTypes.length > 0 ? allowedOperatorTypes : Object.keys(metadataStore.getAllOperatorTypes());

for (const operatorType of operatorTypes) {
const compactSchema = metadataStore.getCompactSchema(operatorType);
const description = metadataStore.getDescription(operatorType);

if (compactSchema) {
schemas.push(
`## ${operatorType}\n` +
(description ? `Description: ${description}\n` : "") +
`Schema:\n\`\`\`json\n${JSON.stringify(compactSchema, null, 2)}\n\`\`\``
);
}
}

return schemas.length > 0 ? schemas.join("\n\n") : "No operators available.";
/** One line per operator type the agent may use: its name and what it does. */
function listOperatorTypes(metadataStore: WorkflowSystemMetadata, allowedOperatorTypes: string[]): string {
const types = usableOperatorTypes(metadataStore, allowedOperatorTypes);
if (types.length === 0) return "No operator types are available.";
return types.map(type => `- ${type}: ${metadataStore.getDescription(type)}`).join("\n");
}

export function buildSystemPrompt(metadataStore: WorkflowSystemMetadata, allowedOperatorTypes: string[] = []): string {
const operatorSchemas = buildAllowedOperatorSchemas(metadataStore, allowedOperatorTypes);
const allowsAll = allowedOperatorTypes.length === 0;
const pythonAllowed = allowsAll || allowedOperatorTypes.some(t => PYTHON_UDF_OPERATOR_TYPES.includes(t));
const rAllowed = allowsAll || allowedOperatorTypes.some(t => R_UDF_OPERATOR_TYPES.includes(t));
Expand All @@ -291,6 +273,6 @@ export function buildSystemPrompt(metadataStore: WorkflowSystemMetadata, allowed
if (pythonAllowed) extraSections.push(PYTHON_UDF_INSTRUCTIONS);
if (rAllowed) extraSections.push(R_UDF_INSTRUCTIONS);

const base = SYSTEM_PROMPT_TEMPLATE.replace("{{OPERATOR_SCHEMA}}", operatorSchemas);
const base = SYSTEM_PROMPT_TEMPLATE.replace("{{OPERATORS}}", listOperatorTypes(metadataStore, allowedOperatorTypes));
return extraSections.length > 0 ? `${base}\n${extraSections.join("\n\n")}\n` : base;
}
71 changes: 69 additions & 2 deletions agent-service/src/agent/texera-agent.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,12 @@ describe("sendMessage", () => {
});
expect(call.abortSignal).toBeInstanceOf(AbortSignal);
expect(call.abortSignal.aborted).toBe(false);
expect(call.tools.map((t: any) => t.name)).toEqual(["deleteOperator", "addOperator", "modifyOperator"]);
expect(call.tools.map((t: any) => t.name)).toEqual([
"get_operator_schema",
"deleteOperator",
"addOperator",
"modifyOperator",
]);
expect(call.prompt[0]).toEqual({ role: "system", content: "SYS-XYZ" });
const txt = call.prompt[1].content[0].text;
expect(txt).toContain("# Ongoing Task");
Expand Down Expand Up @@ -683,8 +688,9 @@ describe("sendMessage", () => {
agent.updateSettings({ disabledTools: new Set(["modifyOperator"]) });
await agent.sendMessage("go");
const call = (model as any).doGenerateCalls[0];
expect(call.tools.map((t: any) => t.name)).toEqual(["deleteOperator", "addOperator"]);
expect(call.tools.map((t: any) => t.name)).toEqual(["get_operator_schema", "deleteOperator", "addOperator"]);
expect(agent.getSystemInfo().tools.map(t => [t.name, t.enabled])).toEqual([
["get_operator_schema", true],
["deleteOperator", true],
["addOperator", true],
["modifyOperator", false],
Expand Down Expand Up @@ -729,6 +735,67 @@ describe("sendMessage", () => {
expect(agent.getWorkflowState().getOperator("op1")).toBeDefined();
});

test("the prompt, the lookup and the add tool share one list of operator types", async () => {
const saved = (WorkflowSystemMetadata as any).instance;
(WorkflowSystemMetadata as any).instance = undefined;
try {
const op = (operatorType: string, description: string) => ({
operatorType,
jsonSchema: { type: "object", properties: {} },
additionalMetadata: {
userFriendlyName: operatorType,
operatorDescription: description,
inputPorts: [],
outputPorts: [{}],
},
});
WorkflowSystemMetadata.getInstance().loadFromMetadata({
operators: [op("CSVFileScan", "Read a CSV file"), op("Aggregate", "Aggregate by group")],
} as any);
let n = 0;
const calls = [
{ toolName: "get_operator_schema", input: { operatorTypes: ["Aggregate"] } },
{
toolName: "addOperator",
input: { operatorId: "op1", operatorType: "Aggregate", properties: {}, summary: "s" },
},
];
const model = new MockLanguageModelV4({
doGenerate: async () => {
const next = calls[n++];
if (next)
return {
content: [
{ type: "tool-call", toolCallId: "c" + n, toolName: next.toolName, input: JSON.stringify(next.input) },
],
finishReason: finish("tool-calls"),
usage: usage(1, 1),
warnings: [],
} as any;
return {
content: [{ type: "text", text: "x" }],
finishReason: finish("stop"),
usage: usage(1, 1),
warnings: [],
} as any;
},
});
const agent = makeAgentWith(model);
agent.updateSettings({ allowedOperatorTypes: ["CSVFileScan"] });
await agent.sendMessage("add an aggregate");
const [lookedUp, added] = agent.getAllSteps().slice(1, 3);
expect(agent.getSystemInfo().systemPrompt).toContain("- CSVFileScan: Read a CSV file");
expect(agent.getSystemInfo().systemPrompt).not.toContain("- Aggregate:");
expect(lookedUp.toolResults?.[0]).toMatchObject({ isError: true });
expect(String(added.toolResults?.[0]?.output)).toStartWith(
'[ERROR] Unknown operator type: "Aggregate". Available types: CSVFileScan.'
);
expect(agent.getWorkflowState().getAllOperators()).toEqual([]);
} finally {
(WorkflowSystemMetadata as any).instance = saved;
}
});

test("caps the loop at maxSteps", async () => {
// The cap is the only thing standing between a looping model and an unbounded run.
let n = 0;
Expand Down
10 changes: 9 additions & 1 deletion agent-service/src/agent/texera-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ import {
TOOL_NAME_EXECUTE_OPERATOR,
type ExecutionConfig,
} from "./tools/workflow-execution-tools";
import {
createGetOperatorSchemaTool,
TOOL_NAME_GET_OPERATOR_SCHEMA,
usableOperatorTypes,
} from "./tools/operator-tools";
import { assembleContext } from "./util/context-utils";
import { compileWorkflowAsync, type WorkflowCompilationResponse } from "../api/compile-api";
import { createLogger } from "../logger";
Expand Down Expand Up @@ -199,8 +204,10 @@ export class TexeraAgent {
}

private createTools(): Record<string, any> {
// One list of operator types: what the prompt names and the lookup describes is what can be added.
const usableTypes = usableOperatorTypes(this.metadataStore, this.settings.allowedOperatorTypes);
const operatorSchemas = new Map<string, any>();
for (const type of Object.keys(this.metadataStore.getAllOperatorTypes())) {
for (const type of usableTypes) {
const jsonSchema = this.metadataStore.getSchema(type);
const additionalMetadata = this.metadataStore.getAdditionalMetadata(type);
if (jsonSchema) {
Expand All @@ -220,6 +227,7 @@ export class TexeraAgent {
};

const tools: Record<string, any> = {
[TOOL_NAME_GET_OPERATOR_SCHEMA]: createGetOperatorSchemaTool(this.metadataStore, usableTypes),
[TOOL_NAME_DELETE_OPERATOR]: createDeleteOperatorTool(this.workflowState, context),
[TOOL_NAME_ADD_OPERATOR]: createAddOperatorTool(this.workflowState, operatorSchemas, context),
[TOOL_NAME_MODIFY_OPERATOR]: createModifyOperatorTool(this.workflowState, context),
Expand Down
1 change: 1 addition & 0 deletions agent-service/src/agent/tools/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@
export * from "./tools-utility";
export * from "./workflow-crud-tools";
export * from "./workflow-execution-tools";
export * from "./operator-tools";
100 changes: 100 additions & 0 deletions agent-service/src/agent/tools/operator-tools.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import { describe, expect, test } from "bun:test";
import type { OperatorSchema } from "../../api/backend-api";
import { WorkflowSystemMetadata } from "../util/workflow-system-metadata";
import { createGetOperatorSchemaTool, usableOperatorTypes } from "./operator-tools";

function schema(operatorType: string, friendly: string, group: string, description: string): OperatorSchema {
return {
operatorType,
operatorVersion: "1.0",
jsonSchema: {
properties: {
groupByKeys: { type: "array", items: { type: "string" } },
dummyPropertyList: { type: "array" },
},
required: ["groupByKeys"],
},
additionalMetadata: {
userFriendlyName: friendly,
operatorGroupName: group,
operatorDescription: description,
inputPorts: [],
outputPorts: [],
},
};
}

function metadata(): WorkflowSystemMetadata {
const store = new WorkflowSystemMetadata();
store.loadFromMetadata({
operators: [
schema("Aggregate", "Aggregate", "Data Cleaning", "Calculate aggregates by group"),
schema("HashJoin", "Hash Join", "Join", "Join two inputs on a key"),
schema("Sort", "Sort", "Data Cleaning", "Sort rows by columns"),
],
groups: [],
});
return store;
}

const run = async (agentTool: any, args: Record<string, unknown>) =>
agentTool.execute(args, { toolCallId: "test-call", messages: [] });

describe("usableOperatorTypes", () => {
test("is every known type when none are allowed by name", () => {
expect(usableOperatorTypes(metadata(), []).sort()).toEqual(["Aggregate", "HashJoin", "Sort"]);
});

test("keeps the allowed types the metadata knows, and drops the ones it does not", () => {
expect(usableOperatorTypes(metadata(), ["Sort", "Missing", "Aggregate"])).toEqual(["Sort", "Aggregate"]);
});
});

describe("get_operator_schema", () => {
const store = metadata();
const groupBy = { groupByKeys: { type: "array", items: { type: "string" } } };

test("gives each type's description and compact schema, without placeholder properties", async () => {
const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, []));
expect(JSON.parse(await run(get, { operatorTypes: ["Aggregate", "Sort"] }))).toEqual({
Aggregate: { description: "Calculate aggregates by group", properties: groupBy, required: ["groupByKeys"] },
Sort: { description: "Sort rows by columns", properties: groupBy, required: ["groupByKeys"] },
});
});

test.each([
["an unknown type", ["Pivot"], [], "Pivot"],
["a type that is not allowed", ["Sort", "HashJoin"], ["Sort"], "HashJoin"],
["a type in another case", ["aggregate"], [], "aggregate"],
])("refuses %s, naming it", async (_, operatorTypes, allowed, refused) => {
const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, allowed as string[]));
await expect(run(get, { operatorTypes })).rejects.toThrow(
`Not operator types you can use: ${refused}. Use the types the system prompt lists.`
);
});

test("takes at least one type", async () => {
const get = createGetOperatorSchemaTool(store, usableOperatorTypes(store, []));
expect(get.inputSchema).toBeDefined();
expect((get.inputSchema as any).safeParse({ operatorTypes: [] }).success).toBe(false);
});
});
Loading
Loading