diff --git a/agent-service/.env.example b/agent-service/.env.example index 605f157ca0c..820fa13793b 100644 --- a/agent-service/.env.example +++ b/agent-service/.env.example @@ -18,4 +18,8 @@ LLM_ENDPOINT=http://localhost:9096 # Texera backend services TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://localhost:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://localhost:9090 -WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://localhost:8085 \ No newline at end of file +WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://localhost:8085 + +# Docs read by the agent's searchDocs/readDoc tools. The image copies them to ./docs; +# a local run from agent-service/ needs the repo's docs folder. +TEXERA_DOCS_DIR=../docs diff --git a/agent-service/scripts/docs-search-eval.json b/agent-service/scripts/docs-search-eval.json new file mode 100644 index 00000000000..c7b3a4fce97 --- /dev/null +++ b/agent-service/scripts/docs-search-eval.json @@ -0,0 +1,115 @@ +[ + { + "group": "platform", + "question": "where do I find a workflow that someone shared with me", + "expected": "getting-started/ui-overview" + }, + { + "group": "platform", + "question": "how do I upload a file to a dataset and use it in a workflow", + "expected": "tutorials/create-dataset-upload-data" + }, + { + "group": "platform", + "question": "how do I log in to Texera for the first time", + "expected": "getting-started/AccessLogin" + }, + { "group": "platform", "question": "what is a computing unit and who can use one", "expected": "security/_index" }, + { + "group": "platform", + "question": "how do I run a workflow and see the results in the browser", + "expected": "tutorials/guide-for-how-to-use-texera" + }, + { + "group": "operator", + "question": "how do I remove duplicate rows", + "expected": "reference/operators/data-cleaning/distinct" + }, + { + "group": "operator", + "question": "CSV file scan delimiter and header options", + "expected": "reference/operators/data-input/csv-file-scan" + }, + { + "group": "operator", + "question": "join two tables on a key column", + "expected": "reference/operators/data-cleaning/join/hash-join" + }, + { + "group": "operator", + "question": "group by a column and compute sum or average", + "expected": "reference/operators/data-cleaning/aggregate/aggregate" + }, + { + "group": "operator", + "question": "keep only the first 100 rows", + "expected": "reference/operators/data-cleaning/limit" + }, + { + "group": "operator", + "question": "search a string column with a regular expression", + "expected": "reference/operators/search/regular-expression" + }, + { + "group": "operator", + "question": "how to write a Python UDF operator", + "expected": "tutorials/guide-to-use-python-udf" + }, + { + "group": "operator", + "question": "convert a column from string to integer", + "expected": "reference/operators/data-cleaning/type-casting" + }, + { + "group": "operator", + "question": "draw a word cloud from text", + "expected": "reference/operators/visualization/media/word-cloud" + }, + { + "group": "operator", + "question": "sort rows by a column", + "expected": "reference/operators/data-cleaning/sort/sort" + }, + { + "group": "deploy", + "question": "install Texera with docker compose", + "expected": "getting-started/installing-using-docker" + }, + { + "group": "deploy", + "question": "run Texera on kubernetes with helm", + "expected": "getting-started/run-on-kubernetes" + }, + { + "group": "deploy", + "question": "how do I enable the AI agent and configure LiteLLM", + "expected": "tutorials/guide-to-enable-llm-agent" + }, + { + "group": "deploy", + "question": "stop and restart the Texera containers", + "expected": "getting-started/installing-using-docker" + }, + { "group": "platform", "question": "what are the user roles and access privileges", "expected": "security/_index" }, + { + "group": "platform", + "question": "convert my Jupyter notebook into a workflow", + "expected": "tutorials/migrate-jupyter-notebook" + }, + { + "group": "operator", + "question": "read data from a PostgreSQL database", + "expected": "reference/operators/database-connector/postgresql-source" + }, + { + "group": "operator", + "question": "list the files in a dataset version", + "expected": "reference/operators/data-input/file-lister" + }, + { + "group": "operator", + "question": "what does set snapshot output mode mean", + "expected": "reference/operators/output-modes" + }, + { "group": "platform", "question": "overview of Texera architecture and core concepts", "expected": "overview" } +] diff --git a/agent-service/scripts/docs-search-eval.ts b/agent-service/scripts/docs-search-eval.ts new file mode 100644 index 00000000000..0f801c54b16 --- /dev/null +++ b/agent-service/scripts/docs-search-eval.ts @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Measures recall@3 and recall@5 of the docs keyword index over a fixed question set. +// Usage: bun run scripts/docs-search-eval.ts [docsDir] (default: ../docs) + +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { loadDocsIndex } from "../src/agent/tools/docs-index"; + +interface EvalQuestion { + group: string; + question: string; + expected: string; +} + +const docsDir = process.argv[2] ?? join(import.meta.dir, "..", "..", "docs"); +const index = loadDocsIndex(docsDir); +if (!index) { + console.error(`docs directory not found: ${docsDir}`); + process.exit(1); +} + +const questions: EvalQuestion[] = JSON.parse(readFileSync(join(import.meta.dir, "docs-search-eval.json"), "utf8")); + +let hitAt3 = 0; +let hitAt5 = 0; +const misses: string[] = []; +const noHit: string[] = []; + +for (const q of questions) { + const hits = index.search(q.question, 5); + const paths = [...new Set(hits.map(h => h.path))]; + const rank = paths.indexOf(q.expected); + if (rank >= 0 && rank < 3) hitAt3++; + if (rank >= 0) hitAt5++; + if (hits.length === 0) noHit.push(q.question); + const mark = rank < 0 ? "MISS" : `#${rank + 1}`; + console.log(`${mark.padEnd(5)} [${q.group}] ${q.question}`); + if (rank < 0) { + misses.push(`${q.question} -> expected ${q.expected}, got ${paths.slice(0, 3).join(", ") || "(nothing)"}`); + } +} + +console.log(`\npages indexed: ${index.pages().length}`); +console.log(`recall@3: ${hitAt3}/${questions.length} = ${(hitAt3 / questions.length).toFixed(2)}`); +console.log(`recall@5: ${hitAt5}/${questions.length} = ${(hitAt5 / questions.length).toFixed(2)}`); +if (misses.length > 0) { + console.log("\nmissed (expected page not in top 5):"); + for (const m of misses) console.log(`- ${m}`); +} +console.log(`\nno hit at all (candidate docs gaps): ${noHit.length === 0 ? "none" : ""}`); +for (const q of noHit) console.log(`- ${q}`); diff --git a/agent-service/src/agent/prompts.spec.ts b/agent-service/src/agent/prompts.spec.ts index d4ae6e21516..d40806a8c59 100644 --- a/agent-service/src/agent/prompts.spec.ts +++ b/agent-service/src/agent/prompts.spec.ts @@ -89,6 +89,47 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain("## R UDF Guide"); }); + test("describes the Documentation Read section between the ongoing task and the dataflow", () => { + const prompt = buildSystemPrompt(makeMetadataStore()); + + const bullets = [ + "- `# Ongoing Task` — the current task, including turns you've taken so far", + "- `# Documentation Read` — the output of your searchDocs and readDoc calls in the current task (omitted if none)", + "- `# Current Dataflow` — the live DAG: every operator's current state", + ]; + expect(prompt).toContain(bullets.join("\n")); + expect(prompt).not.toContain("three top-level sections"); + }); + + test("shows the Documentation Read section in the layout the way the context renders it", () => { + const prompt = buildSystemPrompt(makeMetadataStore()); + + expect(prompt).toContain( + [ + "# Documentation Read", + "Output of your searchDocs and readDoc calls in this task, oldest first.", + "", + '## Turn 1: searchDocs {"query":""}', + " ", + "", + "# Current Dataflow", + ].join("\n") + ); + }); + + test("keeps the rule about reading documentation and adds the docs-tools rule right after it", () => { + const prompt = buildSystemPrompt(makeMetadataStore()); + + const kept = + "- **Read documentation first**: When the task mentions abstract concepts, load documentation to understand exact definitions."; + const added = "- **Answer questions about Texera from its documentation**:"; + expect(prompt).toContain(`${kept}\n${added}`); + expect(prompt).toContain("call searchDocs first, then readDoc on the best match, and answer with the page link."); + expect(prompt).toContain("Only cite links that these tools returned"); + expect(prompt).toContain("say so when the documentation does not cover the question"); + expect(prompt).toContain("Do not edit the workflow for a question."); + }); + test("reports that no operators are available when a restricted type is absent from metadata", () => { const prompt = buildSystemPrompt(makeMetadataStore(), ["MissingOperator"]); diff --git a/agent-service/src/agent/prompts.ts b/agent-service/src/agent/prompts.ts index 064eed2e3e5..dd0a36d70df 100644 --- a/agent-service/src/agent/prompts.ts +++ b/agent-service/src/agent/prompts.ts @@ -163,10 +163,11 @@ Dataflow represents data analysis as a DAG (directed acyclic graph) where: ## Context Format -Your conversation context is a single message with three top-level sections, in this order: +Your conversation context is a single message with these top-level sections, in this order: - \`# Completed Tasks\` — previous tasks you've already finished (omitted if none) - \`# Ongoing Task\` — the current task, including turns you've taken so far +- \`# Documentation Read\` — the output of your searchDocs and readDoc calls in the current task (omitted if none) - \`# Current Dataflow\` — the live DAG: every operator's current state **Overall layout:** @@ -215,6 +216,12 @@ Thought: ... - Error: +# Documentation Read +Output of your searchDocs and readDoc calls in this task, oldest first. + +## Turn 1: searchDocs {"query":""} + + # Current Dataflow ## Operators @@ -241,6 +248,7 @@ Result: - **One operation per operator**: Each operator does one task (join, filter, aggregate, etc.). Use links to connect them. - **Build incrementally**: Link new operators to existing ones. Never recreate data already in the workflow. - **Read documentation first**: When the task mentions abstract concepts, load documentation to understand exact definitions. +- **Answer questions about Texera from its documentation**: If the searchDocs tool is available and the user asks how Texera or one of its features works, call searchDocs first, then readDoc on the best match, and answer with the page link. Only cite links that these tools returned, and say so when the documentation does not cover the question. Do not edit the workflow for a question. If the user asks you to build or change something, you may still consult the docs before acting. - **Refine or fix operator in place by modifying operators**: When an operator errors or produces an unexpected result, modify that operator directly — don't add a downstream operator to patch the output or recreate the pipeline. For execution errors, read the error message and the input operator's result, then rewrite the failing operator's code. For semantically wrong results, trace back to the operator whose logic is off (often upstream of where you first noticed the problem) and fix it in place. - **Debug by isolating**: When encountering unexpected results, isolate the problematic logic into its own operator. - **Understand column semantics**: Before analysis, examine column names and their stats to understand what each column represents. Columns may carry semantic meaning that affects how data should be filtered or interpreted — respect these signals and apply appropriate preprocessing before computing results. diff --git a/agent-service/src/agent/texera-agent.spec.ts b/agent-service/src/agent/texera-agent.spec.ts index 4a529704e03..a0c82131870 100644 --- a/agent-service/src/agent/texera-agent.spec.ts +++ b/agent-service/src/agent/texera-agent.spec.ts @@ -23,6 +23,11 @@ import { AgentState, INITIAL_STEP_ID, type ReActStep } from "../types/agent"; import { MockLanguageModelV4 } from "ai/test"; import { WorkflowSystemMetadata } from "./util/workflow-system-metadata"; import { OperatorState, type OperatorExecutionSummary, type Tuple } from "../types/execution"; +import { DocsIndex } from "./tools/docs-index"; +import { env } from "../config/env"; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; /** * These tests cover the agent's bookkeeping — the ReAct step tree, settings, and client set — which @@ -35,6 +40,7 @@ function makeAgent(): TexeraAgent { modelType: "test-model", agentId: "agent-1", userToken: "user-token", + docsIndex: null, }); } @@ -403,6 +409,7 @@ function makeAgentWith(model: any): TexeraAgent { agentId: "agent-1", systemPrompt: "SYS-XYZ", userToken: "user-token", + docsIndex: null, }); liveAgents.push(agent); return agent; @@ -1229,3 +1236,231 @@ describe("delegate mode", () => { expect(errs[0][1]).toBe("failed to auto-persist workflow"); }); }); + +describe("docs tools", () => { + const docsIndex = new DocsIndex([ + { + path: "getting-started/datasets", + title: "Datasets", + description: "Upload files", + body: "## Uploading\nClick Upload.", + }, + ]); + + test("registers searchDocs and readDoc when a docs index is given", () => { + const agent = new TexeraAgent({ model: {} as any, modelType: "m", agentId: "a", docsIndex }); + const names = agent.getSystemInfo().tools.map(t => t.name); + expect(names).toEqual(["deleteOperator", "addOperator", "modifyOperator", "searchDocs", "readDoc"]); + }); + + test("registers neither tool when the docs index is null", () => { + const agent = new TexeraAgent({ model: {} as any, modelType: "m", agentId: "a", docsIndex: null }); + const names = agent.getSystemInfo().tools.map(t => t.name); + expect(names).toEqual(["deleteOperator", "addOperator", "modifyOperator"]); + }); + + test("reports a docs tool as disabled through the same setting as every other tool", () => { + const agent = new TexeraAgent({ model: {} as any, modelType: "m", agentId: "a", docsIndex }); + agent.updateSettings({ disabledTools: new Set(["readDoc"]) }); + const info = agent.getSystemInfo().tools; + expect(info.find(t => t.name === "readDoc")?.enabled).toBe(false); + expect(info.find(t => t.name === "searchDocs")?.enabled).toBe(true); + }); + + test("passes the docs tools to the model alongside the workflow tools", async () => { + const model = textModel("ok"); + const agent = new TexeraAgent({ model, modelType: "m", agentId: "a", systemPrompt: "S", docsIndex }); + liveAgents.push(agent); + await agent.sendMessage("hi"); + const call = (model as any).doGenerateCalls[0]; + expect(call.tools.map((t: any) => t.name)).toContain("searchDocs"); + expect(call.tools.map((t: any) => t.name)).toContain("readDoc"); + }); + + /** A model that makes the given tool calls one per step, then answers with the given text. */ + function scripted(turns: Array<{ tool: string; input: unknown } | string>) { + let n = 0; + return new MockLanguageModelV4({ + doGenerate: async () => { + const turn = turns[Math.min(n, turns.length - 1)]; + n++; + if (typeof turn === "string") { + return { + content: [{ type: "text", text: turn }], + finishReason: finish("stop"), + usage: usage(1, 1), + warnings: [], + } as any; + } + return { + content: [{ type: "tool-call", toolCallId: `c${n}`, toolName: turn.tool, input: JSON.stringify(turn.input) }], + finishReason: finish("tool-calls"), + usage: usage(1, 1), + warnings: [], + } as any; + }, + }); + } + + const contextOf = (model: any, call: number): string => model.doGenerateCalls[call].prompt[1].content[0].text; + + function docsAgent(model: any, index: DocsIndex = docsIndex): TexeraAgent { + const agent = new TexeraAgent({ model, modelType: "m", agentId: "a", systemPrompt: "S", docsIndex: index }); + liveAgents.push(agent); + return agent; + } + + test("shows the output of a docs call to the model on the next step", async () => { + const model = scripted([{ tool: "searchDocs", input: { query: "upload" } }, "Use the Upload button."]); + const res = await docsAgent(model).sendMessage("how do I upload a file"); + + expect(res.response).toBe("Use the Upload button."); + expect((model as any).doGenerateCalls).toHaveLength(2); + expect(contextOf(model, 0)).not.toContain("# Documentation Read"); + const second = contextOf(model, 1); + expect(second).toContain("- searchDocs (succeeded)"); + expect(second).toContain( + "# Documentation Read\nOutput of your searchDocs and readDoc calls in this task, oldest first." + ); + expect(second).toContain( + '## Turn 1: searchDocs {"query":"upload"}\n 1. Datasets > Uploading\n path: getting-started/datasets' + ); + expect(second).toContain("Click Upload."); + }); + + test("shows a readDoc output and an unknown-page error the same way", async () => { + const model = scripted([ + { tool: "readDoc", input: { path: "getting-started/nope" } }, + { tool: "readDoc", input: { path: "getting-started/datasets", section: "Uploading" } }, + "done", + ]); + await docsAgent(model).sendMessage("read it"); + + const third = contextOf(model, 2); + expect(third).toContain( + '## Turn 1: readDoc {"path":"getting-started/nope"}\n [ERROR] Unknown page "getting-started/nope".' + ); + expect(third).toContain('## Turn 2: readDoc {"path":"getting-started/datasets","section":"Uploading"}'); + expect(third).toContain(" Click Upload."); + }); + + test("tells the model when a docs call was rejected for its arguments", async () => { + // A call that fails the tool's input schema never runs, so the stored step has no result for it. + const model = scripted([{ tool: "searchDocs", input: { limit: 3 } }, "I will search again."]); + const agent = docsAgent(model); + await agent.sendMessage("how do I upload a file"); + + const step = agent.getAllSteps()[1]; + expect(step.toolCalls?.map(c => c.toolName)).toEqual(["searchDocs"]); + expect(step.toolResults ?? []).toEqual([]); + expect(contextOf(model, 1)).toContain( + '## Turn 1: searchDocs {"limit":3}\n (no output, the tool is unavailable or its arguments were invalid)' + ); + }); + + test("tells the model the same when it calls a docs tool that is not registered", async () => { + const model = scripted([{ tool: "searchDocs", input: { query: "upload" } }, "I cannot search the docs here."]); + const agent = new TexeraAgent({ model, modelType: "m", agentId: "a", systemPrompt: "S", docsIndex: null }); + liveAgents.push(agent); + await agent.sendMessage("how do I upload a file"); + + expect(contextOf(model, 1)).toContain( + '## Turn 1: searchDocs {"query":"upload"}\n (no output, the tool is unavailable or its arguments were invalid)' + ); + }); + + test("answers a limit or an offset out of range instead of rejecting the call", async () => { + const model = scripted([ + { tool: "searchDocs", input: { query: "upload", limit: 50 } }, + { tool: "readDoc", input: { path: "getting-started/datasets", offset: -1 } }, + "done", + ]); + await docsAgent(model).sendMessage("how do I upload a file"); + + const third = contextOf(model, 2); + expect(third).toContain('## Turn 1: searchDocs {"query":"upload","limit":50}\n 1. Datasets'); + expect(third).toContain( + '## Turn 2: readDoc {"path":"getting-started/datasets","offset":-1}\n [ERROR] offset must be a whole number of characters, 0 or more.' + ); + }); + + describe("without a configured index", () => { + const original = env.TEXERA_DOCS_DIR; + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "texera-agent-docs-")); + }); + + afterEach(() => { + env.TEXERA_DOCS_DIR = original; + rmSync(dir, { recursive: true, force: true }); + }); + + const toolNames = (): string[] => + new TexeraAgent({ model: {} as any, modelType: "m", agentId: "a" }).getSystemInfo().tools.map(t => t.name); + + test("loads the index from TEXERA_DOCS_DIR and registers the docs tools", () => { + writeFileSync(join(dir, "page.md"), "# Page\nSome text."); + env.TEXERA_DOCS_DIR = dir; + expect(toolNames()).toEqual(["deleteOperator", "addOperator", "modifyOperator", "searchDocs", "readDoc"]); + }); + + test("registers no docs tool when TEXERA_DOCS_DIR does not exist", () => { + env.TEXERA_DOCS_DIR = join(dir, "missing"); + expect(toolNames()).toEqual(["deleteOperator", "addOperator", "modifyOperator"]); + }); + + test("registers no docs tool, and still creates the agent, when the folder cannot be read", () => { + writeFileSync(join(dir, "page.md"), "# Page\nSome text."); + symlinkSync(join(dir, "nowhere"), join(dir, "broken.md")); + env.TEXERA_DOCS_DIR = dir; + expect(toolNames()).toEqual(["deleteOperator", "addOperator", "modifyOperator"]); + }); + }); + + test("drops the docs output of a finished task from the next user request", async () => { + const model = scripted([ + { tool: "searchDocs", input: { query: "upload" } }, + "Use the Upload button.", + "You are welcome.", + ]); + const agent = docsAgent(model); + await agent.sendMessage("how do I upload a file"); + await agent.sendMessage("thanks"); + + const next = contextOf(model, 2); + expect(next).toContain("# Completed Tasks"); + expect(next).toContain("- searchDocs (succeeded)"); + expect(next).toContain("Thought: Use the Upload button."); + expect(next).not.toContain("# Documentation Read"); + expect(next).not.toContain("path: getting-started/datasets"); + }); + + test("leaves out the oldest docs output and says so once the outputs pass the budget", async () => { + const letters = ["a", "b", "c", "d", "e", "f"]; + const big = new DocsIndex([ + { + path: "big/page", + title: "Big", + description: "", + body: letters.map(l => `## Part ${l}\n${l.repeat(3800)}`).join("\n\n"), + }, + ]); + const model = scripted([ + ...letters.map(l => ({ tool: "readDoc", input: { path: "big/page", section: `Part ${l}` } })), + "done", + ]); + await docsAgent(model, big).sendMessage("read everything"); + + const sixth = contextOf(model, 5); + expect(sixth).not.toContain("dropped to save space"); + expect(sixth).toContain("a".repeat(3800)); + + const last = contextOf(model, 6); + expect(last).toContain("(1 earlier output dropped to save space, call the tool again if you need it)"); + expect(last).not.toContain('## Turn 1: readDoc {"path":"big/page","section":"Part a"}'); + expect(last).not.toContain("aaaa"); + for (const l of letters.slice(1)) expect(last).toContain(l.repeat(3800)); + }); +}); diff --git a/agent-service/src/agent/texera-agent.ts b/agent-service/src/agent/texera-agent.ts index ee6f7601c4a..b7fef8641a5 100644 --- a/agent-service/src/agent/texera-agent.ts +++ b/agent-service/src/agent/texera-agent.ts @@ -47,6 +47,15 @@ import { TOOL_NAME_EXECUTE_OPERATOR, type ExecutionConfig, } from "./tools/workflow-execution-tools"; +import { + createReadDocTool, + createSearchDocsTool, + getSharedDocsIndex, + TOOL_NAME_READ_DOC, + TOOL_NAME_SEARCH_DOCS, +} from "./tools/docs-tools"; +import type { DocsIndex } from "./tools/docs-index"; +import { env } from "../config/env"; import { assembleContext } from "./util/context-utils"; import { compileWorkflowAsync, type WorkflowCompilationResponse } from "../api/compile-api"; import { createLogger } from "../logger"; @@ -62,6 +71,8 @@ export interface TexeraAgentConfig { systemPrompt?: string; /** JWT of the user the agent acts for, sent on backend calls that are not tied to a workflow. */ userToken?: string; + /** Docs index for the searchDocs and readDoc tools. Omitted loads the shared one from TEXERA_DOCS_DIR; null registers neither tool. */ + docsIndex?: DocsIndex | null; } export interface AgentMessageResult { @@ -122,6 +133,7 @@ export class TexeraAgent { private messageCounter = 0; private tools: Record; + private readonly docsIndex: DocsIndex | null; private abortController: AbortController | null = null; @@ -161,6 +173,7 @@ export class TexeraAgent { systemPrompt: this.systemPrompt, }; + this.docsIndex = config.docsIndex !== undefined ? config.docsIndex : getSharedDocsIndex(env.TEXERA_DOCS_DIR); this.tools = this.createTools(); } @@ -234,6 +247,11 @@ export class TexeraAgent { ); } + if (this.docsIndex) { + tools[TOOL_NAME_SEARCH_DOCS] = createSearchDocsTool(this.docsIndex); + tools[TOOL_NAME_READ_DOC] = createReadDocTool(this.docsIndex); + } + return tools; } diff --git a/agent-service/src/agent/tools/docs-index.spec.ts b/agent-service/src/agent/tools/docs-index.spec.ts new file mode 100644 index 00000000000..42f45e83c62 --- /dev/null +++ b/agent-service/src/agent/tools/docs-index.spec.ts @@ -0,0 +1,1173 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DOCS_SITE_BASE, DocsIndex, cutText, loadDocsIndex, resolveDocLinks, tokenize } from "./docs-index"; + +const ASF_HEADER = ` +`; + +const ROOT_INDEX = `${ASF_HEADER} +--- +title: Texera Documentation +description: Root of the docs +--- + +Welcome to Texera. +`; + +const DATASETS_PAGE = `${ASF_HEADER} +--- +title: "Datasets" +description: "Upload and manage datasets" +weight: 20 +--- + +{{% pageinfo %}} +A dataset is a versioned collection of files. +{{% /pageinfo %}} + +## Uploading files + +Click the Upload button in the dataset page to add files. Files are grouped into a version. + +{{< alert color="success" title="Tip" >}} +Drag and drop also works for uploading. +{{< /alert >}} + +### Supported formats + +CSV, JSON and Parquet files are supported. + +## Sharing a dataset + +Use the share dialog to grant read access to another user. +`; + +const CSV_SCAN_PAGE = `${ASF_HEADER} +--- +title: "CSVFileScan" +description: "Reads a CSV file into tuples" +--- + +## Overview + +CSVFileScan reads a comma separated file from a dataset path. + +## Properties + +The delimiter property sets the column separator. +`; + +const LONG_SECTION = "word ".repeat(2000).trim(); + +const LONG_PAGE = `${ASF_HEADER} +--- +title: "Long Reference" +description: "A page with a long section" +--- + +## Big section + +${LONG_SECTION} +`; + +const FOLDED_PAGE = `--- +title: "Texera UI Overview" +description: > + Explore the User Dashboard and its components. +categories: [Texera] +--- + +## User Dashboard + +The dashboard lists your workflows. +`; + +const FENCED_PAGE = `${ASF_HEADER} +--- +title: "Python UDF" +description: "User-defined function operator in Python" +--- + +## Template + +\`\`\`python +# comment line inside the fence +class ProcessTupleOperator(UDFOperatorV2): + ## not a heading + def process_tuple(self, tuple_, port): + yield tuple_ +\`\`\` + +## Notes + +Edit the template in the editor. +`; + +const NO_FRONT_MATTER_FENCED_PAGE = `\`\`\` +# fake heading inside a fence +\`\`\` + +# Real title + +Body text. +`; + +const NO_FRONT_MATTER_PAGE = `${ASF_HEADER} +# Plain page + +Content without front matter. +`; + +const DEV_PAGE = `--- +title: "Guide for developers" +description: "How to build Texera from source" +--- + +## Building + +Run sbt compile to build the backend. +`; + +let docsDir: string; +let index: DocsIndex; + +beforeAll(() => { + docsDir = mkdtempSync(join(tmpdir(), "texera-docs-index-")); + mkdirSync(join(docsDir, "getting-started"), { recursive: true }); + mkdirSync(join(docsDir, "reference", "operators", "data-input"), { recursive: true }); + mkdirSync(join(docsDir, "contribution-guidelines"), { recursive: true }); + writeFileSync(join(docsDir, "_index.md"), ROOT_INDEX); + writeFileSync(join(docsDir, "getting-started", "datasets.md"), DATASETS_PAGE); + writeFileSync(join(docsDir, "reference", "operators", "data-input", "csv-file-scan.md"), CSV_SCAN_PAGE); + writeFileSync(join(docsDir, "reference", "long.md"), LONG_PAGE); + writeFileSync(join(docsDir, "reference", "plain.md"), NO_FRONT_MATTER_PAGE); + writeFileSync(join(docsDir, "getting-started", "ui-overview.md"), FOLDED_PAGE); + writeFileSync(join(docsDir, "reference", "udf.md"), FENCED_PAGE); + writeFileSync(join(docsDir, "reference", "fenced-title.md"), NO_FRONT_MATTER_FENCED_PAGE); + writeFileSync(join(docsDir, "contribution-guidelines", "guide-for-developers.md"), DEV_PAGE); + writeFileSync(join(docsDir, "reference", "notes.txt"), "not markdown"); + index = loadDocsIndex(docsDir)!; +}); + +afterAll(() => { + rmSync(docsDir, { recursive: true, force: true }); +}); + +describe("tokenize", () => { + test("splits camelCase, lowercases, drops one-char tokens, and keeps the unsplit word", () => { + expect(tokenize("CSVFileScan")).toEqual(["csv", "file", "scan", "csvfilescan"]); + }); + + test("splits on punctuation and whitespace and drops stopwords", () => { + expect(tokenize("Upload a file, then share-it!")).toEqual(["upload", "file", "then", "share"]); + }); + + test("returns nothing for a stopword-only query", () => { + expect(tokenize("how do I")).toEqual([]); + expect(tokenize("What is the")).toEqual([]); + }); + + test("keeps the whole word even when camelCase splitting leaves one part", () => { + expect(tokenize("iOS")).toEqual(["os", "ios"]); + expect(tokenize("upload")).toEqual(["upload"]); + }); + + test("returns an empty list for empty input", () => { + expect(tokenize("")).toEqual([]); + }); + + test("stems plural and verb endings so related words share a token", () => { + expect(tokenize("queries")).toEqual(["query"]); + expect(tokenize("filtered")).toEqual(["filter"]); + expect(tokenize("joining")).toEqual(["join"]); + expect(tokenize("rows")).toEqual(["row"]); + }); + + test("leaves a word ending in ss and a word too short for its suffix alone", () => { + expect(tokenize("access")).toEqual(["access"]); + expect(tokenize("class")).toEqual(["class"]); + expect(tokenize("ties")).toEqual(["tie"]); + expect(tokenize("used")).toEqual(["used"]); + expect(tokenize("bring")).toEqual(["bring"]); + expect(tokenize("gas")).toEqual(["gas"]); + }); +}); + +describe("cutText", () => { + test("cuts at the limit and leaves shorter text alone", () => { + expect(cutText("abcdef", 4)).toBe("abcd"); + expect(cutText("abc", 4)).toBe("abc"); + expect(cutText("abcd", 4)).toBe("abcd"); + }); + + test("steps back one unit instead of splitting an emoji", () => { + expect(cutText("abc📚def", 4)).toBe("abc"); + expect(cutText("abc📚def", 5)).toBe("abc📚"); + expect(cutText("📚", 1)).toBe(""); + }); +}); + +describe("loadDocsIndex", () => { + test("returns null for a missing directory", () => { + expect(loadDocsIndex(join(docsDir, "does-not-exist"))).toBeNull(); + }); + + test("returns null for a path that is a file and for a folder without markdown pages", () => { + expect(loadDocsIndex(join(docsDir, "reference", "notes.txt"))).toBeNull(); + const empty = mkdtempSync(join(tmpdir(), "texera-docs-empty-")); + try { + expect(loadDocsIndex(empty)).toBeNull(); + writeFileSync(join(empty, "readme.txt"), "not markdown"); + expect(loadDocsIndex(empty)).toBeNull(); + } finally { + rmSync(empty, { recursive: true, force: true }); + } + }); + + test("indexes every markdown page except the excluded directories and non-markdown files", () => { + const paths = index + .pages() + .map(p => p.path) + .sort(); + expect(paths).toEqual([ + "_index", + "getting-started/datasets", + "getting-started/ui-overview", + "reference/fenced-title", + "reference/long", + "reference/operators/data-input/csv-file-scan", + "reference/plain", + "reference/udf", + ]); + }); + + test("does not treat # lines inside code fences as headings and keeps the code block whole", () => { + expect(index.sections("reference/udf")).toEqual(["Python UDF", "Template", "Notes"]); + const template = index.read("reference/udf", "Template")!; + expect(template).toContain("# comment line inside the fence"); + expect(template).toContain("## not a heading"); + expect(template).toContain("yield tuple_"); + expect(template.startsWith("```python")).toBe(true); + expect(template.endsWith("```")).toBe(true); + }); + + test("takes the title from the first real heading, skipping fenced ones", () => { + const page = index.pages().find(p => p.path === "reference/fenced-title")!; + expect(page.title).toBe("Real title"); + }); + + test("includes an excluded directory when the exclude list is overridden", () => { + const all = loadDocsIndex(docsDir, { exclude: [] })!; + expect(all.pages().map(p => p.path)).toContain("contribution-guidelines/guide-for-developers"); + }); + + test("reads title and description from front matter and strips the ASF header", () => { + const page = index.pages().find(p => p.path === "getting-started/datasets")!; + expect(page.title).toBe("Datasets"); + expect(page.description).toBe("Upload and manage datasets"); + expect(index.read(page.path, "Uploading files")).not.toContain("Licensed to the Apache"); + expect(index.read(page.path, "Uploading files")).not.toContain("---"); + }); + + test("joins a folded multi-line description into one line", () => { + const page = index.pages().find(p => p.path === "getting-started/ui-overview")!; + expect(page.description).toBe("Explore the User Dashboard and its components."); + }); + + test("falls back to the first heading or path as the title without front matter", () => { + const page = index.pages().find(p => p.path === "reference/plain")!; + expect(page.title).toBe("Plain page"); + expect(page.description).toBe(""); + }); + + test("keeps shortcode inner text and drops the shortcode wrappers", () => { + const intro = index.read("getting-started/datasets", "Datasets"); + expect(intro).toContain("A dataset is a versioned collection of files."); + expect(intro).not.toContain("pageinfo"); + const upload = index.read("getting-started/datasets", "Uploading files"); + expect(upload).toContain("Drag and drop also works for uploading."); + expect(upload).not.toContain("alert"); + }); + + test("splits a page into sections by headings with the pre-heading text titled by the page", () => { + expect(index.sections("getting-started/datasets")).toEqual([ + "Datasets", + "Uploading files", + "Supported formats", + "Sharing a dataset", + ]); + }); +}); + +describe("DocsIndex.url", () => { + test("maps _index.md to its directory and a page to its own directory", () => { + expect(index.url("_index")).toBe(`${DOCS_SITE_BASE}/`); + expect(index.url("getting-started/datasets")).toBe(`${DOCS_SITE_BASE}/getting-started/datasets/`); + expect(index.url("tutorials/_index")).toBe(`${DOCS_SITE_BASE}/tutorials/`); + expect(index.url("reference/operators/data-input/csv-file-scan")).toBe( + `${DOCS_SITE_BASE}/reference/operators/data-input/csv-file-scan/` + ); + }); +}); + +describe("DocsIndex.search", () => { + test("ranks the section whose heading matches the query first", () => { + const hits = index.search("how do I upload a file to a dataset"); + expect(hits[0].path).toBe("getting-started/datasets"); + expect(hits[0].headingChain).toEqual(["Datasets", "Uploading files"]); + expect(hits[0].url).toBe(`${DOCS_SITE_BASE}/getting-started/datasets/`); + expect(hits[0].title).toBe("Datasets"); + expect(hits[0].snippet).toContain("Click the Upload button"); + expect(hits[0].score).toBeGreaterThan(0); + }); + + test("matches a camelCase operator name from its split words", () => { + const hits = index.search("csv scan delimiter"); + expect(hits[0].path).toBe("reference/operators/data-input/csv-file-scan"); + }); + + test("matches a camelCase operator name typed as one word", () => { + const hits = index.search("CSVFileScan"); + expect(hits[0].path).toBe("reference/operators/data-input/csv-file-scan"); + }); + + test("honours the limit and returns nothing for a query with no known terms", () => { + expect(index.search("dataset", 1)).toHaveLength(1); + expect(index.search("zzzz qqqq")).toEqual([]); + expect(index.search("")).toEqual([]); + }); + + test("returns at most one hit per section and sorts by descending score", () => { + const hits = index.search("dataset files"); + const keys = hits.map(h => `${h.path}#${h.headingChain.join("/")}`); + expect(new Set(keys).size).toBe(keys.length); + for (let i = 1; i < hits.length; i++) { + expect(hits[i - 1].score).toBeGreaterThanOrEqual(hits[i].score); + } + }); +}); + +describe("DocsIndex.read", () => { + test("returns the named section's text", () => { + expect(index.read("getting-started/datasets", "Sharing a dataset")).toBe( + "Use the share dialog to grant read access to another user." + ); + }); + + test("matches the section name case-insensitively", () => { + expect(index.read("getting-started/datasets", "sharing a dataset")).toContain("share dialog"); + }); + + test("returns the section list and the first section when no section is given", () => { + const text = index.read("getting-started/datasets")!; + expect( + text.startsWith("Sections:\n- Datasets\n- Uploading files\n- Supported formats\n- Sharing a dataset\n\n") + ).toBe(true); + expect(text).toContain("A dataset is a versioned collection of files."); + expect(text).not.toContain("Click the Upload button"); + }); + + test("returns undefined for an unknown page or section", () => { + expect(index.read("nope/missing")).toBeUndefined(); + expect(index.read("getting-started/datasets", "Nope")).toBeUndefined(); + }); + + test("returns a long section in full so the caller decides how to clip it", () => { + expect(index.read("reference/long", "Big section")!.length).toBe(LONG_SECTION.length); + }); +}); + +describe("DocsIndex — headings", () => { + const page = (path: string, title: string, body: string, description = "") => ({ path, title, description, body }); + const chains = (index: DocsIndex, path: string, query: string) => + index + .search(query, 20) + .filter(h => h.path === path) + .map(h => h.headingChain.join(" > ")); + + test("gives sibling headings of the same level the same parent", () => { + const index = new DocsIndex([ + page( + "ops/cast", + "Type Casting", + "### Input Properties\nmarker one\n\n### Output Ports\nmarker two\n\n### Notes\nmarker three" + ), + ]); + expect(chains(index, "ops/cast", "marker").sort()).toEqual([ + "Type Casting > Input Properties", + "Type Casting > Notes", + "Type Casting > Output Ports", + ]); + }); + + test("nests a deeper heading under the nearest shallower one and pops back out", () => { + const body = [ + "## Install", + "marker a", + "### On Linux", + "marker b", + "### On macOS", + "marker c", + "## Run", + "marker d", + "# Appendix", + "marker e", + "### Deep under appendix", + "marker f", + ].join("\n"); + const index = new DocsIndex([page("guide", "Guide", body)]); + expect(chains(index, "guide", "marker").sort()).toEqual( + [ + "Guide > Install", + "Guide > Install > On Linux", + "Guide > Install > On macOS", + "Guide > Run", + "Guide > Appendix", + "Guide > Appendix > Deep under appendix", + ].sort() + ); + }); + + test("does not credit a heading's words to a later sibling", () => { + const index = new DocsIndex([ + page("ops/cast", "Type Casting", "### Zebra Options\nfirst\n\n### Output Ports\nsecond"), + ]); + expect(index.search("zebra", 5).map(h => h.headingChain.join(" > "))).toEqual(["Type Casting > Zebra Options"]); + }); + + test("splits at heading levels one to three only", () => { + const index = new DocsIndex([page("p", "P", "## Two\nalpha\n#### Four\nbeta\n##### Five\ngamma")]); + expect(index.sections("p")).toEqual(["Two"]); + expect(index.read("p", "Two")).toBe("alpha\n#### Four\nbeta\n##### Five\ngamma"); + }); + + test("ranks a word in a heading above the same word twice in another section's text", () => { + const index = new DocsIndex([ + page("p1", "One", "## Alpha\nw x y z"), + page("p2", "Two", "## Other\nalpha alpha q r"), + ]); + expect(index.search("alpha", 2).map(h => h.path)).toEqual(["p1", "p2"]); + }); + + test("names a heading by its link text and without backslash escapes", () => { + const index = new DocsIndex([ + page( + "_index", + "Texera Documentation", + "Welcome.\n\n## 📚 [Getting Started](/docs/getting-started/)\nNew here.\n\n## 4\\. Accessing the Texera UI\nOpen the browser.\n\n## The `readDoc` tool\nReads." + ), + ]); + expect(index.sections("_index")).toEqual([ + "Texera Documentation", + "📚 Getting Started", + "4. Accessing the Texera UI", + "The `readDoc` tool", + ]); + const hit = index.search("new here", 1)[0]; + expect(hit.headingChain).toEqual(["Texera Documentation", "📚 Getting Started"]); + }); + + test("finds a section by its name whatever the case, punctuation or emoji", () => { + const index = new DocsIndex([ + page( + "_index", + "Docs", + "Welcome.\n\n## 📚 [Getting Started](/docs/getting-started/)\nNew here.\n\n## 4\\. Accessing the Texera UI\nOpen the browser." + ), + ]); + for (const name of [ + "📚 Getting Started", + "Getting Started", + "getting started", + " GETTING-STARTED ", + "Docs > 📚 Getting Started", + ]) { + expect(index.read("_index", name)).toBe("New here."); + } + for (const name of ["4. Accessing the Texera UI", "4\\. Accessing the Texera UI", "4 accessing the texera ui"]) { + expect(index.read("_index", name)).toBe("Open the browser."); + } + expect(index.read("_index", "Getting")).toBeUndefined(); + expect(index.read("_index", "xxGetting Started")).toBeUndefined(); + expect(index.read("_index", "Getting Started > Nope")).toBeUndefined(); + }); + + test("treats an empty section name as no section", () => { + const index = new DocsIndex([page("p", "P", "Intro.\n\n## A\nBody.")]); + expect(index.read("p", "")).toBe(index.read("p")); + expect(index.read("p", " ")).toBe(index.read("p")); + }); + + test("numbers later sections that share a name so each can be read", () => { + const index = new DocsIndex([ + page( + "ops/join", + "Hash Join", + "Joins two inputs.\n\n## Hash Join\nThe operator.\n\n## Example\nFirst.\n\n## Example\nSecond.\n\n## example\nThird." + ), + ]); + expect(index.sections("ops/join")).toEqual(["Hash Join", "Hash Join (2)", "Example", "Example (2)", "example (3)"]); + expect(index.read("ops/join", "Hash Join")).toBe("Joins two inputs."); + expect(index.read("ops/join", "Hash Join (2)")).toBe("The operator."); + expect(index.read("ops/join", "Example")).toBe("First."); + expect(index.read("ops/join", "Example (2)")).toBe("Second."); + expect(index.read("ops/join", "example (3)")).toBe("Third."); + expect(index.search("second", 1)[0].headingChain).toEqual(["Hash Join", "Example (2)"]); + }); +}); + +describe("DocsIndex — text shown to the reader", () => { + const page = (path: string, title: string, body: string, description = "") => ({ path, title, description, body }); + + test("removes HTML comments and img tags outside code fences", () => { + const body = [ + "", + "Intro text.", + "", + '', + 'Before after.', + "```html", + "", + '', + "```", + ].join("\n"); + const text = new DocsIndex([page("p", "P", body)]).read("p", "P")!; + expect(text).toBe( + ["Intro text.", "Before after.", "```html", "", '', "```"].join("\n") + ); + }); + + test("introduces a page whose introduction is only generator comments by its description", () => { + const index = new DocsIndex([ + page( + "ops/_index", + "Operators", + "\n\n\n## Quick Links\nLinks.", + "All operators" + ), + ]); + expect(index.read("ops/_index", "Operators")).toBe("All operators"); + }); + + test("cuts a snippet at 200 characters with an ellipsis and leaves a short one whole", () => { + const index = new DocsIndex([ + page("long", "Long", "needle " + "x".repeat(400)), + page("short", "Short", "needle only"), + ]); + const snippets = Object.fromEntries(index.search("needle", 5).map(h => [h.path, h.snippet])); + expect(snippets.long).toBe(`needle ${"x".repeat(193)}…`); + expect(snippets.short).toBe("needle only"); + }); + + test("does not split an emoji at the end of a snippet", () => { + const index = new DocsIndex([page("e", "E", `needle ${"x".repeat(192)}📚 tail`)]); + expect(index.search("needle", 1)[0].snippet).toBe(`needle ${"x".repeat(192)}…`); + }); + + test("returns a whole page as its introduction followed by each section under its heading", () => { + const index = new DocsIndex([ + page("p", "Page", "Intro.\n\n## First\nOne.\n\n### Nested\nTwo.\n\n## Second\nThree."), + ]); + expect(index.page("p")).toBe("Intro.\n\n## First\n\nOne.\n\n### Nested\n\nTwo.\n\n## Second\n\nThree."); + expect(index.page("nope")).toBeUndefined(); + }); + + test("starts a whole page with a heading when the page has no introduction", () => { + const index = new DocsIndex([page("p", "Page", "## Only\nBody.")]); + expect(index.page("p")).toBe("## Only\n\nBody."); + }); +}); + +describe("DocsIndex — path lookup details", () => { + const page = (path: string, title: string) => ({ path, title, description: "", body: "x" }); + const index = new DocsIndex([ + page("overview", "Overview"), + page("getting-started/install", "Install"), + page("Guide", "Upper"), + page("guide", "Lower"), + ]); + + test("ignores a query string, a ./ prefix and doubled slashes", () => { + expect(index.resolve("overview?x=1")).toBe("overview"); + expect(index.resolve("./overview")).toBe("overview"); + expect(index.resolve("getting-started//install")).toBe("getting-started/install"); + expect(index.resolve("./getting-started/./install/")).toBe("getting-started/install"); + }); + + test("accepts the published link whatever the case of the site prefix", () => { + expect(index.resolve("HTTPS://TEXERA.APACHE.ORG/docs/latest/overview/")).toBe("overview"); + }); + + test("prefers the page whose case matches when two paths differ only by case", () => { + expect(index.resolve("Guide")).toBe("Guide"); + expect(index.resolve("guide")).toBe("guide"); + expect(index.resolve("GUIDE")).toBe("Guide"); + }); +}); + +describe("DocsIndex.outline — default threshold", () => { + const filler = (dir: string, count: number) => + Array.from({ length: count }, (_, i) => ({ + path: `${dir}/page-${i + 1}`, + title: `Page ${i + 1}`, + description: "", + body: "x", + })); + + test("expands a directory of 61 pages and keeps one of 60 whole", () => { + const index = new DocsIndex([ + ...filler("big/a", 31), + ...filler("big/b", 30), + ...filler("edge/a", 30), + ...filler("edge/b", 30), + ]); + expect(index.outline().split("\n")).toEqual(["- big/a/ (31 pages)", "- big/b/ (30 pages)", "- edge/ (60 pages)"]); + }); +}); + +describe("DocsIndex — what is indexed", () => { + const page = (path: string, title: string, body: string, description = "") => ({ path, title, description, body }); + + test("does not index words that appear only in an HTML comment or an img tag", () => { + const index = new DocsIndex([ + page( + "p", + "P", + '\nIntro text.\nScreenshot of the canvas\n\n## S\nBody.' + ), + ]); + expect(index.search("screenshot canvas", 5)).toEqual([]); + expect(index.search("generated", 5)).toEqual([]); + expect(index.search("1600 images", 5)).toEqual([]); + expect(index.search("intro", 5).map(h => h.path)).toEqual(["p"]); + }); + + test("keeps tag-like text inside an inline code span, for reading and for search", () => { + const index = new DocsIndex([ + page("p", "P", 'Write `` or `` in markdown.\n'), + ]); + expect(index.read("p", "P")).toBe('Write `` or `` in markdown.'); + expect(index.search("zebra", 5).map(h => h.path)).toEqual(["p"]); + expect(index.search("quagga", 5).map(h => h.path)).toEqual(["p"]); + expect(index.search("real comment", 5)).toEqual([]); + }); +}); + +describe("DocsIndex — finding a section", () => { + const page = (path: string, title: string, body: string, description = "") => ({ path, title, description, body }); + + test("uses the parent in a heading chain to tell two sections of the same name apart", () => { + const index = new DocsIndex([ + page("p", "P", "## Install\ni\n### Linux\ninstall on linux\n## Uninstall\nu\n### Linux\nremove on linux"), + ]); + expect(index.sections("p")).toEqual(["Install", "Linux", "Uninstall", "Linux (2)"]); + expect(index.read("p", "Linux")).toBe("install on linux"); + expect(index.read("p", "Linux (2)")).toBe("remove on linux"); + expect(index.read("p", "Install > Linux")).toBe("install on linux"); + expect(index.read("p", "Uninstall > Linux")).toBe("remove on linux"); + expect(index.read("p", "P > Uninstall > Linux")).toBe("remove on linux"); + expect(index.read("p", "uninstall > linux")).toBe("remove on linux"); + expect(index.read("p", "Uninstall! > Linux")).toBe("remove on linux"); + }); + + test("falls back to the first section of that name when the parent given matches none", () => { + const index = new DocsIndex([page("p", "P", "## Install\ni\n### Linux\ninstall on linux")]); + expect(index.read("p", "Elsewhere > Linux")).toBe("install on linux"); + }); + + test("prefers a section named like the longest end of the chain", () => { + const index = new DocsIndex([page("p", "Page", "## Export\nplain export\n\n## File > Export\nfile menu export")]); + expect(index.read("p", "Page > File > Export")).toBe("file menu export"); + expect(index.read("p", "File > Export")).toBe("file menu export"); + expect(index.read("p", "Export")).toBe("plain export"); + expect(index.read("p", "Page > Export")).toBe("plain export"); + }); + + test("does not take one name for another that differs only in + or #", () => { + const sharp = new DocsIndex([page("p", "P", "## C#\nsharp")]); + expect(sharp.read("p", "C++")).toBeUndefined(); + expect(sharp.read("p", "c#")).toBe("sharp"); + const both = new DocsIndex([page("p", "P", "## C#\nsharp\n\n## C++\nplus")]); + expect(both.sections("p")).toEqual(["C#", "C++"]); + expect(both.read("p", "C++")).toBe("plus"); + expect(both.read("p", "C#")).toBe("sharp"); + }); + + test("matches a name of only symbols exactly or not at all", () => { + const index = new DocsIndex([page("p", "P", "## 📚\nbooks\n\n## ***\nstars")]); + expect(index.sections("p")).toEqual(["📚", "***"]); + expect(index.read("p", "📚")).toBe("books"); + expect(index.read("p", "!!!")).toBeUndefined(); + expect(index.read("p", "🎓")).toBeUndefined(); + }); + + test("prefers an exact name over one that only matches without punctuation", () => { + const index = new DocsIndex([page("p", "P", "## Set-up\nwith a hyphen\n\n## Set up\nwith a space")]); + expect(index.read("p", "Set up")).toBe("with a space"); + expect(index.read("p", "set-up")).toBe("with a hyphen"); + expect(index.read("p", "SET_UP")).toBe("with a hyphen"); + }); + + test("drops a closing run of # from a heading only when a space comes before it", () => { + const index = new DocsIndex([page("p", "P", "## Closed ##\none\n\n## F#\ntwo")]); + expect(index.sections("p")).toEqual(["Closed", "F#"]); + }); + + test("accepts a name written with its markdown heading marks", () => { + const index = new DocsIndex([page("p", "P", "## Getting Started\nbegin")]); + expect(index.read("p", "## Getting Started")).toBe("begin"); + }); + + test("reads a heading whose own name starts with a hash mark", () => { + const index = new DocsIndex([page("p", "P", "## # of workers\ncount\n\n## Getting Started\nbegin")]); + expect(index.read("p", "# of workers")).toBe("count"); + expect(index.read("p", "## # of workers")).toBe("count"); + expect(index.read("p", "## Getting Started")).toBe("begin"); + }); +}); + +describe("DocsIndex — a heading with no text of its own", () => { + const page = (path: string, title: string, body: string, description = "") => ({ path, title, description, body }); + const body = [ + "Intro.", + "## 5. Troubleshooting", + "### File Upload Error", + "forward the port", + "### Port in use", + "stop the other process", + "## Next steps", + "read more", + ].join("\n"); + const index = new DocsIndex([page("k8s", "Run on Kubernetes", body)]); + + test("is listed among the sections", () => { + expect(index.sections("k8s")).toEqual([ + "Run on Kubernetes", + "5. Troubleshooting", + "File Upload Error", + "Port in use", + "Next steps", + ]); + }); + + test("reads as the list of the sections directly under it", () => { + expect(index.read("k8s", "5. Troubleshooting")).toBe("Sub-sections:\n- File Upload Error\n- Port in use"); + expect(index.read("k8s", "Run on Kubernetes > 5. Troubleshooting")).toBe( + "Sub-sections:\n- File Upload Error\n- Port in use" + ); + }); + + test("is found by search and shown as a heading line in the whole page", () => { + expect(index.search("troubleshooting", 1)[0].headingChain).toEqual(["Run on Kubernetes", "5. Troubleshooting"]); + expect(index.page("k8s")).toBe( + [ + "Intro.", + "## 5. Troubleshooting", + "### File Upload Error\n\nforward the port", + "### Port in use\n\nstop the other process", + "## Next steps\n\nread more", + ].join("\n\n") + ); + }); + + test("lists only its direct sub-sections, and those that are headings without text too", () => { + const nested = new DocsIndex([ + page("n", "N", ["## Outer", "### Middle", "#### not a split", "### Empty middle", "## After", "text"].join("\n")), + ]); + expect(nested.sections("n")).toEqual(["Outer", "Middle", "After"]); + expect(nested.read("n", "Outer")).toBe("Sub-sections:\n- Middle"); + expect(nested.read("n", "Middle")).toBe("#### not a split"); + }); + + test("is dropped when nothing under it has text either", () => { + const empty = new DocsIndex([page("e", "E", ["## Hollow", "### Also hollow", "## Real", "text"].join("\n"))]); + expect(empty.sections("e")).toEqual(["Real"]); + expect(empty.search("hollow", 5)).toEqual([]); + }); + + test("keeps a chain of headings without text down to the one that has some", () => { + const deep = new DocsIndex([page("d", "D", ["# Top", "## Mid", "### Leaf", "leaf text"].join("\n"))]); + expect(deep.sections("d")).toEqual(["Top", "Mid", "Leaf"]); + expect(deep.read("d", "Top")).toBe("Sub-sections:\n- Mid"); + expect(deep.read("d", "Mid")).toBe("Sub-sections:\n- Leaf"); + expect(deep.page("d")).toBe("## Top\n\n### Mid\n\n#### Leaf\n\nleaf text"); + }); +}); + +describe("resolveDocLinks", () => { + const OPERATOR_PAGE = "reference/operators/visualization/nested-table"; + + test("maps a site-relative /docs link onto the published base", () => { + expect(resolveDocLinks("[Guide](/docs/getting-started/run-on-kubernetes/)", "security/_index")).toBe( + `[Guide](${DOCS_SITE_BASE}/getting-started/run-on-kubernetes/)` + ); + }); + + test("keeps a /docs/latest link and other site-absolute paths on the site origin", () => { + expect(resolveDocLinks("[a](/docs/latest/tutorials/)", "overview")).toBe( + "[a](https://texera.apache.org/docs/latest/tutorials/)" + ); + expect(resolveDocLinks("![shot](/images/github-assets/a.png)", "overview")).toBe( + "![shot](https://texera.apache.org/images/github-assets/a.png)" + ); + }); + + test("resolves a page-relative link against the page's published URL", () => { + expect(resolveDocLinks("[Home](../)", "reference/operators/output-modes")).toBe( + `[Home](${DOCS_SITE_BASE}/reference/operators/)` + ); + expect(resolveDocLinks("[modes](../../output-modes/#single-snapshot)", OPERATOR_PAGE)).toBe( + `[modes](${DOCS_SITE_BASE}/reference/operators/output-modes/#single-snapshot)` + ); + }); + + test("resolves a bare relative link on an _index page against its directory", () => { + expect(resolveDocLinks("[Data Input](data-input/)", "reference/operators/_index")).toBe( + `[Data Input](${DOCS_SITE_BASE}/reference/operators/data-input/)` + ); + expect(resolveDocLinks("[Start](getting-started/)", "_index")).toBe(`[Start](${DOCS_SITE_BASE}/getting-started/)`); + }); + + test("resolves an anchor-only link against the page itself", () => { + expect(resolveDocLinks("[roles](#user-roles)", "security/_index")).toBe( + `[roles](${DOCS_SITE_BASE}/security/#user-roles)` + ); + }); + + test("resolves a link to a markdown source file relative to the file's directory", () => { + expect(resolveDocLinks("[udf](guide-to-use-python-udf.md)", "tutorials/first-workflow")).toBe( + `[udf](${DOCS_SITE_BASE}/tutorials/guide-to-use-python-udf/)` + ); + expect(resolveDocLinks("[sec](../security/_index.md#roles)", "tutorials/first-workflow")).toBe( + `[sec](${DOCS_SITE_BASE}/security/#roles)` + ); + }); + + test("leaves external links and other schemes alone", () => { + const text = + "[a](https://example.com/x) [b](http://localhost:8080) [c](mailto:dev@texera.apache.org) [d](//cdn.example.com/lib.js)"; + expect(resolveDocLinks(text, OPERATOR_PAGE)).toBe(text); + }); + + test("maps /docs/latest without a trailing slash onto the site origin", () => { + expect(resolveDocLinks("[root](/docs/latest)", "overview")).toBe("[root](https://texera.apache.org/docs/latest)"); + }); + + test("resolves a target written in angle brackets", () => { + expect(resolveDocLinks("[up](<../>)", "tutorials/first-workflow")).toBe(`[up](<${DOCS_SITE_BASE}/tutorials/>)`); + }); + + test("keeps a link title and resolves every link on a line", () => { + expect(resolveDocLinks('[a](../ "Up") and [b](#x)', "tutorials/first-workflow")).toBe( + `[a](${DOCS_SITE_BASE}/tutorials/ "Up") and [b](${DOCS_SITE_BASE}/tutorials/first-workflow/#x)` + ); + }); + + test("does not touch links inside a fenced code block", () => { + const text = ["before [a](../)", "```md", "[b](../)", "```", "~~~", "[c](../)", "~~~", "after [d](../)"].join("\n"); + const out = resolveDocLinks(text, "tutorials/first-workflow").split("\n"); + expect(out[0]).toBe(`before [a](${DOCS_SITE_BASE}/tutorials/)`); + expect(out[2]).toBe("[b](../)"); + expect(out[5]).toBe("[c](../)"); + expect(out[7]).toBe(`after [d](${DOCS_SITE_BASE}/tutorials/)`); + }); + + test("returns text without links unchanged", () => { + expect(resolveDocLinks("plain text (with parens) and [brackets]", "overview")).toBe( + "plain text (with parens) and [brackets]" + ); + }); +}); + +describe("DocsIndex — links, paths and outline", () => { + const page = (path: string, title: string, body: string, description = "") => ({ path, title, description, body }); + const filler = (dir: string, count: number) => + Array.from({ length: count }, (_, i) => page(`${dir}/page-${i + 1}`, `Page ${i + 1}`, "Text.")); + + const tree = new DocsIndex([ + page("_index", "Texera Documentation", "Start with [Getting Started](/docs/getting-started/)."), + page("overview", "Overview", "What Texera is."), + page("getting-started/_index", "Getting Started", "Install and launch Texera."), + page("getting-started/AccessLogin", "Access/Login to Texera", "## Sign in\nUse the login form."), + page("getting-started/install", "Install Texera", "Run the installer."), + page("reference/operators/_index", "Operators", "All operators, see [Data Input](data-input/)."), + page( + "reference/operators/output-modes", + "Output Port Modes", + "[← Home](../)\n\nPorts advertise a mode. See [files](files/data.csv).\n\n## Set Snapshot\nRe-emits." + ), + page("reference/operators/data-input/_index", "Data Input", "Operators that read data."), + page( + "reference/operators/data-input/csv-file-scan", + "CSV File Scan", + [ + "[Home](../../) > [Data Input](../)", + "", + "Reads a CSV file. See [output modes](../../output-modes/#set-snapshot) and the [zebra guide](../zebra-crossing/).", + "", + "```python", + "[Home](../../) > [Data Input](../)", + "```", + ].join("\n") + ), + ...filler("reference/operators/visualization", 3), + page("tutorials/guide", "Guide", "A [link](other.md)."), + page( + "tutorials/download", + "Download", + "[Home](../) > [Tutorials](../) > Download\n\n[Sample data](files/data.csv)" + ), + ]); + + test("stores section text with links resolved to published URLs", () => { + expect(tree.read("_index", "Texera Documentation")).toBe( + `Start with [Getting Started](${DOCS_SITE_BASE}/getting-started/).` + ); + expect(tree.read("reference/operators/_index", "Operators")).toContain( + `[Data Input](${DOCS_SITE_BASE}/reference/operators/data-input/)` + ); + expect(tree.read("tutorials/guide", "Guide")).toBe(`A [link](${DOCS_SITE_BASE}/tutorials/other/).`); + }); + + test("drops a breadcrumb line but keeps the same text inside a code fence", () => { + const text = tree.read("reference/operators/data-input/csv-file-scan", "CSV File Scan")!; + expect(text.startsWith("Reads a CSV file.")).toBe(true); + expect(text.match(/\[Home\]/g)).toHaveLength(1); + expect(text).toContain("```python\n[Home](../../) > [Data Input](../)\n```"); + }); + + test("drops a lone up link and a breadcrumb that ends in the page name", () => { + expect(tree.read("reference/operators/output-modes", "Output Port Modes")).toBe( + `Ports advertise a mode. See [files](${DOCS_SITE_BASE}/reference/operators/output-modes/files/data.csv).` + ); + expect(tree.read("tutorials/download", "Download")).toBe( + `[Sample data](${DOCS_SITE_BASE}/tutorials/download/files/data.csv)` + ); + }); + + test("keeps a line whose only link does not point up the tree", () => { + expect(tree.read("tutorials/download", "Download")).toContain("[Sample data]("); + expect(tree.read("_index", "Texera Documentation")).toContain("[Getting Started]("); + }); + + test("shows link text without the URL in a search snippet", () => { + const hit = tree.search("csv file scan", 1)[0]; + expect(hit.path).toBe("reference/operators/data-input/csv-file-scan"); + expect(hit.snippet).toContain("See output modes and the zebra guide."); + expect(hit.snippet).not.toContain("https://"); + }); + + test("indexes the source text, so the site prefix that link resolution adds is not searchable", () => { + expect(tree.search("zebra crossing", 5).map(h => h.path)).toEqual(["reference/operators/data-input/csv-file-scan"]); + expect(tree.search("latest", 5)).toEqual([]); + expect(tree.search("https apache", 5)).toEqual([]); + }); + + test("still finds a page by a word of the breadcrumb that is no longer shown", () => { + const hits = tree.search("home", 5).map(h => h.path); + expect(hits).toContain("reference/operators/data-input/csv-file-scan"); + expect(tree.read("reference/operators/data-input/csv-file-scan", "CSV File Scan")!.startsWith("Reads")).toBe(true); + }); + + describe("a page whose introduction is empty once the breadcrumb is gone", () => { + const operators = new DocsIndex([ + page( + "ops/distinct", + "Distinct", + "[Home](../../) > [Data Cleaning](../)\n\n### Input Properties\nNo properties.", + "Remove duplicate rows" + ), + page("ops/limit", "Limit", "### Input Properties\nlimit: how many rows to keep.", "Keep the first N rows"), + page("ops/bare", "Bare", "### Input Properties\nNothing here."), + page("ops/empty", "Empty", "", "Only a description"), + ]); + + test("uses the description as the introduction, so the page is found by it", () => { + expect(operators.search("remove duplicate rows", 1)[0].path).toBe("ops/distinct"); + expect(operators.search("keep first rows", 1)[0].path).toBe("ops/limit"); + expect(operators.sections("ops/distinct")).toEqual(["Distinct", "Input Properties"]); + expect(operators.read("ops/distinct")).toBe("Sections:\n- Distinct\n- Input Properties\n\nRemove duplicate rows"); + expect(operators.read("ops/empty")).toBe("Sections:\n- Empty\n\nOnly a description"); + }); + + test("keeps the breadcrumb's words searchable for a page whose introduction was only the breadcrumb", () => { + expect(operators.search("data cleaning", 5).map(h => h.path)).toEqual(["ops/distinct"]); + }); + + test("adds no introduction to a page that has neither text nor description before its first heading", () => { + expect(operators.sections("ops/bare")).toEqual(["Input Properties"]); + }); + }); + + describe("resolve", () => { + test("returns the canonical path for an exact, slashed, .md or differently cased path", () => { + expect(tree.resolve("getting-started/install")).toBe("getting-started/install"); + expect(tree.resolve("/getting-started/install/")).toBe("getting-started/install"); + expect(tree.resolve("getting-started/install.md")).toBe("getting-started/install"); + expect(tree.resolve("getting-started/accesslogin")).toBe("getting-started/AccessLogin"); + expect(tree.resolve("GETTING-STARTED/ACCESSLOGIN.MD")).toBe("getting-started/AccessLogin"); + }); + + test("resolves a directory to its _index page and the empty path to the root page", () => { + expect(tree.resolve("getting-started")).toBe("getting-started/_index"); + expect(tree.resolve("getting-started/")).toBe("getting-started/_index"); + expect(tree.resolve("Reference/Operators")).toBe("reference/operators/_index"); + expect(tree.resolve("")).toBe("_index"); + expect(tree.resolve("/")).toBe("_index"); + }); + + test("accepts a published URL or a /docs path, with or without an anchor", () => { + expect(tree.resolve(`${DOCS_SITE_BASE}/getting-started/AccessLogin/`)).toBe("getting-started/AccessLogin"); + expect(tree.resolve(`${DOCS_SITE_BASE}/reference/operators/output-modes/#set-snapshot`)).toBe( + "reference/operators/output-modes" + ); + expect(tree.resolve("/docs/getting-started/")).toBe("getting-started/_index"); + expect(tree.resolve("/docs/latest/overview/")).toBe("overview"); + expect(tree.resolve(`${DOCS_SITE_BASE}/`)).toBe("_index"); + }); + + test("returns undefined for a page that does not exist", () => { + expect(tree.resolve("getting-started/quickstart")).toBeUndefined(); + expect(tree.resolve("../../etc/passwd")).toBeUndefined(); + expect(tree.resolve("reference")).toBeUndefined(); + expect(tree.resolve("https://example.com/getting-started/install")).toBeUndefined(); + }); + + test("keeps the canonical case in the page URL", () => { + expect(tree.url(tree.resolve("getting-started/accesslogin")!)).toBe( + `${DOCS_SITE_BASE}/getting-started/AccessLogin/` + ); + }); + }); + + describe("directory and children", () => { + test("names a directory without an _index page, whatever the case or slashes", () => { + expect(tree.directory("reference")).toBe("reference"); + expect(tree.directory("/Reference/")).toBe("reference"); + expect(tree.directory("tutorials")).toBe("tutorials"); + expect(tree.directory("")).toBe(""); + expect(tree.directory("getting-started/install")).toBeUndefined(); + expect(tree.directory("nope")).toBeUndefined(); + }); + + test("lists the pages directly under a directory without its _index page", () => { + const { pages, dirs } = tree.children("getting-started"); + expect(pages.map(p => p.path)).toEqual(["getting-started/AccessLogin", "getting-started/install"]); + expect(dirs).toEqual([]); + }); + + test("lists sub-directories with their page count and _index title", () => { + const { pages, dirs } = tree.children("reference/operators"); + expect(pages.map(p => p.path)).toEqual(["reference/operators/output-modes"]); + expect(dirs).toEqual([ + { path: "reference/operators/data-input", pageCount: 2, title: "Data Input" }, + { path: "reference/operators/visualization", pageCount: 3, title: undefined }, + ]); + }); + + test("lists the top level for the root", () => { + const { pages, dirs } = tree.children(""); + expect(pages.map(p => p.path)).toEqual(["overview"]); + expect(dirs.map(d => `${d.path}:${d.pageCount}`)).toEqual(["getting-started:3", "reference:7", "tutorials:2"]); + }); + + test("returns nothing for a directory that does not exist", () => { + expect(tree.children("nope")).toEqual({ pages: [], dirs: [] }); + }); + }); + + describe("outline", () => { + test("lists top-level pages and directories with page counts and titles", () => { + expect(tree.outline(100).split("\n")).toEqual([ + "- overview: Overview", + "- getting-started/ (3 pages): Getting Started", + "- reference/ (7 pages)", + "- tutorials/ (2 pages)", + ]); + }); + + test("writes a single page count in the singular", () => { + const one = new DocsIndex([page("a/only", "Only", "x")]); + expect(one.outline()).toBe("- a/ (1 page)"); + }); + + test("expands a directory holding more pages than the threshold into its children", () => { + expect(tree.outline(5).split("\n")).toEqual([ + "- overview: Overview", + "- getting-started/ (3 pages): Getting Started", + "- reference/operators/output-modes: Output Port Modes", + "- reference/operators/data-input/ (2 pages): Data Input", + "- reference/operators/visualization/ (3 pages)", + "- tutorials/ (2 pages)", + ]); + }); + + test("never lists every page of a large tree", () => { + const big = new DocsIndex([ + page("_index", "Root", "x"), + ...filler("a", 80), + ...filler("b/c", 70), + ...filler("b/d", 70), + ]); + const lines = big.outline().split("\n"); + expect(lines.length).toBeLessThan(10); + expect(lines).toContain("- b/c/ (70 pages)"); + }); + + test("counts the own pages of an expanded directory instead of naming more than eight", () => { + const mixed = new DocsIndex([...filler("b", 9), ...filler("b/c", 70), ...filler("d", 8), ...filler("d/e", 70)]); + expect(mixed.outline().split("\n")).toEqual([ + "- b/ (9 pages of its own)", + "- b/c/ (70 pages)", + ...Array.from({ length: 8 }, (_, i) => `- d/page-${i + 1}: Page ${i + 1}`), + "- d/e/ (70 pages)", + ]); + }); + + test("counts the top-level pages instead of naming more than eight", () => { + const flat = new DocsIndex(Array.from({ length: 9 }, (_, i) => page(`p${i}`, `P${i}`, "x"))); + expect(flat.outline()).toBe("- the top level (9 pages of its own)"); + }); + }); + + describe("closest", () => { + test("ranks pages sharing words with the requested path, best first", () => { + expect(tree.closest("getting-started/login").map(p => p.path)[0]).toBe("getting-started/AccessLogin"); + expect(tree.closest("operators/csv-scan").map(p => p.path)[0]).toBe( + "reference/operators/data-input/csv-file-scan" + ); + }); + + test("honours the limit and returns nothing when no word is shared", () => { + expect(tree.closest("reference/operators/visualization/page", 2)).toHaveLength(2); + expect(tree.closest("zzzz/qqqq")).toEqual([]); + expect(tree.closest("")).toEqual([]); + }); + }); +}); diff --git a/agent-service/src/agent/tools/docs-index.ts b/agent-service/src/agent/tools/docs-index.ts new file mode 100644 index 00000000000..de57a4820fa --- /dev/null +++ b/agent-service/src/agent/tools/docs-index.ts @@ -0,0 +1,678 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative, sep } from "node:path"; + +/** + * Base of the page links handed to the model. Each page under it is a stub on the published + * site that forwards to the same page of the current release. + */ +export const DOCS_SITE_BASE = "https://texera.apache.org/docs/latest"; + +const SITE_ORIGIN = new URL(DOCS_SITE_BASE).origin; + +const DEFAULT_EXCLUDED_DIRS = ["contribution-guidelines"]; +const HEADING_WEIGHT = 3; +const SNIPPET_CHAR_LIMIT = 200; +/** A directory with more pages than this is shown in the outline through its sub-directories. */ +const OUTLINE_EXPAND_ABOVE = 60; +/** Most pages of an expanded directory the outline names one by one. */ +const OUTLINE_DIRECT_PAGES = 8; + +export interface DocsPage { + path: string; + title: string; + description: string; +} + +export interface DocsSection { + path: string; + /** Page title, then the headings above the section, then its own name, unique on the page. */ + headingChain: string[]; + text: string; +} + +export interface DocsDirectory { + path: string; + pageCount: number; + title?: string; +} + +export interface DocsSearchHit { + path: string; + title: string; + headingChain: string[]; + url: string; + snippet: string; + score: number; +} + +export interface LoadDocsIndexOptions { + exclude?: string[]; +} + +/** The first `limit` UTF-16 units of text, one fewer when that would split a surrogate pair. */ +export function cutText(text: string, limit: number): string { + if (text.length <= limit) return text; + const last = text.charCodeAt(limit - 1); + return text.slice(0, last >= 0xd800 && last <= 0xdbff ? limit - 1 : limit); +} + +/** Strips common English suffixes so singular and plural or verb forms share a token. */ +function stem(token: string): string { + if (token.length > 4 && token.endsWith("ies")) return `${token.slice(0, -3)}y`; + if (token.length > 5 && token.endsWith("ing")) return token.slice(0, -3); + if (token.length > 4 && token.endsWith("ed")) return token.slice(0, -2); + if (token.length > 3 && token.endsWith("s") && !token.endsWith("ss")) return token.slice(0, -1); + return token; +} + +const STOPWORDS = new Set([ + "a", + "an", + "the", + "is", + "are", + "do", + "does", + "how", + "what", + "where", + "i", + "my", + "to", + "in", + "of", + "for", + "on", + "with", + "and", + "or", + "it", + "this", + "that", + "can", + "you", +]); + +/** Splits camelCase, lowercases, stems, drops one-char tokens and stopwords, and keeps each unsplit word too. */ +export function tokenize(text: string): string[] { + const tokens: string[] = []; + for (const word of text.split(/[^A-Za-z0-9]+/)) { + if (word.length === 0) continue; + const lower = word.toLowerCase(); + if (STOPWORDS.has(lower)) continue; + const parts = word + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .toLowerCase() + .split(" ") + .filter(t => t.length > 1 && !STOPWORDS.has(t)) + .map(stem); + tokens.push(...parts); + const whole = stem(lower); + if (whole.length > 1 && !(parts.length === 1 && parts[0] === whole)) tokens.push(whole); + } + return tokens; +} + +function stripAsfHeader(markdown: string): string { + const trimmed = markdown.trimStart(); + if (!trimmed.startsWith(""); + return end === -1 ? markdown : trimmed.slice(end + 3); +} + +function parseFrontMatter(markdown: string): { title?: string; description?: string; body: string } { + const trimmed = markdown.trimStart(); + const match = trimmed.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/); + if (!match) return { body: trimmed }; + const fields: Record = {}; + const lines = match[1].split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const kv = lines[i].match(/^([A-Za-z_]+):\s*(.*)$/); + if (!kv) continue; + let value = kv[2].trim(); + if (/^[>|][-+]?$/.test(value)) { + const folded: string[] = []; + while (i + 1 < lines.length && /^\s+\S/.test(lines[i + 1])) folded.push(lines[++i].trim()); + value = folded.join(" "); + } + fields[kv[1]] = value.replace(/^["'](.*)["']$/, "$1"); + } + return { title: fields.title, description: fields.description, body: trimmed.slice(match[0].length) }; +} + +/** Published URL of a page. An _index page is published at its directory. */ +export function docsPageUrl(path: string): string { + if (path === "_index") return `${DOCS_SITE_BASE}/`; + const dir = path.endsWith("/_index") ? path.slice(0, -"/_index".length) : path; + return `${DOCS_SITE_BASE}/${dir}/`; +} + +function resolveLinkTarget(target: string, pagePath: string): string { + if (/^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith("//")) return target; + if (target.startsWith("#")) return `${docsPageUrl(pagePath)}${target}`; + if (target === "/docs/latest" || target.startsWith("/docs/latest/")) return `${SITE_ORIGIN}${target}`; + if (target.startsWith("/docs/")) return `${DOCS_SITE_BASE}${target.slice("/docs".length)}`; + if (target.startsWith("/")) return `${SITE_ORIGIN}${target}`; + try { + const source = target.match(/^([^#?]*)\.md([#?].*)?$/); + if (source) { + // A link to a markdown file is relative to the directory of the file it is written in. + const sourceDir = pagePath.includes("/") ? pagePath.slice(0, pagePath.lastIndexOf("/") + 1) : ""; + const file = new URL(source[1], `${DOCS_SITE_BASE}/${sourceDir}`).href; + const page = file.endsWith("/_index") ? file.slice(0, -"_index".length) : `${file}/`; + return `${page}${source[2] ?? ""}`; + } + return new URL(target, docsPageUrl(pagePath)).href; + } catch { + return target; + } +} + +const FENCE = /^\s*(```|~~~)/; +const LINK_TARGET = /(\]\(\s*]+)(>?(?:\s+"[^"]*")?\s*\))/g; +/** One or more links that only point up the tree, joined by ">", optionally ending in the page name. */ +const BREADCRUMB = /^\[[^\]]+\]\((?:\.\.\/)+\)(?:\s*>\s*\[[^\]]+\]\((?:\.\.\/)+\))*(?:\s*>\s*[^[\]]+)?$/; + +/** Applies a rewrite to each run of lines outside fenced code blocks. */ +function mapProse(markdown: string, rewrite: (prose: string) => string): string { + const out: string[] = []; + let prose: string[] = []; + let inFence = false; + const flush = () => { + if (prose.length > 0) out.push(rewrite(prose.join("\n"))); + prose = []; + }; + for (const line of markdown.split(/\r?\n/)) { + if (FENCE.test(line)) { + if (!inFence) flush(); + inFence = !inFence; + out.push(line); + } else if (inFence) { + out.push(line); + } else { + prose.push(line); + } + } + flush(); + return out.join("\n"); +} + +/** Rewrites relative and site-relative link targets to the published URL they lead to. */ +export function resolveDocLinks(markdown: string, pagePath: string): string { + return mapProse(markdown, prose => + prose.replace(LINK_TARGET, (_, open, target, close) => `${open}${resolveLinkTarget(target, pagePath)}${close}`) + ); +} + +const REMOVED = "\u0000"; + +/** Drops HTML comments and img tags, which only a browser can use. Inline code is left as it is. */ +function dropHtmlNoise(markdown: string): string { + return mapProse(markdown, prose => + prose + .replace(/(`+)[^`\n]*\1||]*>/gi, match => (match.startsWith("`") ? match : REMOVED)) + .split("\n") + // A line that held nothing else goes with what was removed from it. + .filter(line => !line.includes(REMOVED) || line.split(REMOVED).join("").trim() !== "") + .join("\n") + .split(REMOVED) + .join("") + ); +} + +function dropBreadcrumbs(markdown: string): string { + return mapProse(markdown, prose => + prose + .split("\n") + .filter(line => !BREADCRUMB.test(line.trim())) + .join("\n") + ); +} + +/** Replaces each link or image by its text, for tokenizing, snippets and heading names. */ +function withoutLinkTargets(text: string): string { + return text.replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1"); +} + +/** A heading as it is shown and matched: link text instead of links, no backslash escapes. */ +function cleanHeading(raw: string): string { + return withoutLinkTargets(raw) + .replace(/\\([\\`*_{}[\]()#+\-.!>~|])/g, "$1") + .replace(/\s+/g, " ") + .trim(); +} + +/** Form in which section names are compared loosely: case, punctuation and symbols other than + and # do not count. */ +function sectionKey(name: string): string { + return name + .toLowerCase() + .replace(/[^\p{L}\p{N}+#]+/gu, " ") + .trim(); +} + +/** What makes two section names on a page the same name. */ +function nameIdentity(name: string): string { + return sectionKey(name) || name.toLowerCase(); +} + +function sameName(a: string, b: string): boolean { + return a.toLowerCase() === b.toLowerCase() || (sectionKey(a) !== "" && sectionKey(a) === sectionKey(b)); +} + +/** A section as stored: its name before numbering, and whether it is a heading with only sub-sections. */ +interface StoredSection extends DocsSection { + ownName: string; + container: boolean; +} + +function stripShortcodes(markdown: string): string { + return markdown.replace(/\{\{%\s*\/?\s*pageinfo[^%]*%\}\}/g, "").replace(/\{\{<\s*\/?\s*alert[^>]*>\}\}/g, ""); +} + +/** Every section of a page in order, the introduction first, including those without text. */ +function splitSections(path: string, pageTitle: string, body: string): DocsSection[] { + const all: DocsSection[] = []; + const chain: string[] = [pageTitle]; + // Level of each chain entry, so a heading closes every open heading of its own level or deeper. + const levels: number[] = [0]; + let current: DocsSection = { path, headingChain: [...chain], text: "" }; + let buffer: string[] = []; + const flush = () => { + current.text = buffer.join("\n").trim(); + all.push(current); + buffer = []; + }; + let inFence = false; + for (const line of body.split(/\r?\n/)) { + if (/^\s*(```|~~~)/.test(line)) { + inFence = !inFence; + buffer.push(line); + continue; + } + const heading = inFence ? null : line.match(/^(#{1,3})\s+(.+?)(?:\s+#+)?\s*$/); + if (!heading) { + buffer.push(line); + continue; + } + flush(); + const level = heading[1].length; + while (levels.length > 1 && levels[levels.length - 1] >= level) { + levels.pop(); + chain.pop(); + } + levels.push(level); + chain.push(cleanHeading(heading[2])); + current = { path, headingChain: [...chain], text: "" }; + } + flush(); + return all; +} + +/** Title of the first heading section on a page without front matter, if the page has one. */ +function firstHeading(path: string, body: string): string | undefined { + return splitSections(path, "", body).find(s => s.headingChain.length > 1)?.headingChain[1]; +} + +function listMarkdownFiles(root: string, excluded: Set): string[] { + const files: string[] = []; + const walk = (dir: string) => { + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const rel = relative(root, full).split(sep).join("/"); + if (statSync(full).isDirectory()) { + if (!excluded.has(rel)) walk(full); + } else if (entry.endsWith(".md")) { + files.push(rel); + } + } + }; + walk(root); + return files.sort(); +} + +/** In-memory tf-idf index over the markdown pages of a docs directory, split by heading. */ +export class DocsIndex { + private readonly pageList: DocsPage[] = []; + private readonly pageByPath = new Map(); + private readonly pathByKey = new Map(); + private readonly directoryByKey = new Map([["", ""]]); + private readonly sectionsByPath = new Map(); + private readonly sectionList: DocsSection[] = []; + private readonly termFrequencies: Map[] = []; + private readonly documentFrequency = new Map(); + private readonly sectionLengths: number[] = []; + + constructor(pages: { path: string; title: string; description: string; body: string }[]) { + for (const page of pages) { + const entry = { path: page.path, title: page.title, description: page.description }; + this.pageList.push(entry); + this.pageByPath.set(page.path, entry); + if (!this.pathByKey.has(page.path.toLowerCase())) this.pathByKey.set(page.path.toLowerCase(), page.path); + const segments = page.path.split("/"); + for (let depth = 1; depth < segments.length; depth++) { + const dir = segments.slice(0, depth).join("/"); + this.directoryByKey.set(dir.toLowerCase(), dir); + } + // Ranking uses the source text without HTML comments and img tags, so it does not depend on + // how links and breadcrumbs are shown to the reader. + const prepared = splitSections(page.path, page.title, page.body).map(source => { + const isIntroduction = source.headingChain.length === 1; + const indexed = dropHtmlNoise(source.text); + let text = resolveDocLinks(dropBreadcrumbs(indexed), page.path).trim(); + if (text.length === 0 && isIntroduction) text = page.description; + return { source, isIntroduction, indexed, text }; + }); + // A heading without text of its own stays when a section under it stays. + const kept = prepared.map(entry => entry.text.length > 0); + for (let i = prepared.length - 1; i >= 0; i--) { + if (kept[i] || prepared[i].isIntroduction) continue; + const depth = prepared[i].source.headingChain.length; + for (let j = i + 1; j < prepared.length && prepared[j].source.headingChain.length > depth; j++) { + if (kept[j]) kept[i] = true; + } + } + const built = prepared + .filter((_, i) => kept[i]) + .map(({ source, indexed, text }) => { + const ownName = source.headingChain[source.headingChain.length - 1]; + const section: StoredSection = { ...source, text, ownName, container: text.length === 0 }; + return { section, indexed }; + }); + // A page with nothing before its first heading is introduced by its description. + if (built[0]?.section.headingChain.length !== 1 && (page.description || built.length === 0)) { + const introduction: StoredSection = { + path: page.path, + headingChain: [page.title], + text: page.description, + ownName: page.title, + container: false, + }; + built.unshift({ section: introduction, indexed: "" }); + } + // A later section with a name already used on the page gets a number, so each can be read. + const used = new Set(); + for (const { section } of built) { + let name = section.ownName; + for (let n = 2; used.has(nameIdentity(name)); n++) name = `${section.ownName} (${n})`; + used.add(nameIdentity(name)); + section.headingChain = [...section.headingChain.slice(0, -1), name]; + } + const sections = built.map(entry => entry.section); + // A heading without text reads as the list of the sections directly under it. + sections.forEach((section, i) => { + if (!section.container) return; + const depth = section.headingChain.length; + const names: string[] = []; + for (let j = i + 1; j < sections.length && sections[j].headingChain.length > depth; j++) { + if (sections[j].headingChain.length === depth + 1) names.push(`- ${DocsIndex.nameOf(sections[j])}`); + } + section.text = `Sub-sections:\n${names.join("\n")}`; + }); + for (const { section, indexed } of built) this.addSection(section, indexed, page.description); + this.sectionsByPath.set(page.path, sections); + } + } + + /** Indexes a section by its headings, the page description for an introduction, and its source text. */ + private addSection(section: DocsSection, sourceText: string, description: string): void { + const tf = new Map(); + const bump = (tokens: string[], weight: number) => { + for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + weight); + }; + bump(tokenize(section.headingChain.join(" ")), HEADING_WEIGHT); + if (section.headingChain.length === 1) bump(tokenize(description), HEADING_WEIGHT); + bump(tokenize(sourceText), 1); + let length = 0; + for (const [term, count] of tf) { + length += count; + this.documentFrequency.set(term, (this.documentFrequency.get(term) ?? 0) + 1); + } + this.sectionList.push(section); + this.termFrequencies.push(tf); + this.sectionLengths.push(length); + } + + pages(): DocsPage[] { + return [...this.pageList]; + } + + private static nameOf(section: DocsSection): string { + return section.headingChain[section.headingChain.length - 1]; + } + + sections(path: string): string[] { + return (this.sectionsByPath.get(path) ?? []).map(DocsIndex.nameOf); + } + + url(path: string): string { + return docsPageUrl(path); + } + + /** A path without site prefix, anchor, query, empty or "." segments and .md, in the case it was given. */ + private static clean(input: string): string { + let path = input.trim(); + if (path.toLowerCase().startsWith(DOCS_SITE_BASE.toLowerCase())) path = path.slice(DOCS_SITE_BASE.length); + return path + .replace(/[#?].*$/, "") + .split("/") + .filter(segment => segment !== "" && segment !== ".") + .join("/") + .replace(/\.md$/i, ""); + } + + /** Canonical path of the page a path, directory, /docs path or published URL names, whatever its case. */ + resolve(input: string): string | undefined { + const path = DocsIndex.clean(input); + const candidates = [path, path.replace(/^docs\/latest(\/|$)/i, ""), path.replace(/^docs(\/|$)/i, "")]; + const withIndex = candidates.flatMap(c => [c, c === "" ? "_index" : `${c}/_index`]); + // A page spelled exactly as asked wins over one that differs only by case. + return ( + withIndex.find(c => this.pageByPath.has(c)) ?? + withIndex.map(c => this.pathByKey.get(c.toLowerCase())).find(found => found !== undefined) + ); + } + + /** Canonical name of a directory that holds pages, "" for the root. */ + directory(input: string): string | undefined { + return this.directoryByKey.get(DocsIndex.clean(input).toLowerCase()); + } + + /** Pages directly inside a directory, without its _index page, and its sub-directories. */ + children(dir: string): { pages: DocsPage[]; dirs: DocsDirectory[] } { + const prefix = dir === "" ? "" : `${dir}/`; + const pages: DocsPage[] = []; + const counts = new Map(); + for (const page of this.pageList) { + if (!page.path.startsWith(prefix)) continue; + const rest = page.path.slice(prefix.length); + const slash = rest.indexOf("/"); + if (slash === -1) { + if (rest !== "_index") pages.push(page); + } else { + const sub = `${prefix}${rest.slice(0, slash)}`; + counts.set(sub, (counts.get(sub) ?? 0) + 1); + } + } + const dirs = [...counts.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([path, pageCount]) => ({ path, pageCount, title: this.pageByPath.get(`${path}/_index`)?.title })); + return { pages, dirs }; + } + + /** Short map of the documentation: top-level pages and directories with their page counts. */ + outline(expandAbove = OUTLINE_EXPAND_ABOVE): string { + return this.outlineLines("", expandAbove).join("\n"); + } + + private outlineLines(dir: string, expandAbove: number): string[] { + const { pages, dirs } = this.children(dir); + const lines = + pages.length <= OUTLINE_DIRECT_PAGES + ? pages.map(page => `- ${page.path}: ${page.title}`) + : [`- ${dir === "" ? "the top level" : `${dir}/`} (${pages.length} pages of its own)`]; + for (const sub of dirs) { + if (sub.pageCount > expandAbove && this.children(sub.path).dirs.length > 0) { + lines.push(...this.outlineLines(sub.path, expandAbove)); + } else { + const count = sub.pageCount === 1 ? "1 page" : `${sub.pageCount} pages`; + lines.push(`- ${sub.path}/ (${count})${sub.title ? `: ${sub.title}` : ""}`); + } + } + return lines; + } + + /** Pages whose path or title shares the most words with a path that names no page. */ + closest(input: string, limit = 5): DocsPage[] { + const wanted = new Set(tokenize(input)); + if (wanted.size === 0) return []; + const scored: { page: DocsPage; score: number }[] = []; + for (const page of this.pageList) { + const have = new Set(tokenize(`${page.path} ${page.title}`)); + let score = 0; + for (const token of wanted) if (have.has(token)) score++; + if (score > 0) scored.push({ page, score }); + } + scored.sort( + (a, b) => b.score - a.score || a.page.path.length - b.page.path.length || a.page.path.localeCompare(b.page.path) + ); + return scored.slice(0, limit).map(entry => entry.page); + } + + /** Scores every section by tf-idf over the query tokens and returns the best ones. */ + search(query: string, limit = 5): DocsSearchHit[] { + const queryTerms = [...new Set(tokenize(query))]; + if (queryTerms.length === 0) return []; + const total = this.sectionList.length; + const scored: { index: number; score: number }[] = []; + for (let i = 0; i < total; i++) { + const tf = this.termFrequencies[i]; + let score = 0; + for (const term of queryTerms) { + const count = tf.get(term); + if (!count) continue; + const df = this.documentFrequency.get(term) ?? 1; + const idf = Math.log(1 + total / df); + score += (count / Math.sqrt(this.sectionLengths[i] || 1)) * idf; + } + if (score > 0) scored.push({ index: i, score }); + } + scored.sort((a, b) => b.score - a.score); + return scored.slice(0, limit).map(({ index, score }) => { + const section = this.sectionList[index]; + const page = this.pageByPath.get(section.path)!; + return { + path: section.path, + title: page.title, + headingChain: section.headingChain, + url: this.url(section.path), + snippet: this.snippet(section.text), + score: Math.round(score * 1000) / 1000, + }; + }); + } + + /** Returns a section's text, or the section list plus the first section when no section is named. */ + read(path: string, section?: string): string | undefined { + const sections = this.sectionsByPath.get(path); + if (!sections) return undefined; + if (section === undefined || section.trim() === "") { + const names = this.sections(path).map(name => `- ${name}`); + return `Sections:\n${names.join("\n")}\n\n${sections[0].text}`; + } + return DocsIndex.find(sections, section)?.text; + } + + /** Sections whose name is the one given, those named exactly so first, then by loose comparison. */ + private static named(sections: StoredSection[], name: string): StoredSection[] { + const byName = (same: (candidate: string) => boolean) => [ + ...sections.filter(s => same(DocsIndex.nameOf(s))), + ...sections.filter(s => !same(DocsIndex.nameOf(s)) && same(s.ownName)), + ]; + const exact = byName(candidate => candidate.toLowerCase() === name.toLowerCase()); + if (exact.length > 0) return exact; + const key = sectionKey(name); + return key === "" ? [] : byName(candidate => sectionKey(candidate) === key); + } + + /** + * The section a name or a "Page > Heading > Section" chain points at. The longest end of the + * chain that names a section wins, and the part before it picks among sections of that name. + */ + private static find(sections: StoredSection[], wanted: string): StoredSection | undefined { + const parts = wanted.split(" > ").map(part => part.trim()); + for (let start = 0; start < parts.length; start++) { + const name = parts.slice(start).join(" > "); + // As given first, so a heading that itself starts with "#" is found; then without heading marks. + const asGiven = DocsIndex.named(sections, name); + const candidates = asGiven.length > 0 ? asGiven : DocsIndex.named(sections, name.replace(/^#+\s*/, "")); + if (candidates.length === 0) continue; + const parent = start > 0 ? parts[start - 1] : undefined; + const underParent = + parent === undefined + ? undefined + : candidates.find( + c => c.headingChain.length > 1 && sameName(c.headingChain[c.headingChain.length - 2], parent) + ); + return underParent ?? candidates[0]; + } + return undefined; + } + + /** The whole page: its introduction, then each section under its heading. */ + page(path: string): string | undefined { + const sections = this.sectionsByPath.get(path); + if (!sections) return undefined; + return sections + .map(s => { + const depth = s.headingChain.length; + if (depth === 1) return s.text; + const heading = `${"#".repeat(Math.min(6, depth))} ${s.headingChain[depth - 1]}`; + return s.container ? heading : `${heading}\n\n${s.text}`; + }) + .join("\n\n"); + } + + private snippet(text: string): string { + const flat = withoutLinkTargets(text).replace(/\s+/g, " ").trim(); + return flat.length <= SNIPPET_CHAR_LIMIT ? flat : `${cutText(flat, SNIPPET_CHAR_LIMIT)}…`; + } +} + +/** Reads every markdown page under dir into a DocsIndex, or returns null when dir is missing or has no page. */ +export function loadDocsIndex(dir: string, options: LoadDocsIndexOptions = {}): DocsIndex | null { + if (!existsSync(dir) || !statSync(dir).isDirectory()) return null; + const excluded = new Set(options.exclude ?? DEFAULT_EXCLUDED_DIRS); + const pages = listMarkdownFiles(dir, excluded).map(rel => { + const raw = readFileSync(join(dir, rel), "utf8"); + const { title, description, body } = parseFrontMatter(stripAsfHeader(raw)); + const path = rel.slice(0, -".md".length); + const cleaned = stripShortcodes(body); + return { + path, + title: title ?? firstHeading(path, cleaned) ?? path, + description: description ?? "", + body: cleaned, + }; + }); + return pages.length === 0 ? null : new DocsIndex(pages); +} diff --git a/agent-service/src/agent/tools/docs-tools.spec.ts b/agent-service/src/agent/tools/docs-tools.spec.ts new file mode 100644 index 00000000000..7ccce57dfee --- /dev/null +++ b/agent-service/src/agent/tools/docs-tools.spec.ts @@ -0,0 +1,724 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DOCS_SITE_BASE, DocsIndex, loadDocsIndex } from "./docs-index"; +import { + DOC_RESULT_CHAR_LIMIT, + TOOL_NAME_READ_DOC, + TOOL_NAME_SEARCH_DOCS, + createReadDocTool, + createSearchDocsTool, + getSharedDocsIndex, +} from "./docs-tools"; + +const DATASETS_PAGE = `--- +title: "Datasets" +description: "Upload and manage datasets" +--- + +A dataset is a versioned collection of files. + +## Uploading files + +Click the Upload button in the dataset page to add files. + +## Sharing a dataset + +Use the share dialog to grant read access to another user. +`; + +const LONG_SECTION_LENGTH = "word ".repeat(2000).trim().length; + +const LONG_PAGE = `--- +title: "Long Reference" +description: "A page with a long section" +--- + +## Big section + +${"word ".repeat(2000).trim()} +`; + +const CSV_PAGE = `--- +title: "CSVFileScan" +description: "Reads a CSV file into tuples" +--- + +## Properties + +The delimiter property sets the column separator. +`; + +const GETTING_STARTED_INDEX = `--- +title: "Getting Started" +description: "Install and launch Texera" +--- + +Follow the [install guide](install/) first. +`; + +const LOGIN_PAGE = `--- +title: "Access/Login to Texera" +description: "Sign in" +--- + +## Sign in + +Use the login form. +`; + +const EXACT_BODY = "e".repeat(DOC_RESULT_CHAR_LIMIT); + +const EXACT_PAGE = `--- +title: "Exact" +description: "A section exactly at the clip limit" +--- + +## At limit + +${EXACT_BODY} +`; + +let docsDir: string; +let index: DocsIndex; + +beforeAll(() => { + docsDir = mkdtempSync(join(tmpdir(), "texera-docs-tools-")); + mkdirSync(join(docsDir, "getting-started"), { recursive: true }); + mkdirSync(join(docsDir, "getting-started", "advanced"), { recursive: true }); + writeFileSync(join(docsDir, "getting-started", "_index.md"), GETTING_STARTED_INDEX); + writeFileSync(join(docsDir, "getting-started", "AccessLogin.md"), LOGIN_PAGE); + writeFileSync(join(docsDir, "getting-started", "advanced", "tuning.md"), CSV_PAGE); + writeFileSync(join(docsDir, "getting-started", "datasets.md"), DATASETS_PAGE); + writeFileSync(join(docsDir, "long.md"), LONG_PAGE); + writeFileSync(join(docsDir, "csv-file-scan.md"), CSV_PAGE); + writeFileSync(join(docsDir, "exact.md"), EXACT_PAGE); + index = loadDocsIndex(docsDir)!; +}); + +afterAll(() => { + rmSync(docsDir, { recursive: true, force: true }); +}); + +async function run(toolDef: any, args: Record): Promise { + return toolDef.execute(args, { toolCallId: "t1", messages: [] }); +} + +describe("tool names", () => { + test("are the names the prompt refers to", () => { + expect(TOOL_NAME_SEARCH_DOCS).toBe("searchDocs"); + expect(TOOL_NAME_READ_DOC).toBe("readDoc"); + }); +}); + +describe("searchDocs", () => { + test("lists each hit with heading chain, path, url and snippet", async () => { + const out = await run(createSearchDocsTool(index), { query: "upload a file" }); + expect(out.startsWith("[ERROR]")).toBe(false); + expect(out).toContain("Datasets > Uploading files"); + expect(out).toContain("path: getting-started/datasets"); + expect(out).toContain(`${DOCS_SITE_BASE}/getting-started/datasets/`); + expect(out).toContain("Click the Upload button"); + expect(out).toContain("readDoc"); + }); + + test("honours the limit argument", async () => { + const out = await run(createSearchDocsTool(index), { query: "dataset", limit: 1 }); + expect(out.match(/path: /g)).toHaveLength(1); + }); + + test("falls back to the outline of the documentation when nothing matches", async () => { + const out = await run(createSearchDocsTool(index), { query: "zzzz qqqq" }); + expect(out.startsWith("[ERROR]")).toBe(false); + expect(out).toContain('No section matched "zzzz qqqq".'); + expect(out).toContain("- long: Long Reference"); + expect(out).toContain("- getting-started/ (4 pages): Getting Started"); + expect(out).not.toContain("getting-started/datasets"); + expect(out).toContain("readDoc"); + }); + + test("falls back to the outline for an empty query", async () => { + const out = await run(createSearchDocsTool(index), { query: " " }); + expect(out).toContain('No section matched "".'); + expect(out).toContain("- getting-started/ (4 pages)"); + }); + + test("falls back to the outline for a stopword-only query", async () => { + const out = await run(createSearchDocsTool(index), { query: "how do I" }); + expect(out).toContain("No section matched"); + expect(out).not.toContain("path: "); + }); + + test("falls back to the outline when no section shares a term", async () => { + const out = await run(createSearchDocsTool(index), { query: "banana smoothie" }); + expect(out).toContain("No section matched"); + }); + + test("returns hits, not the outline, for a query with matching terms", async () => { + const out = await run(createSearchDocsTool(index), { query: "csv scan" }); + expect(out).not.toContain("No section matched"); + expect(out).toContain("path: csv-file-scan"); + }); + + describe("limit", () => { + const many = new DocsIndex( + Array.from({ length: 12 }, (_, i) => ({ + path: `p${i}`, + title: `P${i}`, + description: "", + body: `needle number ${i}`, + })) + ); + const hits = async (args: Record) => + ((await run(createSearchDocsTool(many), { query: "needle", ...args })).match(/path: /g) ?? []).length; + + test("returns five sections when no limit is given", async () => { + expect(await hits({})).toBe(5); + }); + + test("clamps a limit outside 1 to 10 instead of failing", async () => { + expect(await hits({ limit: 50 })).toBe(10); + expect(await hits({ limit: 10 })).toBe(10); + expect(await hits({ limit: 0 })).toBe(1); + expect(await hits({ limit: -3 })).toBe(1); + }); + + test("says so when it changed the limit, and nothing when it did not", async () => { + const note = (given: number, used: number) => + `(limit ${given} was changed to ${used}, it has to be a whole number from 1 to 10)`; + const out = (limit?: number) => + run(createSearchDocsTool(many), { query: "needle", ...(limit === undefined ? {} : { limit }) }); + expect((await out(50)).endsWith(`\n${note(50, 10)}`)).toBe(true); + expect((await out(0)).endsWith(`\n${note(0, 1)}`)).toBe(true); + expect((await out(2.9)).endsWith(`\n${note(2.9, 2)}`)).toBe(true); + for (const limit of [undefined, 1, 5, 10]) expect(await out(limit)).not.toContain("was changed to"); + }); + + test("rounds a fractional limit down and falls back to five for a value that is not a number", async () => { + expect(await hits({ limit: 2.9 })).toBe(2); + expect(await hits({ limit: Number.NaN })).toBe(5); + }); + }); + + test("returns an error result instead of throwing when the index fails", async () => { + const broken = { + search: () => { + throw new Error("boom"); + }, + } as unknown as DocsIndex; + expect(await run(createSearchDocsTool(broken), { query: "x" })).toBe("[ERROR] boom"); + }); + + test("echoes at most 80 characters of a query that matched nothing", async () => { + const out = await run(createSearchDocsTool(index), { query: "z".repeat(5000) }); + expect(out).toContain(`No section matched "${"z".repeat(80)}…".`); + expect(out.length).toBeLessThan(600); + }); + + test("keeps the fallback small for a documentation tree with hundreds of pages", async () => { + const pages = []; + for (const area of ["data-input", "data-cleaning", "visualization", "machine-learning", "search"]) { + pages.push({ path: `reference/operators/${area}/_index`, title: area, description: "", body: "x" }); + for (let i = 0; i < 60; i++) { + pages.push({ + path: `reference/operators/${area}/op-${i}`, + title: `Operator ${i}`, + description: "A long description", + body: "x", + }); + } + } + const big = new DocsIndex(pages); + const miss = await run(createSearchDocsTool(big), { query: "zzzz" }); + const unknown = await run(createReadDocTool(big), { path: "reference/operators/nope" }); + expect(miss.length).toBeLessThan(2000); + expect(unknown.length).toBeLessThan(2000); + expect(miss).toContain("- reference/operators/visualization/ (61 pages): visualization"); + }); +}); + +describe("readDoc", () => { + test("returns a section with its page link", async () => { + const out = await run(createReadDocTool(index), { path: "getting-started/datasets", section: "Sharing a dataset" }); + expect(out).toContain("Use the share dialog"); + expect(out).toContain(`${DOCS_SITE_BASE}/getting-started/datasets/`); + expect(out).not.toContain("truncated"); + }); + + test("returns the whole page without a section argument when it fits", async () => { + const out = await run(createReadDocTool(index), { path: "getting-started/datasets" }); + expect(out).toBe( + [ + `getting-started/datasets (${DOCS_SITE_BASE}/getting-started/datasets/)`, + "", + "A dataset is a versioned collection of files.", + "", + "## Uploading files", + "", + "Click the Upload button in the dataset page to add files.", + "", + "## Sharing a dataset", + "", + "Use the share dialog to grant read access to another user.", + ].join("\n") + ); + }); + + test("returns the section list and the introduction when the page is over the limit", async () => { + const out = await run(createReadDocTool(index), { path: "long" }); + expect(out).toBe( + `long (${DOCS_SITE_BASE}/long/)\n\nSections:\n- Long Reference\n- Big section\n\nA page with a long section` + ); + }); + + test("returns a page of exactly the limit whole and one character more as its introduction", async () => { + const page = (path: string, body: string) => ({ path, title: path, description: "", body }); + // "Intro." + blank line + "## S" + blank line + the body is 14 characters around the body. + const edge = new DocsIndex([ + page("fits", `Intro.\n\n## S\n${"y".repeat(DOC_RESULT_CHAR_LIMIT - 14)}`), + page("over", `Intro.\n\n## S\n${"y".repeat(DOC_RESULT_CHAR_LIMIT - 13)}`), + ]); + const fits = await run(createReadDocTool(edge), { path: "fits" }); + expect(fits.split("\n").slice(2).join("\n").length).toBe(DOC_RESULT_CHAR_LIMIT); + expect(fits).toContain("## S"); + expect(fits).not.toContain("Sections:"); + const over = await run(createReadDocTool(edge), { path: "over" }); + expect(over.endsWith("Sections:\n- over\n- S\n\nIntro.")).toBe(true); + }); + + test("treats a null section, offset or limit as absent", async () => { + const whole = await run(createReadDocTool(index), { path: "getting-started/datasets" }); + expect(await run(createReadDocTool(index), { path: "getting-started/datasets", section: null, offset: null })).toBe( + whole + ); + const section = await run(createReadDocTool(index), { path: "long", section: "Big section" }); + expect(await run(createReadDocTool(index), { path: "long", section: "Big section", offset: null })).toBe(section); + const hits = await run(createSearchDocsTool(index), { query: "dataset" }); + expect(await run(createSearchDocsTool(index), { query: "dataset", limit: null })).toBe(hits); + }); + + test("treats an empty section name as no section", async () => { + const whole = await run(createReadDocTool(index), { path: "getting-started/datasets" }); + expect(await run(createReadDocTool(index), { path: "getting-started/datasets", section: "" })).toBe(whole); + expect(await run(createReadDocTool(index), { path: "getting-started/datasets", section: " " })).toBe(whole); + }); + + test("finds a section by the heading chain that searchDocs prints", async () => { + const out = await run(createReadDocTool(index), { + path: "getting-started/datasets", + section: "Datasets > Sharing a dataset", + }); + expect(out).toContain("Use the share dialog"); + }); + + test("accepts a path with a trailing .md or slash", async () => { + const a = await run(createReadDocTool(index), { path: "getting-started/datasets.md" }); + const b = await run(createReadDocTool(index), { path: "/getting-started/datasets/" }); + expect(a).toContain("A dataset is a versioned collection of files."); + expect(b).toContain("A dataset is a versioned collection of files."); + }); + + test("matches the page path whatever its case and keeps the canonical case in the link", async () => { + const out = await run(createReadDocTool(index), { path: "Getting-Started/accesslogin", section: "sign in" }); + expect(out.startsWith(`getting-started/AccessLogin (${DOCS_SITE_BASE}/getting-started/AccessLogin/)`)).toBe(true); + expect(out).toContain("Use the login form."); + }); + + test("accepts the published link of a page as its path", async () => { + const out = await run(createReadDocTool(index), { path: `${DOCS_SITE_BASE}/getting-started/datasets/#uploading` }); + expect(out).toContain("A dataset is a versioned collection of files."); + }); + + test("reads a directory as its _index page followed by the pages under it", async () => { + for (const path of ["getting-started", "getting-started/", "getting-started/_index"]) { + const out = await run(createReadDocTool(index), { path }); + expect(out.startsWith(`getting-started/_index (${DOCS_SITE_BASE}/getting-started/)`)).toBe(true); + expect(out).toContain(`Follow the [install guide](${DOCS_SITE_BASE}/getting-started/install/) first.`); + expect(out).toContain( + [ + "Pages under getting-started/:", + "- getting-started/AccessLogin: Access/Login to Texera", + "- getting-started/datasets: Datasets", + "- getting-started/advanced/ (1 page)", + ].join("\n") + ); + } + }); + + test("lists a directory that has no _index page", async () => { + const out = await run(createReadDocTool(index), { path: "getting-started/advanced" }); + expect(out.startsWith("[ERROR]")).toBe(false); + expect(out).toBe("Pages under getting-started/advanced/:\n- getting-started/advanced/tuning: CSVFileScan"); + }); + + test("lists the top level for an empty path when there is no root page", async () => { + const out = await run(createReadDocTool(index), { path: "" }); + expect(out).toBe( + [ + "Pages under the documentation root:", + "- csv-file-scan: CSVFileScan", + "- exact: Exact", + "- long: Long Reference", + "- getting-started/ (4 pages): Getting Started", + ].join("\n") + ); + }); + + test("lists at most 50 entries of a directory and says how many are left", async () => { + const pages = Array.from({ length: 61 }, (_, i) => ({ + path: `ops/op-${String(i).padStart(2, "0")}`, + title: `Operator ${i}`, + description: "", + body: "x", + })); + const out = await run(createReadDocTool(new DocsIndex(pages)), { path: "ops" }); + const lines = out.split("\n"); + expect(lines).toHaveLength(52); + expect(lines[50]).toBe("- ops/op-49: Operator 49"); + expect(lines[51]).toBe("- and 11 more, use searchDocs to find them"); + }); + + test("does not append a page list when a section of an _index page is read", async () => { + const out = await run(createReadDocTool(index), { path: "getting-started", section: "Getting Started" }); + expect(out).toContain("Follow the [install guide]"); + expect(out).not.toContain("Pages under"); + }); + + test("clips a long section at the limit and says how to continue", async () => { + const out = await run(createReadDocTool(index), { path: "long", section: "Big section" }); + const body = out + .split("\n") + .filter(l => l.startsWith("word")) + .join("\n"); + expect(body.length).toBe(DOC_RESULT_CHAR_LIMIT); + const left = LONG_SECTION_LENGTH - DOC_RESULT_CHAR_LIMIT; + expect( + out.endsWith( + `[truncated, ${left} more characters. Call readDoc again with the same path and section and offset ${DOC_RESULT_CHAR_LIMIT} to continue.]` + ) + ).toBe(true); + }); + + test("continues a clipped section from the offset and stops clipping at its end", async () => { + const tool = createReadDocTool(index); + let offset = 0; + let text = ""; + for (let i = 0; i < 10; i++) { + const out = await run(tool, { path: "long", section: "Big section", ...(offset > 0 ? { offset } : {}) }); + const [, , ...rest] = out.split("\n"); + const more = rest[rest.length - 1].match(/^\[truncated, \d+ more characters\. .* offset (\d+) to continue\.\]$/); + text += (more ? rest.slice(0, -1) : rest).join("\n"); + if (!more) break; + offset = Number(more[1]); + } + expect(text).toBe("word ".repeat(2000).trim()); + }); + + test("cuts a long directory listing between two entries and continues with the next one", async () => { + const entry = /^- area\/page-\d\d: Page \d+ t{90}$/; + const area = new DocsIndex([ + { path: "area/_index", title: "Area", description: "", body: "Intro." }, + ...Array.from({ length: 50 }, (_, i) => ({ + path: `area/page-${String(i).padStart(2, "0")}`, + title: `Page ${i} ${"t".repeat(90)}`, + description: "", + body: "x", + })), + ]); + const tool = createReadDocTool(area); + const entries: string[] = []; + let offset = 0; + let calls = 0; + for (; calls < 10; calls++) { + const out = await run(tool, { path: "area", ...(offset > 0 ? { offset } : {}) }); + const lines = out.split("\n").slice(2); + const more = lines[lines.length - 1].match( + /^\[truncated, \d+ more characters\. .* offset (\d+) to continue\.\]$/ + ); + const body = more ? lines.slice(0, -1) : lines; + // A continuation starts with the next entry, not with the line break that was cut at. + if (offset > 0) expect(body[0]).toMatch(entry); + for (const line of body.filter(l => l.startsWith("- "))) { + expect(line).toMatch(entry); + entries.push(line); + } + if (!more) break; + offset = Number(more[1]); + } + expect(calls).toBeGreaterThan(0); + expect(entries).toHaveLength(50); + expect(new Set(entries).size).toBe(50); + }); + + test("cuts mid-line when the last line break is far from the limit", async () => { + const body = `first line\n${"w".repeat(DOC_RESULT_CHAR_LIMIT * 2)}`; + const wide = new DocsIndex([{ path: "w", title: "W", description: "", body: `## S\n${body}` }]); + const out = await run(createReadDocTool(wide), { path: "w", section: "S" }); + expect(out).toContain(`offset ${DOC_RESULT_CHAR_LIMIT} to continue.]`); + const rest = await run(createReadDocTool(wide), { path: "w", section: "S", offset: DOC_RESULT_CHAR_LIMIT }); + const shown = (text: string) => + text + .split("\n") + .slice(2) + .filter(l => !l.startsWith("[truncated")) + .join("\n"); + expect((shown(out) + shown(rest)).startsWith(body.slice(0, DOC_RESULT_CHAR_LIMIT * 2))).toBe(true); + }); + + test("says which arguments to repeat when a page read without a section is clipped", async () => { + const tall = new DocsIndex([{ path: "tall", title: "Tall", description: "", body: "intro ".repeat(2000) }]); + const out = await run(createReadDocTool(tall), { path: "tall" }); + expect(out).toMatch(/Call readDoc again with the same path and offset 4000 to continue\.\]$/); + const rest = await run(createReadDocTool(tall), { path: "tall", offset: 4000 }); + expect(rest).toMatch(/Call readDoc again with the same path and offset 8000 to continue\.\]$/); + }); + + test("rejects an offset at or past the end of the text", async () => { + const out = await run(createReadDocTool(index), { + path: "getting-started/datasets", + section: "Sharing a dataset", + offset: 5000, + }); + expect(out.startsWith("[ERROR]")).toBe(true); + expect(out).toContain("offset 5000 is past the end"); + const atEnd = await run(createReadDocTool(index), { + path: "exact", + section: "At limit", + offset: DOC_RESULT_CHAR_LIMIT, + }); + expect(atEnd.startsWith("[ERROR]")).toBe(true); + }); + + test("reads the tail of a section from an offset inside it", async () => { + const out = await run(createReadDocTool(index), { + path: "exact", + section: "At limit", + offset: DOC_RESULT_CHAR_LIMIT - 3, + }); + expect(out.endsWith("\n\neee")).toBe(true); + expect(out).not.toContain("truncated"); + }); + + test("leaves a section exactly at the limit unclipped", async () => { + const out = await run(createReadDocTool(index), { path: "exact", section: "At limit" }); + expect(out).toContain(EXACT_BODY); + expect(out).not.toContain("truncated"); + }); + + test("errors on an unknown page with the closest pages and the outline", async () => { + const out = await run(createReadDocTool(index), { path: "getting-started/login" }); + expect( + out.startsWith( + '[ERROR] Unknown page "getting-started/login". Closest pages:\n- getting-started/AccessLogin: Access/Login to Texera' + ) + ).toBe(true); + expect(out).toContain("- getting-started/ (4 pages): Getting Started"); + expect(out).toContain("- long: Long Reference"); + }); + + test("errors on an unknown page with only the outline when no page name is close", async () => { + const out = await run(createReadDocTool(index), { path: "../../../../etc/passwd" }); + expect(out.startsWith('[ERROR] Unknown page "../../../../etc/passwd".')).toBe(true); + expect(out).not.toContain("Closest pages"); + expect(out).toContain("- getting-started/ (4 pages)"); + }); + + test("echoes at most 80 characters of an unknown path", async () => { + const out = await run(createReadDocTool(index), { path: "q".repeat(3000) }); + expect(out).toContain(`Unknown page "${"q".repeat(80)}…".`); + expect(out.length).toBeLessThan(600); + }); + + test("errors on an unknown section and lists the page's sections", async () => { + const out = await run(createReadDocTool(index), { path: "getting-started/datasets", section: "Nope" }); + expect(out).toBe( + '[ERROR] Unknown section "Nope" on page "getting-started/datasets". Available sections:\n- Datasets\n- Uploading files\n- Sharing a dataset' + ); + }); + + test("rejects an offset that is negative or not a whole number", async () => { + for (const offset of [-5, -1, 1.5, Number.NaN]) { + const out = await run(createReadDocTool(index), { path: "long", section: "Big section", offset }); + expect(out).toBe("[ERROR] offset must be a whole number of characters, 0 or more."); + } + }); + + test("does not split an emoji at the clip limit and continues after it", async () => { + const body = `${"x".repeat(DOC_RESULT_CHAR_LIMIT - 1)}📚tail`; + const emoji = new DocsIndex([{ path: "e", title: "E", description: "", body: `## S\n${body}` }]); + const first = await run(createReadDocTool(emoji), { path: "e", section: "S" }); + const [, , chunk, trailer] = first.split("\n"); + expect(chunk).toBe("x".repeat(DOC_RESULT_CHAR_LIMIT - 1)); + expect(trailer).toBe( + `[truncated, 6 more characters. Call readDoc again with the same path and section and offset ${DOC_RESULT_CHAR_LIMIT - 1} to continue.]` + ); + const rest = await run(createReadDocTool(emoji), { path: "e", section: "S", offset: DOC_RESULT_CHAR_LIMIT - 1 }); + expect(rest.endsWith("\n\n📚tail")).toBe(true); + // An offset that points into the middle of the emoji starts at the emoji. + const inside = await run(createReadDocTool(emoji), { path: "e", section: "S", offset: DOC_RESULT_CHAR_LIMIT }); + expect(inside.endsWith("\n\n📚tail")).toBe(true); + }); + + test("returns an error result instead of throwing when the index fails", async () => { + const broken = { + resolve: () => { + throw new Error("boom"); + }, + } as unknown as DocsIndex; + expect(await run(createReadDocTool(broken), { path: "x" })).toBe("[ERROR] boom"); + const plain = { + resolve: () => { + throw "plain failure"; + }, + } as unknown as DocsIndex; + expect(await run(createReadDocTool(plain), { path: "x" })).toBe("[ERROR] plain failure"); + }); + + test("says in its description what a call without a section returns", () => { + const description = (createReadDocTool(index) as any).description as string; + expect(description).toContain("returns the whole page when it is short"); + expect(description).toContain("offset"); + }); +}); + +describe("input schemas", () => { + const accepts = (toolDef: any, input: unknown): boolean => toolDef.inputSchema.safeParse(input).success; + + test("searchDocs accepts any number as the limit, so a bad one reaches the tool and is clamped", () => { + const search = createSearchDocsTool(index); + for (const limit of [1, 10, 50, 0, -1, 2.5]) expect(accepts(search, { query: "x", limit })).toBe(true); + expect(accepts(search, { query: "x" })).toBe(true); + }); + + test("searchDocs still requires a query string and a numeric limit", () => { + const search = createSearchDocsTool(index); + expect(accepts(search, {})).toBe(false); + expect(accepts(search, { query: 5 })).toBe(false); + expect(accepts(search, { query: "x", limit: "5" })).toBe(false); + }); + + test("readDoc accepts any number as the offset, so a bad one comes back as tool output", () => { + const read = createReadDocTool(index); + for (const offset of [0, 4000, -1, 1.5]) expect(accepts(read, { path: "p", offset })).toBe(true); + expect(accepts(read, { path: "p", section: "S" })).toBe(true); + }); + + test("both accept null for an optional argument, which some models send instead of leaving it out", () => { + expect(accepts(createSearchDocsTool(index), { query: "x", limit: null })).toBe(true); + expect(accepts(createReadDocTool(index), { path: "p", section: null, offset: null })).toBe(true); + expect(accepts(createSearchDocsTool(index), { query: null })).toBe(false); + expect(accepts(createReadDocTool(index), { path: null })).toBe(false); + }); + + test("readDoc still requires a path string and a numeric offset", () => { + const read = createReadDocTool(index); + expect(accepts(read, {})).toBe(false); + expect(accepts(read, { path: "p", offset: "4000" })).toBe(false); + expect(accepts(read, { path: "p", section: 3 })).toBe(false); + }); +}); + +describe("getSharedDocsIndex — logging and failures", () => { + const fakeLog = () => { + const calls: { level: string; fields: any; message: string }[] = []; + const log = { + info: (fields: any, message: string) => calls.push({ level: "info", fields, message }), + warn: (fields: any, message: string) => calls.push({ level: "warn", fields, message }), + }; + return { log: log as any, calls }; + }; + const scratch: string[] = []; + const tempDir = (): string => { + const dir = mkdtempSync(join(tmpdir(), "texera-docs-shared-")); + scratch.push(dir); + return dir; + }; + + afterAll(() => { + for (const dir of scratch) rmSync(dir, { recursive: true, force: true }); + }); + + test("logs the page count once when an index loads and reuses the index", () => { + const dir = tempDir(); + writeFileSync(join(dir, "a.md"), "# A\ntext"); + writeFileSync(join(dir, "b.md"), "# B\ntext"); + const { log, calls } = fakeLog(); + const first = getSharedDocsIndex(dir, log); + const second = getSharedDocsIndex(dir, log); + expect(first).not.toBeNull(); + expect(second).toBe(first); + expect(calls).toEqual([{ level: "info", fields: { dir, pages: 2 }, message: "docs index loaded" }]); + }); + + test("warns once and returns null for a missing directory", () => { + const dir = join(tempDir(), "missing"); + const { log, calls } = fakeLog(); + expect(getSharedDocsIndex(dir, log)).toBeNull(); + expect(getSharedDocsIndex(dir, log)).toBeNull(); + expect(calls).toEqual([ + { + level: "warn", + fields: { dir }, + message: "docs directory not found, docs tools stay off until the service restarts", + }, + ]); + }); + + test("warns and returns null for a directory without a markdown page", () => { + const dir = tempDir(); + writeFileSync(join(dir, "notes.txt"), "not markdown"); + const { log, calls } = fakeLog(); + expect(getSharedDocsIndex(dir, log)).toBeNull(); + expect(calls.map(c => c.message)).toEqual([ + "no documentation pages in the docs directory, docs tools stay off until the service restarts", + ]); + }); + + test("catches a load error, warns once with it and keeps returning null", () => { + const dir = tempDir(); + writeFileSync(join(dir, "a.md"), "# A\ntext"); + symlinkSync(join(dir, "does-not-exist"), join(dir, "dangling.md")); + const { log, calls } = fakeLog(); + expect(getSharedDocsIndex(dir, log)).toBeNull(); + expect(calls).toHaveLength(1); + expect(calls[0].level).toBe("warn"); + expect(calls[0].message).toBe("failed to load the docs index, docs tools stay off until the service restarts"); + expect(calls[0].fields.dir).toBe(dir); + expect(String(calls[0].fields.err)).toContain("ENOENT"); + + // The outcome is cached: fixing the folder later does not bring the tools back in this process. + rmSync(join(dir, "dangling.md")); + expect(getSharedDocsIndex(dir, log)).toBeNull(); + expect(calls).toHaveLength(1); + }); +}); + +describe("getSharedDocsIndex", () => { + test("returns the same index for the same directory and null for a missing one", () => { + const a = getSharedDocsIndex(docsDir); + const b = getSharedDocsIndex(docsDir); + expect(a).not.toBeNull(); + expect(a).toBe(b); + expect(getSharedDocsIndex(join(docsDir, "missing"))).toBeNull(); + }); +}); diff --git a/agent-service/src/agent/tools/docs-tools.ts b/agent-service/src/agent/tools/docs-tools.ts new file mode 100644 index 00000000000..8b6b9e3ca27 --- /dev/null +++ b/agent-service/src/agent/tools/docs-tools.ts @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { z } from "zod"; +import { tool } from "ai"; +import type { Logger } from "pino"; +import { createErrorResult, createToolResult } from "./tools-utility"; +import { existsSync } from "node:fs"; +import { cutText, loadDocsIndex, type DocsIndex } from "./docs-index"; +import { createLogger } from "../../logger"; + +export const TOOL_NAME_SEARCH_DOCS = "searchDocs"; +export const TOOL_NAME_READ_DOC = "readDoc"; + +/** Largest readDoc output; the rest is read by calling again with the offset the last line gives. */ +export const DOC_RESULT_CHAR_LIMIT = 4000; +const DEFAULT_SEARCH_LIMIT = 5; +const MAX_SEARCH_LIMIT = 10; +/** Most characters of a query or path repeated back in a "nothing found" message. */ +const ECHO_CHAR_LIMIT = 80; +/** A clip ends at the last line break when one lies this close to the limit, so no line is cut in two. */ +const LINE_CUT_WINDOW = 500; +/** Most entries listed for one directory. */ +const DIRECTORY_LISTING_LIMIT = 50; +const BROWSE_HINT = "call readDoc with one of these paths to browse that area"; +const TOOLS_OFF = "docs tools stay off until the service restarts"; + +const moduleLog = createLogger("DocsTools"); +const sharedIndexes = new Map(); + +/** + * Loads the index for a directory once per process and reuses it for every agent. A directory + * that is missing, has no page or cannot be read gives null, logged once. + */ +export function getSharedDocsIndex(dir: string, log: Pick = moduleLog): DocsIndex | null { + if (!sharedIndexes.has(dir)) { + let index: DocsIndex | null = null; + try { + index = loadDocsIndex(dir); + if (index) log.info({ dir, pages: index.pages().length }, "docs index loaded"); + else if (!existsSync(dir)) log.warn({ dir }, `docs directory not found, ${TOOLS_OFF}`); + else log.warn({ dir }, `no documentation pages in the docs directory, ${TOOLS_OFF}`); + } catch (error) { + log.warn({ dir, err: error }, `failed to load the docs index, ${TOOLS_OFF}`); + } + sharedIndexes.set(dir, index); + } + return sharedIndexes.get(dir) ?? null; +} + +function echo(text: string): string { + const trimmed = text.trim(); + return trimmed.length <= ECHO_CHAR_LIMIT ? trimmed : `${cutText(trimmed, ECHO_CHAR_LIMIT)}…`; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** The part of text from offset on, clipped at the limit with a line that says how to read on. */ +function clip(text: string, offset: number, hasSection: boolean): string { + // An offset inside a surrogate pair starts at the pair. + const code = text.charCodeAt(offset); + const start = offset > 0 && code >= 0xdc00 && code <= 0xdfff ? offset - 1 : offset; + const rest = text.slice(start); + if (rest.length <= DOC_RESULT_CHAR_LIMIT) return rest; + let shown = cutText(rest, DOC_RESULT_CHAR_LIMIT); + let next = start + shown.length; + const lineEnd = shown.lastIndexOf("\n"); + if (lineEnd >= DOC_RESULT_CHAR_LIMIT - LINE_CUT_WINDOW) { + shown = shown.slice(0, lineEnd); + next = start + lineEnd + 1; + } + const same = hasSection ? "the same path and section" : "the same path"; + return `${shown}\n[truncated, ${text.length - next} more characters. Call readDoc again with ${same} and offset ${next} to continue.]`; +} + +/** The pages and sub-directories directly under a directory, or undefined when it holds none. */ +function directoryListing(index: DocsIndex, dir: string): string | undefined { + const { pages, dirs } = index.children(dir); + const entries = [ + ...pages.map(page => `- ${page.path}: ${page.title}`), + ...dirs.map(sub => { + const count = sub.pageCount === 1 ? "1 page" : `${sub.pageCount} pages`; + return `- ${sub.path}/ (${count})${sub.title ? `: ${sub.title}` : ""}`; + }), + ]; + if (entries.length === 0) return undefined; + const shown = entries.slice(0, DIRECTORY_LISTING_LIMIT); + if (entries.length > shown.length) { + shown.push(`- and ${entries.length - shown.length} more, use searchDocs to find them`); + } + const name = dir === "" ? "the documentation root" : `${dir}/`; + return `Pages under ${name}:\n${shown.join("\n")}`; +} + +function unknownPageResult(index: DocsIndex, path: string): string { + const close = index.closest(path).map(page => `- ${page.path}: ${page.title}`); + const closest = + close.length > 0 ? ` Closest pages:\n${close.join("\n")}\nOr ${BROWSE_HINT}:` : ` To find a page, ${BROWSE_HINT}:`; + return createErrorResult(`Unknown page "${echo(path)}".${closest}\n${index.outline()}`); +} + +export function createSearchDocsTool(index: DocsIndex) { + return tool({ + description: + "Search the Texera documentation by keywords. Returns the best matching sections with the page path to pass to readDoc and the link to the published page. When nothing matches, returns a short outline of the documentation areas instead.", + inputSchema: z.object({ + query: z + .string() + .describe("Keywords describing what to look up, e.g. 'upload dataset' or 'CSVFileScan delimiter'"), + // Any number is accepted and clamped, so a value out of range still gets an answer. Null counts as absent. + limit: z.number().nullish().describe("Maximum number of sections to return, 1 to 10, 5 by default"), + }), + execute: async (args: { query: string; limit?: number | null }) => { + try { + const given = typeof args.limit === "number" && Number.isFinite(args.limit) ? args.limit : undefined; + const limit = + given === undefined ? DEFAULT_SEARCH_LIMIT : Math.min(MAX_SEARCH_LIMIT, Math.max(1, Math.trunc(given))); + // No indexable term or no section sharing one: an outline to browse is more useful than nothing. + const hits = index.search(args.query, limit); + if (hits.length === 0) { + return createToolResult( + `No section matched "${echo(args.query)}". Try other keywords, or ${BROWSE_HINT}:\n${index.outline()}` + ); + } + const lines = hits.map( + (h, i) => `${i + 1}. ${h.headingChain.join(" > ")}\n path: ${h.path}\n link: ${h.url}\n ${h.snippet}` + ); + const changed = + given === undefined || given === limit + ? "" + : `\n(limit ${given} was changed to ${limit}, it has to be a whole number from 1 to ${MAX_SEARCH_LIMIT})`; + return createToolResult( + `${lines.join("\n")}\n\nCall readDoc with a path to read the page, or with a path and a section name for one section.${changed}` + ); + } catch (error) { + return createErrorResult(errorText(error)); + } + }, + }); +} + +export function createReadDocTool(index: DocsIndex) { + return tool({ + description: + "Read a Texera documentation page. Without a section name it returns the whole page when it is short, otherwise its section list and introduction, and for a directory also the pages under it. With a section name it returns that section. Output is clipped; its last line then gives the offset to continue from.", + inputSchema: z.object({ + path: z + .string() + .describe( + "Page or directory path as returned by searchDocs, e.g. 'getting-started/ui-overview' or 'tutorials'" + ), + section: z + .string() + .nullish() + .describe("Section name to read, as listed by searchDocs or by readDoc without a section"), + // Any number is accepted and checked in the tool, so a bad one comes back as tool output. Null counts as absent. + offset: z + .number() + .nullish() + .describe("Character offset to continue a clipped result from, as given in its last line"), + }), + execute: async (args: { path: string; section?: string | null; offset?: number | null }) => { + try { + const offset = args.offset ?? 0; + if (!Number.isInteger(offset) || offset < 0) { + return createErrorResult("offset must be a whole number of characters, 0 or more."); + } + const section = typeof args.section === "string" && args.section.trim() !== "" ? args.section : undefined; + const path = index.resolve(args.path); + if (path === undefined) { + // A directory without an _index page still lists what is under it. + const dir = section === undefined ? index.directory(args.path) : undefined; + const listing = dir === undefined ? undefined : directoryListing(index, dir); + return listing === undefined ? unknownPageResult(index, args.path) : createToolResult(listing); + } + let text: string | undefined; + if (section === undefined) { + const whole = index.page(path) ?? ""; + text = whole.length <= DOC_RESULT_CHAR_LIMIT ? whole : index.read(path); + if (text !== undefined && (path === "_index" || path.endsWith("/_index"))) { + const listing = directoryListing(index, path === "_index" ? "" : path.slice(0, -"/_index".length)); + if (listing !== undefined) text = `${text}\n\n${listing}`; + } + } else { + text = index.read(path, section); + } + if (text === undefined) { + const names = index.sections(path).map(name => `- ${name}`); + return createErrorResult( + `Unknown section "${echo(section ?? "")}" on page "${path}". Available sections:\n${names.join("\n")}` + ); + } + if (offset > 0 && offset >= text.length) { + return createErrorResult(`offset ${offset} is past the end of this text (${text.length} characters).`); + } + return createToolResult(`${path} (${index.url(path)})\n\n${clip(text, offset, section !== undefined)}`); + } catch (error) { + return createErrorResult(errorText(error)); + } + }, + }); +} diff --git a/agent-service/src/agent/tools/index.ts b/agent-service/src/agent/tools/index.ts index 7e2d9570703..18bb88d3370 100644 --- a/agent-service/src/agent/tools/index.ts +++ b/agent-service/src/agent/tools/index.ts @@ -20,3 +20,5 @@ export * from "./tools-utility"; export * from "./workflow-crud-tools"; export * from "./workflow-execution-tools"; +export * from "./docs-index"; +export * from "./docs-tools"; diff --git a/agent-service/src/agent/util/context-utils.spec.ts b/agent-service/src/agent/util/context-utils.spec.ts index 25f487f610e..c0fcde8c512 100644 --- a/agent-service/src/agent/util/context-utils.spec.ts +++ b/agent-service/src/agent/util/context-utils.spec.ts @@ -19,7 +19,7 @@ import { describe, expect, test } from "bun:test"; import type { ModelMessage } from "ai"; -import { assembleContext } from "./context-utils"; +import { assembleContext, DOCS_CONTEXT_CHAR_LIMIT } from "./context-utils"; import { WorkflowState } from "../workflow-state"; import type { ReActStep } from "../../types/agent"; import type { OperatorLink, OperatorPredicate } from "../../types/workflow"; @@ -136,6 +136,309 @@ describe("assembleContext", () => { }); }); +describe("assembleContext — documentation output", () => { + type Call = { tool: string; input: unknown; output?: unknown; isError?: boolean }; + + /** An agent step whose tool calls carry the given outputs; a call without `output` has no result yet. */ + function toolStep(messageId: string, stepId: number, calls: Call[], isEnd = false): ReActStep { + return { + ...step(messageId, "agent", stepId, `turn ${stepId}`, isEnd), + toolCalls: calls.map((c, i) => ({ toolName: c.tool, toolCallId: `${messageId}-${stepId}-${i}`, input: c.input })), + toolResults: calls.flatMap((c, i) => + c.output === undefined + ? [] + : [{ toolCallId: `${messageId}-${stepId}-${i}`, output: c.output, ...(c.isError ? { isError: true } : {}) }] + ), + }; + } + + const dropped = (n: number) => + `(${n} earlier output${n === 1 ? "" : "s"} dropped to save space, call the tool again if you need ${n === 1 ? "it" : "them"})`; + const REJECTED = "(no output, the tool is unavailable or its arguments were invalid)"; + /** What one entry costs against the budget: its label, its indented output and the blank line before it. */ + const cost = (label: string, output: string) => + label.length + + 1 + + output + .split("\n") + .map(l => ` ${l}`) + .join("\n").length + + 2; + const readLabel = (turn: number, path: string) => `## Turn ${turn}: readDoc {"path":"${path}"}`; + /** A one-line output sized so that its entry costs exactly `total`. */ + const sized = (turn: number, path: string, total: number, fill = "x") => + fill.repeat(total - cost(readLabel(turn, path), "")); + const sectionOf = (content: string) => content.slice(content.indexOf("# Documentation Read")); + const HEADER = "# Documentation Read\nOutput of your searchDocs and readDoc calls in this task, oldest first."; + const search = (query: string, output: unknown): Call => ({ tool: "searchDocs", input: { query }, output }); + const read = (path: string, output: unknown): Call => ({ tool: "readDoc", input: { path }, output }); + + test("renders the output of a docs call of the ongoing task under its turn, tool and input", () => { + const steps = [ + step("m1", "user", 0, "how do I upload a dataset", true), + toolStep("m1", 1, [search("upload dataset", "1. Datasets > Uploading files\n path: getting-started/datasets")]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain( + `${HEADER}\n\n## Turn 1: searchDocs {"query":"upload dataset"}\n 1. Datasets > Uploading files\n path: getting-started/datasets` + ); + }); + + test("places the section after the ongoing task and before the dataflow", () => { + const workflowState = new WorkflowState(); + workflowState.addOperator(makeOperator("op1")); + const steps = [step("m1", "user", 0, "q", true), toolStep("m1", 1, [search("x", "found")])]; + const content = contentOf(assembleContext(steps, workflowState, new Map())); + const ongoing = content.indexOf("# Ongoing Task"); + const docs = content.indexOf("# Documentation Read"); + const dataflow = content.indexOf("# Current Dataflow"); + expect(ongoing).toBeGreaterThanOrEqual(0); + expect(docs).toBeGreaterThan(ongoing); + expect(dataflow).toBeGreaterThan(docs); + }); + + test("lists several outputs oldest first, one entry per call", () => { + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [search("a", "search output")]), + toolStep("m1", 2, [read("p/one", "first page"), read("p/two", "second page")]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + const order = [ + "## Turn 1: searchDocs", + '## Turn 2: readDoc {"path":"p/one"}', + '## Turn 2: readDoc {"path":"p/two"}', + ]; + const positions = order.map(label => content.indexOf(label)); + expect(positions.every(p => p >= 0)).toBe(true); + expect([...positions].sort((a, b) => a - b)).toEqual(positions); + expect(content).toContain(" first page"); + expect(content).toContain(" second page"); + }); + + test("includes a failed docs call, since its text says what to do next", () => { + // A tool that fails returns a string starting with [ERROR]; the stored result is not flagged. + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [{ tool: "readDoc", input: { path: "nope" }, output: '[ERROR] Unknown page "nope".' }]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain('## Turn 1: readDoc {"path":"nope"}\n [ERROR] Unknown page "nope".'); + }); + + test("renders no section when the ongoing task made no docs call", () => { + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [{ tool: "addOperator", input: { operatorId: "op1" }, output: "Added operator op1" }]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).not.toContain("# Documentation Read"); + expect(content).not.toContain("Added operator op1"); + }); + + test("says that a docs call without a result was rejected, so the model can correct it", () => { + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [{ tool: "searchDocs", input: { limit: "five" } }]), + toolStep("m1", 2, [search("upload", "found it")]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain(`## Turn 1: searchDocs {"limit":"five"}\n ${REJECTED}`); + expect(content).toContain('## Turn 2: searchDocs {"query":"upload"}\n found it'); + }); + + test("shows the section for a rejected docs call even when it is the only one", () => { + const steps = [step("m1", "user", 0, "q", true), toolStep("m1", 1, [{ tool: "readDoc", input: {} }])]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain("# Documentation Read"); + expect(content).toContain(`## Turn 1: readDoc {}\n ${REJECTED}`); + }); + + test("renders a result without output as having none instead of failing", () => { + const withUndefined: ReActStep = { + ...step("m1", "agent", 1, "t", false), + toolCalls: [{ toolName: "searchDocs", toolCallId: "a", input: { query: "x" } }], + toolResults: [{ toolCallId: "a", output: undefined }], + }; + const content = contentOf( + assembleContext([step("m1", "user", 0, "q", true), withUndefined], new WorkflowState(), new Map()) + ); + expect(content).toContain('## Turn 1: searchDocs {"query":"x"}\n (no output)'); + }); + + test("does not report a rejected call of another tool in the docs section", () => { + const steps = [step("m1", "user", 0, "q", true), toolStep("m1", 1, [{ tool: "addOperator", input: {} }])]; + expect(contentOf(assembleContext(steps, new WorkflowState(), new Map()))).not.toContain("# Documentation Read"); + }); + + test("drops the docs output of a completed task", () => { + const done = [ + step("m1", "user", 0, "first question", true), + toolStep("m1", 1, [search("old", "OLD SEARCH OUTPUT")]), + step("m1", "agent", 2, "the answer", true), + ]; + const alone = contentOf(assembleContext(done, new WorkflowState(), new Map())); + expect(alone).not.toContain("# Documentation Read"); + expect(alone).not.toContain("OLD SEARCH OUTPUT"); + + const next = [ + ...done, + step("m2", "user", 0, "second question", true), + toolStep("m2", 1, [search("new", "NEW OUTPUT")]), + ]; + const content = contentOf(assembleContext(next, new WorkflowState(), new Map())); + expect(content).toContain(" NEW OUTPUT"); + expect(content).not.toContain("OLD SEARCH OUTPUT"); + expect((content.match(/## Turn \d+: searchDocs/g) ?? []).length).toBe(1); + }); + + test("takes the docs output of the latest task only when an earlier one was left unfinished", () => { + const steps = [ + step("m1", "user", 0, "stopped question", true), + toolStep("m1", 1, [search("old", "STOPPED TASK OUTPUT")]), + step("m2", "user", 0, "new question", true), + toolStep("m2", 1, [search("new", "LATEST OUTPUT")]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain(" LATEST OUTPUT"); + expect(content).not.toContain("STOPPED TASK OUTPUT"); + expect((content.match(/# Documentation Read/g) ?? []).length).toBe(1); + }); + + test("matches an output to its call by id, not by position", () => { + const stepWithSwappedResults: ReActStep = { + ...step("m1", "agent", 1, "t", false), + toolCalls: [ + { toolName: "searchDocs", toolCallId: "a", input: { query: "alpha" } }, + { toolName: "readDoc", toolCallId: "b", input: { path: "beta" } }, + ], + toolResults: [ + { toolCallId: "b", output: "BETA PAGE" }, + { toolCallId: "a", output: "ALPHA HITS" }, + ], + }; + const content = contentOf( + assembleContext([step("m1", "user", 0, "q", true), stepWithSwappedResults], new WorkflowState(), new Map()) + ); + expect(content).toContain('## Turn 1: searchDocs {"query":"alpha"}\n ALPHA HITS'); + expect(content).toContain('## Turn 1: readDoc {"path":"beta"}\n BETA PAGE'); + }); + + test("JSON-encodes an output that is not a string and shortens a very long input", () => { + const steps = [step("m1", "user", 0, "q", true), toolStep("m1", 1, [search("q".repeat(500), { hits: 2 })])]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain('\n {"hits":2}'); + const label = content.split("\n").find(l => l.startsWith("## Turn 1: searchDocs"))!; + expect(label.length).toBeLessThan(260); + expect(label.endsWith("…")).toBe(true); + }); + + test("keeps every output while the entries together cost exactly the budget", () => { + const half = DOCS_CONTEXT_CHAR_LIMIT / 2; + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [read("a", sized(1, "a", half, "a"))]), + toolStep("m1", 2, [read("b", sized(2, "b", half, "b"))]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).not.toContain("dropped to save space"); + expect(content).toContain(`${readLabel(1, "a")}\n aaaa`); + expect(content).toContain(`${readLabel(2, "b")}\n bbbb`); + }); + + test("drops the oldest output when the entries cost one character more than the budget", () => { + const half = DOCS_CONTEXT_CHAR_LIMIT / 2; + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [read("a", sized(1, "a", half, "a"))]), + toolStep("m1", 2, [read("b", sized(2, "b", half + 1, "b"))]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain(`oldest first.\n${dropped(1)}\n\n${readLabel(2, "b")}\n bbbb`); + expect(content).not.toContain(readLabel(1, "a")); + expect(content).not.toContain("aaaa"); + }); + + test("over the budget, keeps the newest outputs whole and sums up the older ones in one line", () => { + const big = DOCS_CONTEXT_CHAR_LIMIT / 2; + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [read("a", sized(1, "a", big, "a"))]), + toolStep("m1", 2, [read("b", sized(2, "b", big + 1, "b"))]), + toolStep("m1", 3, [read("c", sized(3, "c", big, "c"))]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain(`oldest first.\n${dropped(2)}\n\n${readLabel(3, "c")}\n cccc`); + expect((content.match(/dropped to save space/g) ?? []).length).toBe(1); + expect(content).not.toContain(readLabel(1, "a")); + expect(content).not.toContain(readLabel(2, "b")); + expect(content).not.toContain("aaaa"); + expect(content).not.toContain("bbbb"); + }); + + test("does not bring back an older output that would fit once a newer one was dropped", () => { + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [read("small", "tiny")]), + toolStep("m1", 2, [read("huge", "h".repeat(DOCS_CONTEXT_CHAR_LIMIT))]), + toolStep("m1", 3, [read("last", "newest")]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain(dropped(2)); + expect(content).toContain(" newest"); + expect(content).not.toContain("tiny"); + expect(content).not.toContain("hhhh"); + }); + + test("always keeps the newest output, even when it alone is over the budget", () => { + const huge = "z".repeat(DOCS_CONTEXT_CHAR_LIMIT + 10); + const steps = [step("m1", "user", 0, "q", true), toolStep("m1", 1, [read("huge", huge)])]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain(huge); + expect(content).not.toContain("dropped to save space"); + }); + + test("keeps the whole section within the budget plus its two fixed lines, however many calls were made", () => { + const bound = DOCS_CONTEXT_CHAR_LIMIT + 200; + // 60 reads of a clipped section: about 4,200 characters over a hundred lines each. + const page = Array.from({ length: 100 }, (_, i) => `line ${i} ${"t".repeat(34)}`).join("\n"); + const reads = [ + step("m1", "user", 0, "q", true), + ...Array.from({ length: 60 }, (_, i) => toolStep("m1", i + 1, [read(`page-${i}`, page)])), + ]; + const readSection = sectionOf(contentOf(assembleContext(reads, new WorkflowState(), new Map()))); + expect(readSection.length).toBeLessThanOrEqual(bound); + expect(readSection).toContain(" line 99 "); + + // 60 searches without a hit, each with a long query and the outline as output. + const outline = Array.from({ length: 26 }, (_, i) => `- area-${i}/ (12 pages): Area ${i} of the docs`).join("\n"); + const searches = [ + step("m1", "user", 0, "q", true), + ...Array.from({ length: 60 }, (_, i) => toolStep("m1", i + 1, [search(`${i} ${"q".repeat(400)}`, outline)])), + ]; + const searchSection = sectionOf(contentOf(assembleContext(searches, new WorkflowState(), new Map()))); + expect(searchSection.length).toBeLessThanOrEqual(bound); + expect(searchSection).toMatch( + /\(\d+ earlier outputs dropped to save space, call the tool again if you need them\)/ + ); + }); + + test("counts the label and the indentation of an entry against the budget", () => { + // Forty lines of five hundred characters: the output alone is under the budget, its entry is not. + const lines = Array.from({ length: 40 }, () => "y".repeat(499)).join("\n"); + expect(lines.length).toBeLessThan(DOCS_CONTEXT_CHAR_LIMIT); + expect(cost(readLabel(2, "b"), lines) + cost(readLabel(1, "a"), "small")).toBeGreaterThan(DOCS_CONTEXT_CHAR_LIMIT); + const steps = [ + step("m1", "user", 0, "q", true), + toolStep("m1", 1, [read("a", "small")]), + toolStep("m1", 2, [read("b", lines)]), + ]; + const content = contentOf(assembleContext(steps, new WorkflowState(), new Map())); + expect(content).toContain(dropped(1)); + expect(content).not.toContain(" small"); + }); +}); + describe("assembleContext — dataflow serialization", () => { test("orders operators and links topologically rather than by insertion order", () => { const workflowState = new WorkflowState(); diff --git a/agent-service/src/agent/util/context-utils.ts b/agent-service/src/agent/util/context-utils.ts index 195692cbf50..8f4a66ae83d 100644 --- a/agent-service/src/agent/util/context-utils.ts +++ b/agent-service/src/agent/util/context-utils.ts @@ -27,10 +27,20 @@ import type { OperatorPredicate, OperatorPortSchemaMap, PortSchema } from "../.. import type { ReActStep } from "../../types/agent"; import type { WorkflowCompilationResponse, WorkflowFatalError } from "../../api/compile-api"; import { extractOperatorInputPortSchemaMap } from "./workflow-utils"; +import { TOOL_NAME_READ_DOC, TOOL_NAME_SEARCH_DOCS } from "../tools/docs-tools"; +import { cutText } from "../tools/docs-index"; import { createLogger } from "../../logger"; const log = createLogger("ContextAssembler"); +/** Tools whose output is text for the model to read, not a change that shows in the dataflow. */ +const DOCS_TOOL_NAMES = new Set([TOOL_NAME_SEARCH_DOCS, TOOL_NAME_READ_DOC]); +/** Most characters the docs calls shown in one context may take, labels and indentation included. */ +export const DOCS_CONTEXT_CHAR_LIMIT = 20000; +const DOCS_INPUT_LABEL_CHAR_LIMIT = 200; +const DOCS_CALL_REJECTED = "(no output, the tool is unavailable or its arguments were invalid)"; +const DOCS_NO_OUTPUT = "(no output)"; + export function assembleContext( visibleSteps: ReActStep[], workflowState: WorkflowState, @@ -53,6 +63,7 @@ export function assembleContext( const sections: string[] = []; let completedCount = 0; let hasOngoing = false; + let ongoingSteps: ReActStep[] = []; for (const msgId of messageIds) { const steps = stepsByMessage.get(msgId)!; @@ -69,6 +80,7 @@ export function assembleContext( completedCount++; } else { hasOngoing = true; + ongoingSteps = steps; sections.push(""); sections.push("# Ongoing Task"); sections.push(serializeTask(steps, "ongoing")); @@ -79,6 +91,13 @@ export function assembleContext( } } + const docsSection = serializeDocsOutput(ongoingSteps); + if (docsSection) { + sections.push(""); + sections.push("# Documentation Read"); + sections.push(docsSection); + } + const dagSection = serializeDag(workflowState, operatorExecutionResults, useRedact, compilationResult); if (dagSection) { sections.push(""); @@ -135,6 +154,53 @@ function serializeTask(steps: ReActStep[], status: "completed" | "ongoing"): str return lines.join("\n").trimEnd(); } +/** + * The output of the searchDocs and readDoc calls of the ongoing task, oldest first. Over the + * budget, the newest calls are kept whole and one line says how many older ones were left out. + */ +function serializeDocsOutput(steps: ReActStep[]): string | null { + const entries: string[] = []; + for (const step of steps) { + if (step.role !== "agent") continue; + for (const call of step.toolCalls ?? []) { + if (!DOCS_TOOL_NAMES.has(call.toolName)) continue; + const result = step.toolResults?.find(r => r.toolCallId === call.toolCallId); + const input = JSON.stringify(call.input ?? {}); + const shownInput = + input.length <= DOCS_INPUT_LABEL_CHAR_LIMIT ? input : `${cutText(input, DOCS_INPUT_LABEL_CHAR_LIMIT)}…`; + // A call to a tool that does not exist, or with arguments the SDK refused, is stored without a result. + let output = DOCS_CALL_REJECTED; + if (result) { + const text = typeof result.output === "string" ? result.output : JSON.stringify(result.output); + output = text === undefined || text === "" ? DOCS_NO_OUTPUT : text; + } + const indented = output.split("\n").map(line => ` ${line}`); + entries.push([`## Turn ${step.stepId}: ${call.toolName} ${shownInput}`, ...indented].join("\n")); + } + } + if (entries.length === 0) return null; + + // An entry costs its text plus the blank line before it; the newest one is always kept. + const cost = (entry: string) => entry.length + 2; + let firstKept = entries.length - 1; + let used = cost(entries[firstKept]); + while (firstKept > 0 && used + cost(entries[firstKept - 1]) <= DOCS_CONTEXT_CHAR_LIMIT) { + firstKept--; + used += cost(entries[firstKept]); + } + + const lines = ["Output of your searchDocs and readDoc calls in this task, oldest first."]; + if (firstKept > 0) { + const [outputs, them] = firstKept === 1 ? ["output", "it"] : ["outputs", "them"]; + lines.push(`(${firstKept} earlier ${outputs} dropped to save space, call the tool again if you need ${them})`); + } + for (const entry of entries.slice(firstKept)) { + lines.push(""); + lines.push(entry); + } + return lines.join("\n"); +} + function serializeDag( workflowState: WorkflowState, operatorExecutionResults: Map, diff --git a/agent-service/src/config/env.ts b/agent-service/src/config/env.ts index 3513286b149..961f36dec63 100644 --- a/agent-service/src/config/env.ts +++ b/agent-service/src/config/env.ts @@ -33,6 +33,7 @@ const EnvSchema = z.object({ WORKFLOW_COMPILING_SERVICE_ENDPOINT: z.string().url().default("http://localhost:9090"), WORKFLOW_EXECUTION_SERVICE_ENDPOINT: z.string().url().default("http://localhost:8085"), EXECUTION_ENDPOINT_TEMPLATE: z.string().optional(), + TEXERA_DOCS_DIR: z.string().default("./docs"), }); export const env = EnvSchema.parse(process.env); diff --git a/bin/dockerfiles/agent-service.dockerfile b/bin/dockerfiles/agent-service.dockerfile index 6deaa5005d9..18cdff26779 100644 --- a/bin/dockerfiles/agent-service.dockerfile +++ b/bin/dockerfiles/agent-service.dockerfile @@ -36,6 +36,7 @@ RUN bun install --frozen-lockfile --production \ && rm -rf /root/.bun/install/cache COPY agent-service/src ./src +COPY docs ./docs COPY agent-service/tsconfig.json ./ COPY agent-service/LICENSE-binary ./LICENSE diff --git a/bin/local-dev/main.sh b/bin/local-dev/main.sh index 9053834acbd..f0ccec2d106 100755 --- a/bin/local-dev/main.sh +++ b/bin/local-dev/main.sh @@ -714,6 +714,8 @@ export LITELLM_BASE_URL="${LITELLM_BASE_URL:-http://localhost:4000}" export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-texera-internal-do-not-share}" export LLM_ENDPOINT="${LLM_ENDPOINT:-http://localhost:8080}" export LLM_API_KEY="${LLM_API_KEY:-dummy}" +# The agent service runs from agent-service/, so its default ./docs would find nothing. +export TEXERA_DOCS_DIR="${TEXERA_DOCS_DIR:-$REPO_ROOT/docs}" # Email verification ships on, and refuses to issue a code when no SMTP sender is configured # rather than logging the code where anyone reading logs could spend it. A local stack has no diff --git a/bin/local-dev/tests/test_local_dev_sh.sh b/bin/local-dev/tests/test_local_dev_sh.sh index 75d14af1cc7..e5be2ceb53c 100755 --- a/bin/local-dev/tests/test_local_dev_sh.sh +++ b/bin/local-dev/tests/test_local_dev_sh.sh @@ -1293,5 +1293,13 @@ else fi rm -rf "$_im_state" +# 45) The agent service is started from agent-service/, where its default +# ./docs does not exist, so the stack has to point it at the repo's docs. +if grep -qE '^export TEXERA_DOCS_DIR="\$\{TEXERA_DOCS_DIR:-\$REPO_ROOT/docs\}"' "$MAIN_SH"; then + _pass "TEXERA_DOCS_DIR defaults to the repo's docs and honours a preset value" +else + _fail "TEXERA_DOCS_DIR is not exported for the agent service" +fi + printf "\n%d passed, %d failed\n" "$PASS" "$FAIL" (( FAIL == 0 ))