diff --git a/CHANGELOG.md b/CHANGELOG.md index 85bcf0b573..80c1d8e19a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,14 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. - Migration `0062_action_policy_labels` adds a `labels` column to `action_policies`, defaulting to empty. Existing boundary rules keep working and show as "Custom rule" until somebody names them. +### A Bot searches with its search tool, never its browser + +Asked to search for something, a Bot could drive its browser to Google or another search engine +instead of using the search tool it held. With a search tool it now searches with that, and keeps +the browser for signing in, clicking through a site, filling in a form or opening a page the person +named. Without one it answers from what it knows, says it has not checked, and shows a card for a +search app (Parallel Search, Exa Search or Firecrawl) that the person can ask an administrator for. + ### Screens share one spacing, row and empty-state design Every configuration screen now takes its spacing from the page frame: 32px under the title, 48px diff --git a/agent-bot/src/history.ts b/agent-bot/src/history.ts index 670cf1b987..c663b2173b 100644 --- a/agent-bot/src/history.ts +++ b/agent-bot/src/history.ts @@ -7,7 +7,8 @@ */ import type { RunAgentInput } from "@ag-ui/core"; import type OpenAI from "openai"; -import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt"; +import { computerGuidance, NO_ANSWER_CAME } from "../../shared/bot-prompt"; +import { researchRoleOfToolName } from "../../shared/research"; import { userContent } from "../../shared/user-content"; export { NO_ANSWER_CAME }; @@ -17,7 +18,19 @@ export function toProviderMessages( input: RunAgentInput, ): OpenAI.Chat.ChatCompletionMessageParam[] { const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [ - { role: "system", content: COMPUTER_GUIDANCE }, + /* + * Worded for the tools this run offers, which is all this Bot can see of its grants: the names + * say whether one of them is a search tool, and that decides whether the browser is told to + * leave searching to it or never to search at all. + */ + { + role: "system", + content: computerGuidance({ + search: (input.tools ?? []).some( + (tool) => researchRoleOfToolName(tool.name) === "search", + ), + }), + }, // AG-UI application context is separate from history. The A2UI catalog and tool instructions // arrive here; omitting them leaves the model guessing component names and action schemas. ...(input.context ?? []).map(({ description, value }) => ({ diff --git a/agent-bot/tests/history.test.ts b/agent-bot/tests/history.test.ts index 6fcb03a095..389e458405 100644 --- a/agent-bot/tests/history.test.ts +++ b/agent-bot/tests/history.test.ts @@ -1,7 +1,44 @@ import { describe, expect, test } from "bun:test"; import type { RunAgentInput } from "@ag-ui/core"; +import { computerGuidance } from "../../shared/bot-prompt"; import { NO_ANSWER_CAME, toProviderMessages } from "../src/history"; +/** + * Whether the browser is told to leave searching to a search tool is decided by the tools this run + * offers, since the names are all this Bot sees of its grants. "search for X" made a Bot holding + * a search tool drive its browser to a search engine instead. + */ +describe("the computer guidance this Bot is given", () => { + const offered = (...names: string[]) => + ({ + messages: [{ id: "u1", role: "user", content: "Search for X." }], + tools: names.map((name) => ({ name, description: "", parameters: {} })), + }) as unknown as RunAgentInput; + + test("leaves searching to a search tool the run offers", () => { + const [system] = toProviderMessages( + offered("show_app_access", "mcp__exa__web_search_exa"), + ); + expect(system).toEqual({ + role: "system", + content: computerGuidance({ search: true }), + }); + }); + + test("never searches in the browser when no search tool is offered", () => { + for (const run of [ + offered("show_app_access", "mcp__google-drive__search_files"), + // A reader alone does not find sources. + offered("mcp__exa__web_fetch_exa"), + { messages: [] } as unknown as RunAgentInput, + ]) + expect(toProviderMessages(run)[0]).toEqual({ + role: "system", + content: computerGuidance({ search: false }), + }); + }); +}); + /** * The Bot that ships in the box, and the conversation a declined handover used to end. * diff --git a/agent-langgraph/src/history.ts b/agent-langgraph/src/history.ts index 6fe53b1ab4..861995d28d 100644 --- a/agent-langgraph/src/history.ts +++ b/agent-langgraph/src/history.ts @@ -13,7 +13,8 @@ import { SystemMessage, ToolMessage, } from "@langchain/core/messages"; -import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt"; +import { computerGuidance, NO_ANSWER_CAME } from "../../shared/bot-prompt"; +import { researchRoleOfToolName } from "../../shared/research"; import { userContent } from "../../shared/user-content"; /* @@ -39,7 +40,14 @@ export function toLangChainMessages( provider = "openai", ): BaseMessage[] { const messages: BaseMessage[] = [ - new SystemMessage(COMPUTER_GUIDANCE), + // Worded for the tools this run offers; see `computerGuidance` and agent-bot's history.ts. + new SystemMessage( + computerGuidance({ + search: (input.tools ?? []).some( + (tool) => researchRoleOfToolName(tool.name) === "search", + ), + }), + ), // AG-UI carries application context separately from conversation history. CopilotKit puts // the A2UI catalog and tool instructions here; dropping it leaves the model guessing the // component schema and can strand the renderer on an invalid, never-painted surface. diff --git a/agent-langgraph/tests/history.test.ts b/agent-langgraph/tests/history.test.ts index 50c468c422..bd18cb9de0 100644 --- a/agent-langgraph/tests/history.test.ts +++ b/agent-langgraph/tests/history.test.ts @@ -7,7 +7,7 @@ import { ToolMessage, } from "@langchain/core/messages"; import { ChatGoogleGenerativeAI } from "@langchain/google-genai"; -import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt"; +import { computerGuidance } from "../../shared/bot-prompt"; import { NO_ANSWER_CAME, toLangChainMessages } from "../src/history"; /** @@ -26,6 +26,27 @@ import { NO_ANSWER_CAME, toLangChainMessages } from "../src/history"; const input = (messages: unknown[]): RunAgentInput => ({ messages }) as unknown as RunAgentInput; +describe("the computer guidance this Bot is given", () => { + // Decided by the tool names the run offers, the only view this Bot has of its grants. + const offered = (...names: string[]) => + ({ + messages: [{ id: "u1", role: "user", content: "Search for X." }], + tools: names.map((name) => ({ name, description: "", parameters: {} })), + }) as unknown as RunAgentInput; + + test("leaves searching to a search tool the run offers", () => { + const [system] = toLangChainMessages( + offered("show_app_access", "mcp__parallel__web_search"), + ); + expect(String(system?.content)).toBe(computerGuidance({ search: true })); + }); + + test("never searches in the browser when no search tool is offered", () => { + const [system] = toLangChainMessages(offered("show_app_access")); + expect(String(system?.content)).toBe(computerGuidance({ search: false })); + }); +}); + const assistantAsking = { role: "assistant", content: "", @@ -273,7 +294,7 @@ describe("a provider that takes one system prompt", () => { expect(system).toHaveLength(1); expect(messages[0]).toBe(system[0] as SystemMessage); const prompt = String(system[0]?.content); - expect(prompt.startsWith(COMPUTER_GUIDANCE)).toBe(true); + expect(prompt.startsWith(computerGuidance({ search: false }))).toBe(true); const at = instructions.map((text) => prompt.indexOf(text)); expect(at.every((index) => index > 0)).toBe(true); expect(at).toEqual([...at].sort((a, b) => a - b)); diff --git a/app/src/lib/copilot/computer-tools.tsx b/app/src/lib/copilot/computer-tools.tsx index 2e6d2ccc64..524d2336d8 100644 --- a/app/src/lib/copilot/computer-tools.tsx +++ b/app/src/lib/copilot/computer-tools.tsx @@ -199,9 +199,10 @@ export function ComputerTools() { available: offered.navigate, name: "computer_navigate", description: - "Open a web page on your own computer so the person can watch. Use this when asked to look " + - "at, visit, open or check a website. Returns the page title and its readable text, so answer " + - "from what comes back rather than telling the person to go and look.", + "Open a web page on your own computer so the person can watch. Use this when asked to open, " + + "visit or read a particular website. It is not a web search: never open a search engine with " + + "it to look something up. Returns the page title and its readable text, so answer from what " + + "comes back rather than telling the person to go and look.", parameters: z.object({ url: z.string().describe("Full web address to open, including https://"), }), diff --git a/server/src/agents/agent-resolver.ts b/server/src/agents/agent-resolver.ts index 1314d4444f..f14926d529 100644 --- a/server/src/agents/agent-resolver.ts +++ b/server/src/agents/agent-resolver.ts @@ -1,4 +1,5 @@ import type { AbstractAgent } from "@ag-ui/client"; +import type { ComputerGuidance } from "@openbot/shared/bot-prompt"; import type { AuditInitiator } from "../audit"; import type { AgentFetch, StallGuard } from "../channels/stall-guard"; import { @@ -49,7 +50,7 @@ export type ActorAgentResolverDependencies = { initiator?: AuditInitiator, depth?: number, ) => SignRun; - computerGuidance?: string; + computerGuidance?: ComputerGuidance; loadVendors?: () => Promise; selectionForActor?: (actorId: string) => ToolSelection; agentFetch?: AgentFetch; diff --git a/server/src/copilot.ts b/server/src/copilot.ts index 5c5bca798a..ac0b822079 100644 --- a/server/src/copilot.ts +++ b/server/src/copilot.ts @@ -6,7 +6,10 @@ import { CopilotRuntime, } from "@copilotkit/runtime/v2"; import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono"; -import { PROVENANCE_GUIDANCE } from "@openbot/shared/bot-prompt"; +import { + type ComputerGuidance, + PROVENANCE_GUIDANCE, +} from "@openbot/shared/bot-prompt"; import { defaultModelFor } from "@openbot/shared/model-providers"; import { BuiltInAgentWithSaneHistory, @@ -65,7 +68,7 @@ import { SELECTION_FLOOR, selectTools, } from "./plugins/selection"; -import { grantedToolGuidance } from "./plugins/tools"; +import { grantedToolGuidance, holdsWebSearch } from "./plugins/tools"; import { UNTRUSTED_GUIDANCE } from "./untrusted-content"; /** @@ -375,8 +378,11 @@ export function builtInAgentConfiguration( * what its hands are. Absent leaves the role alone, which is right for a deployment with no * computer configured, where the browser routes are not mounted and a Bot promised a browser would * be promising something that does not exist. + * + * Worded for the tools passed here, which are what this run is offered: whether one of them finds + * sources decides whether the browser is told to leave searching to it or never to search at all. */ - computerGuidance?: string, + computerGuidance?: ComputerGuidance, /** * Vendors this deployment connects to, whether or not this Bot holds any of their tools. * @@ -445,7 +451,9 @@ export function builtInAgentConfiguration( ...(grantedToolGuidance(tools, connectedVendors) ? [grantedToolGuidance(tools, connectedVendors)] : []), - ...(computerGuidance ? [computerGuidance] : []), + ...(computerGuidance + ? [computerGuidance({ search: holdsWebSearch(tools) })] + : []), ...(learnedSkills?.catalog ? [learnedSkills.catalog] : []), ].join("\n\n"), ...(planModel ? {} : { apiKey: apiKey ?? undefined }), @@ -501,7 +509,7 @@ export async function buildAgents( loadTools: LoadToolsForBot = async () => [], signRun?: SignRun, /** What every built-in Bot is told about the computer. Absent means this deployment has none. */ - computerGuidance?: string, + computerGuidance?: ComputerGuidance, /** * Which vendors this deployment connects to. Asked once per build rather than per Bot, because it * is a fact about the deployment; what differs per Bot is which of them it holds. @@ -847,7 +855,7 @@ async function buildAgent( stallGuard: StallGuard | undefined, loadTools: LoadToolsForBot, signRun?: SignRun, - computerGuidance?: string, + computerGuidance?: ComputerGuidance, connectedVendors: readonly string[] = [], selection?: ToolSelection, agentFetch?: AgentFetch, @@ -1285,7 +1293,7 @@ function remoteAgentWithStandingRole( * the same reason the tools are, so switching a connector on reaches the next run. * * The remote path needs this more than the built-in one, not less. A framework Bot is handed the - * tools as an offer and decides for itself what to call, with `COMPUTER_GUIDANCE` as its whole + * tools as an offer and decides for itself what to call, with `computerGuidance` as its whole * prompt — a page about the browser that mentions connectors nowhere. That is the Bot that browsed * to drive.google.com holding four Drive tools. * @@ -1566,7 +1574,7 @@ export async function resolveRuntimeAgents( stallGuard?: StallGuard, loadTools?: LoadToolsForBot, signRun?: SignRun, - computerGuidance?: string, + computerGuidance?: ComputerGuidance, loadVendors?: () => Promise, selection?: ToolSelection, agentFetch?: AgentFetch, diff --git a/server/src/index.ts b/server/src/index.ts index 77faeca575..fa979735d8 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -3,7 +3,7 @@ import { CopilotKitIntelligence, IntelligenceAgentRunner, } from "@copilotkit/runtime/v2"; -import { COMPUTER_GUIDANCE } from "@openbot/shared/bot-prompt"; +import { computerGuidance } from "@openbot/shared/bot-prompt"; import { workOwner } from "@openbot/shared/work-owner"; import { createTurnRunner } from "@openbot/toolkit/run-turn"; import { and, eq, isNull, ne, sql } from "drizzle-orm"; @@ -1704,7 +1704,7 @@ const actorAgentResolver = createActorAgentResolver({ proactiveReadOnlyRefs, ), signRunForActor, - computerGuidance: config.computer ? COMPUTER_GUIDANCE : undefined, + computerGuidance: config.computer ? computerGuidance : undefined, loadVendors, selectionForActor, agentFetch, diff --git a/server/src/plugins/tools.ts b/server/src/plugins/tools.ts index 9496f2f6a9..843db6146f 100644 --- a/server/src/plugins/tools.ts +++ b/server/src/plugins/tools.ts @@ -80,7 +80,7 @@ export function vendorAnswer(result: { text: string; isError: boolean }) { * A tool array tells a model a tool exists. It does not tell it that the tool is the right way to * reach that system, and it competes with a page of prose about the browser that every Bot is given * whether or not it has any connectors at all. The browser prose wins: it is emphatic, it is about - * capability, and it says "never claim you cannot browse". + * capability, and it used to say "never claim you cannot browse". * * So a Bot holding four Google Drive tools browsed to drive.google.com, met a sign-in page its * container could never satisfy, and asked its person to sign in to a vendor that person had already @@ -204,6 +204,17 @@ type ResearchProvider = { fetch: GrantedTool; }; +/** + * Whether these tools include one that finds sources, from any research provider. + * + * The search role alone, not the full pair `researchProviders` wants: a Bot holding a search tool and + * no reader can still search, and telling it to drive its browser to a search engine instead would + * be the very routing mistake this decides against. See `computerGuidance`. + */ +export function holdsWebSearch(tools: readonly GrantedTool[]): boolean { + return tools.some((tool) => researchRoleFor(tool.ref) === "search"); +} + /** * The research providers this Bot can actually use: a search tool and a fetch tool from one server. * diff --git a/server/tests/copilot.test.ts b/server/tests/copilot.test.ts index 978a0811d9..613e5fa524 100644 --- a/server/tests/copilot.test.ts +++ b/server/tests/copilot.test.ts @@ -5,7 +5,10 @@ import type { AbstractAgent, RunAgentInput } from "@ag-ui/client"; import { HttpAgent } from "@ag-ui/client"; import { LLMock } from "@copilotkit/aimock"; import { BuiltInAgent } from "@copilotkit/runtime/v2"; -import { PROVENANCE_GUIDANCE } from "@openbot/shared/bot-prompt"; +import { + computerGuidance, + PROVENANCE_GUIDANCE, +} from "@openbot/shared/bot-prompt"; import { defaultModelFor } from "@openbot/shared/model-providers"; import { EMPTY, lastValueFrom } from "rxjs"; import { z } from "zod"; @@ -987,7 +990,7 @@ function fakeAgUiEndpoint() { * A Bot is told what it holds, not only handed it. * * A tool array tells a model a tool exists. It does not say the tool is the right way to reach that - * system, and it competes with `COMPUTER_GUIDANCE`: a page of emphatic prose about the browser that + * system, and it competes with `computerGuidance`: a page of emphatic prose about the browser that * every Bot gets whether or not it has a single connector, and that mentions connectors nowhere. * * The browser prose won. A Bot holding four Google Drive tools browsed to drive.google.com, met a @@ -1133,7 +1136,7 @@ describe("what a Bot is told it holds", () => { { provider: "openai", defaultModel: "gpt-5.6-terra" }, "openai-secret", drive, - "BROWSER GUIDANCE HERE", + () => "BROWSER GUIDANCE HERE", ).prompt as string; // Order is the fix, not merely presence: the grants have to land before the browser prose. @@ -1144,6 +1147,88 @@ describe("what a Bot is told it holds", () => { }); }); +/** + * Whether the browser leaves searching to a search tool, decided by what this run holds. + * + * "search for X" in a demo made a Bot drive its browser to a search engine. The guidance has two + * wordings, and which one a built-in Bot gets has to follow the tools it is built with, including a + * narrowed run that was not offered the search tool it was granted. + */ +describe("what a built-in Bot is told about searching", () => { + const promptHolding = ( + tools: { name: string; ref: string }[], + guidance: typeof computerGuidance | null = computerGuidance, + ) => { + const configuration = builtInAgentConfiguration( + { + id: "general-assistant", + name: "General Assistant", + type: "built_in", + systemPrompt: "Be helpful.", + }, + { provider: "openai", defaultModel: "gpt-5.6-terra" }, + "openai-secret", + tools as never[], + guidance ?? undefined, + ); + return String((configuration as { prompt?: unknown }).prompt); + }; + const card = { name: "show_app_access", ref: "app-access/show_app_access" }; + const exaSearch = { + name: "mcp__exa__web_search_exa", + ref: "exa/web_search_exa", + }; + const exaFetch = { + name: "mcp__exa__web_fetch_exa", + ref: "exa/web_fetch_exa", + }; + + test.each([ + ["a provider's pair", [card, exaSearch, exaFetch]], + ["a search tool alone", [card, exaSearch]], + [ + "another provider", + [ + card, + { + name: "mcp__firecrawl__firecrawl_search", + ref: "firecrawl/firecrawl_search", + }, + ], + ], + ])("holding %s, it is told to search with it", (_, tools) => { + const prompt = promptHolding(tools); + expect(prompt).toContain(computerGuidance({ search: true })); + expect(prompt).not.toContain(computerGuidance({ search: false })); + }); + + test.each([ + ["nothing", []], + ["only the card", [card]], + ["a reader with no search", [card, exaFetch]], + [ + "a tool called search that finds no sources", + [ + card, + { + name: "mcp__google-drive__search_files", + ref: "google-drive/search_files", + }, + ], + ], + ])("holding %s, it is told never to search in the browser", (_, tools) => { + const prompt = promptHolding(tools); + expect(prompt).toContain(computerGuidance({ search: false })); + expect(prompt).not.toContain(computerGuidance({ search: true })); + }); + + test("a deployment with no computer is told about neither", () => { + const prompt = promptHolding([card, exaSearch], null); + expect(prompt).toContain("Be helpful."); + expect(prompt).not.toContain("computer_navigate"); + }); +}); + /** * Where an answer came from, on every Bot rather than the ones somebody remembered. * @@ -1277,7 +1362,7 @@ describe("a person's standing instructions", () => { model, "openai-secret", [{ name: "mcp__google-drive__search_files" }] as never[], - "Computer guidance.", + () => "Computer guidance.", [], "Write in British English.", ).prompt as string; diff --git a/server/tests/web-research.test.ts b/server/tests/web-research.test.ts index 4e4d30d92f..e56320d738 100644 --- a/server/tests/web-research.test.ts +++ b/server/tests/web-research.test.ts @@ -2,12 +2,15 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import type { GrantedTool } from "@openbot/toolkit/granted-tool"; import { + RESEARCH_APP_NAMES, RESEARCH_TOOLS as SHARED_RESEARCH_TOOLS, researchRoleOf, + researchRoleOfToolName, } from "@openbot/shared/research"; import { parse } from "yaml"; import { z } from "zod"; import { + CATALOGUE, catalogueEntry, classifyTool, researchRoleFor, @@ -17,7 +20,7 @@ import { } from "../src/plugins/catalogue"; import { listTools } from "../src/plugins/mcp"; import { selectTools } from "../src/plugins/selection"; -import { grantedToolGuidance } from "../src/plugins/tools"; +import { grantedToolGuidance, holdsWebSearch } from "../src/plugins/tools"; const tool = (server: string, name: string): GrantedTool => ({ name: `mcp__${server}__${name}`, @@ -37,6 +40,43 @@ const RESEARCH_TOOLS: Record = { const pair = (server = "parallel") => (RESEARCH_TOOLS[server] ?? []).map((name) => tool(server, name)); +describe("whether a Bot holds a search tool", () => { + test("any provider's search tool counts, from its ref or the name a model sees", () => { + for (const [server, [search, fetch]] of Object.entries(RESEARCH_TOOLS)) { + expect(holdsWebSearch([tool(server, search)])).toBe(true); + expect(holdsWebSearch([tool(server, fetch)])).toBe(false); + expect(researchRoleOfToolName(`mcp__${server}__${search}`)).toBe( + "search", + ); + expect(researchRoleOfToolName(`mcp__${server}__${fetch}`)).toBe("fetch"); + } + expect(holdsWebSearch([])).toBe(false); + expect(holdsWebSearch([tool("google-drive", "search_files")])).toBe(false); + expect(researchRoleOfToolName("web_search")).toBeNull(); + expect( + researchRoleOfToolName("mcp__google-drive__search_files"), + ).toBeNull(); + }); + + test("the search apps a Bot without one offers are the catalogue's, by a name the card resolves", () => { + // The card resolves an app by key or title. A renamed title would leave a Bot offering a card + // for an app this deployment has no name for. + const research = CATALOGUE.filter((entry) => + Object.hasOwn(SHARED_RESEARCH_TOOLS, entry.key), + ); + const named = research.filter((entry) => + RESEARCH_APP_NAMES.includes(entry.title), + ); + expect(named.map((entry) => entry.title).sort()).toEqual( + [...RESEARCH_APP_NAMES].sort(), + ); + // Every research vendor is offered, so the choice stays the administrator's. + expect(new Set(named.map((entry) => entry.vendor))).toEqual( + new Set(research.map((entry) => entry.vendor)), + ); + }); +}); + describe("research providers in the catalogue", () => { test("each is pinned to its vendor's official endpoint", () => { const urls: Record = { diff --git a/shared/bot-prompt.test.ts b/shared/bot-prompt.test.ts index 2f5d14f34d..cb2b0b271b 100644 --- a/shared/bot-prompt.test.ts +++ b/shared/bot-prompt.test.ts @@ -1,31 +1,103 @@ import { describe, expect, test } from "bun:test"; -import { COMPUTER_GUIDANCE } from "./bot-prompt"; +import { computerGuidance } from "./bot-prompt"; +import { RESEARCH_APP_NAMES } from "./research"; -describe("COMPUTER_GUIDANCE", () => { - test("keeps paragraph breaks as blank lines instead of collapsing them into spaces", () => { - expect(COMPUTER_GUIDANCE).toContain("\n\n"); - expect(COMPUTER_GUIDANCE).not.toContain(" "); - }); +const WITH_SEARCH = computerGuidance({ search: true }); +const WITHOUT_SEARCH = computerGuidance({ search: false }); + +/** One sentence can span a source line break, which is not a fact about what the Bot is told. */ +const flat = (text: string) => text.toLowerCase().replace(/\s+/g, " "); - test("keeps each paragraph as one unbroken line of prose", () => { - for (const paragraph of COMPUTER_GUIDANCE.split("\n\n")) { +describe("computerGuidance", () => { + test.each([ + ["with a search tool", WITH_SEARCH], + ["without one", WITHOUT_SEARCH], + ])("keeps paragraph breaks as blank lines %s", (_, guidance) => { + expect(guidance).toContain("\n\n"); + expect(guidance).not.toContain(" "); + for (const paragraph of guidance.split("\n\n")) { expect(paragraph).not.toContain("\n"); expect(paragraph.length).toBeGreaterThan(0); } }); - test("still contains the full instruction text, unchanged in wording", () => { - expect(COMPUTER_GUIDANCE).toContain( - "You are a Bot with your own computer, a real web browser the person can watch you use.", + test.each([ + ["with a search tool", WITH_SEARCH], + ["without one", WITHOUT_SEARCH], + ])( + "still says the computer can open a page the person names, %s", + (_, guidance) => { + expect(guidance).toContain( + "You are a Bot with your own computer, a real web browser the person can watch you use.", + ); + expect(flat(guidance)).toContain( + "when you are asked to open, visit or read a particular web page, call computer_navigate", + ); + expect(guidance).toContain( + "Say what you found or did in plain language, briefly.", + ); + }, + ); + + test.each([ + ["with a search tool", WITH_SEARCH], + ["without one", WITHOUT_SEARCH], + ])( + "no longer insists the Bot never says it cannot browse, %s", + (_, guidance) => { + // The emphatic line that out-argued the hedge beside it in the demo. + expect(flat(guidance)).not.toContain("never claim you cannot browse"); + // "check" sent "check the latest on X" to the browser. + expect(flat(guidance)).not.toContain("look at, open, visit, check"); + }, + ); + + test("with a search tool, finding information goes to it and never to the browser", () => { + const guidance = flat(WITH_SEARCH); + expect(guidance).toContain( + "finding information is not a job for your computer", ); - expect(COMPUTER_GUIDANCE).toContain( - "Say what you found or did in plain language, briefly.", + // The three phrasings the routing has to survive. + expect(guidance).toContain("search for something"); + expect(guidance).toContain("find the latest on a subject"); + expect(guidance).toContain("find a few examples of something"); + expect(guidance).toContain("use your search tool"); + expect(guidance).toContain("do not open a search engine in your browser"); + // What the computer is still for. + expect(guidance).toContain( + "signing in, clicking through a site, filling in a form, or opening a page the person asked to see", ); + // Never told to navigate for a search, and never told it cannot search. + expect(guidance).not.toMatch(/search[^.]*call computer_navigate/); + expect(guidance).not.toContain("you do not hold a search tool"); + expect(guidance).not.toContain("cannot search"); }); - test("does not let the browser pass for a web search", () => { - // Asked "can you search the web?", a Bot with only a browser said yes. - expect(COMPUTER_GUIDANCE).toContain("Opening a page is not a web search."); - expect(COMPUTER_GUIDANCE).toContain("cannot search"); + test("without a search tool, the browser is never a search engine", () => { + const guidance = flat(WITHOUT_SEARCH); + expect(guidance).toContain("opening a page is not a web search"); + expect(guidance).toContain("never drive your browser to a search engine"); + for (const engine of ["google", "bing", "duckduckgo"]) + expect(guidance).toContain(engine); + expect(guidance).not.toContain("use your search tool"); + // Conditional, not a claim: a search tool from Composio or a server added by URL is not + // recognised by role, and a Bot holding one must not be told it holds none. + expect(guidance).toContain("none of your tools searches the web"); + expect(guidance).not.toContain("you do not hold a search tool"); + }); + + test("without a search tool, it answers marked as unchecked and offers a search app", () => { + const guidance = flat(WITHOUT_SEARCH); + expect(guidance).toContain("answer from what you already know"); + expect(guidance).toContain("you have not checked it against a source"); + expect(guidance).toContain("call it once in the conversation"); + expect(guidance).toContain("show_app_access"); + for (const name of RESEARCH_APP_NAMES) + expect(guidance).toContain(name.toLowerCase()); + expect(guidance).toContain( + "otherwise say an administrator can add a search app", + ); + // Asked "can you search the web?", a Bot with only a browser once said yes. + expect(guidance).toContain("cannot search"); }); }); diff --git a/shared/bot-prompt.ts b/shared/bot-prompt.ts index 306e97566b..7b08d31eb2 100644 --- a/shared/bot-prompt.ts +++ b/shared/bot-prompt.ts @@ -9,21 +9,73 @@ * only their role: asked to file an issue on a site it was not signed in to, one browsed to the page * and then said it could not, never calling `computer_request_help` to have a person sign in. */ +import { SHOW_APP_ACCESS_TOOL } from "./connect-prompt"; +import { RESEARCH_APP_NAMES } from "./research"; + +const OPENING_A_PAGE_LINES = [ + "You are a Bot with your own computer, a real web browser the person can watch you use.", + "When you are asked to open, visit or read a particular web page, call computer_navigate: that is", + "something you can actually do.", + "Opening a page returns its title and its readable text. Answer from that text.", + "Never tell the person to go and look at the page themselves: you have already read it.", +]; + +/** + * Where finding something out goes, which depends on whether this run holds a search tool. + * + * Asked to "search for" something in a demo, a Bot drove its browser to a search engine instead of + * calling the search tool it held. The prompt invited it: it said to navigate whenever asked to + * "look at, check or read" anything and to "never claim you cannot browse", and the one sentence + * saying a page is not a search sat beside them as a hedge. A model reading both picks the emphatic + * one. So the Bot is told one thing, chosen by what it holds, instead of two things to reconcile. + * + * WITH A SEARCH TOOL, finding information is the search tool's job, and the computer is kept for what + * only a browser can do. The research guidance generated from the grants says how to use the tool; + * this says only that the browser is not it. + * + * WITHOUT ONE, the browser is still not a search tool. A Bot typing a query into a search engine + * meets cookie walls and CAPTCHAs, reads results pages built for people, and makes the person watch + * it. It answers from what it knows, marked as unchecked as `PROVENANCE_GUIDANCE` asks, and offers a + * search app with the card tool, because "I cannot search" with no way to fix it is the same dead end + * a missing app used to be. The names are the catalogue's search apps, since the card resolves an + * app by name and there is no app called "search"; see `RESEARCH_APP_NAMES`. + * + * Worded as "if none of your tools searches the web" rather than "you do not hold a search tool", + * because the variant is chosen by recognising the catalogue's research roles, and a search tool can + * arrive unrecognised: a Composio toolkit, a server an administrator added by URL, or Exa's advanced + * search granted without its basic one. Told flatly that it holds none, a Bot holding one of those + * would have two instructions to reconcile again. + */ +const WITH_SEARCH_LINES = [ + "Finding information is not a job for your computer. When you are asked to search for something,", + "look something up, find the latest on a subject or find a few examples of something, use your", + "search tool and answer from what it returns. Do not open a search engine in your browser.", + "Use the computer for what a search tool cannot do: signing in, clicking through a site, filling in", + "a form, or opening a page the person asked to see.", +]; + +const WITHOUT_SEARCH_LINES = [ + "Opening a page is not a web search. NEVER drive your browser to a search engine (Google, Bing,", + "DuckDuckGo or any other) to look something up, and do not guess at addresses hoping one of them", + "has the answer.", + "When you are asked to search for something, look something up, find the latest on a subject or", + "find a few examples of something, and none of your tools searches the web, answer from what you", + "already know and say plainly that you have not checked it against a source.", + `If you hold ${SHOW_APP_ACCESS_TOOL}, call it once in the conversation with the name of one search`, + `app (${RESEARCH_APP_NAMES.join(", ")}), preferring one this deployment already connects: the card`, + "lets the person ask an administrator to add it. Otherwise say an administrator can add a search", + "app.", + "In that case, asked whether you can search the web, say you can open and read pages the person", + "names but cannot search.", +]; + /** * The order of operations that makes the computer tools usable. * * The prompt requires snapshot-first computer use. Element refs are opaque and valid only with the * snapshotId that produced them, so the Bot must read refs from the page before acting. */ -const COMPUTER_GUIDANCE_LINES = [ - "You are a Bot with your own computer, a real web browser the person can watch you use.", - "When you are asked to look at, open, visit, check or read a web page, call computer_navigate.", - "Never claim you cannot browse: opening a page is something you can actually do.", - "Opening a page is not a web search. To find sources rather than read a page you can name, use a", - "search tool if you hold one; if you do not, say you can open and read pages but cannot search.", - "Opening a page returns its title and its readable text. Answer from that text.", - "Never tell the person to go and look at the page themselves: you have already read it.", - "", +const ACTING_LINES = [ "You can also ACT on a page: fill in forms, click buttons, tick boxes and follow links.", "The order matters. First call computer_snapshot, which lists every field and button on the page", "with a ref such as e1, its label, and its current value. Then call computer_type and", @@ -69,22 +121,55 @@ const COMPUTER_GUIDANCE_LINES = [ ]; /** - * `COMPUTER_GUIDANCE_LINES` uses `""` as a paragraph break. Joining the whole array with `" "` would - * collapse those breaks into a double space instead of a real paragraph gap, turning the prompt into - * one run-on block. Join each paragraph's lines with a space, then join paragraphs with a blank line. + * These line arrays use `""` as a paragraph break. Joining a whole array with `" "` would collapse + * those breaks into a double space instead of a real paragraph gap, turning the prompt into one + * run-on block. Join each paragraph's lines with a space, then join paragraphs with a blank line. */ -export const COMPUTER_GUIDANCE = COMPUTER_GUIDANCE_LINES.reduce( - (paragraphs, line) => { - if (line === "") { - paragraphs.push(""); - return paragraphs; - } - const last = paragraphs.length - 1; - paragraphs[last] = paragraphs[last] ? `${paragraphs[last]} ${line}` : line; - return paragraphs; - }, - [""], -).join("\n\n"); +function asParagraphs(lines: readonly string[]): string { + return lines + .reduce( + (paragraphs, line) => { + if (line === "") { + paragraphs.push(""); + return paragraphs; + } + const last = paragraphs.length - 1; + paragraphs[last] = paragraphs[last] + ? `${paragraphs[last]} ${line}` + : line; + return paragraphs; + }, + [""], + ) + .join("\n\n"); +} + +/** What a run holds that changes what it is told about its computer. */ +export type ComputerHolding = { + /** Whether this run was offered a tool that finds sources, from any research provider. */ + search: boolean; +}; + +/** + * The computer guidance for one run, chosen by what it holds. + * + * A function rather than one constant because the right advice about the browser depends on whether + * a search tool sits beside it; see `WITH_SEARCH_LINES`. Built-in Bots get it from `copilot.ts`, which + * knows the tools it is building with; `agent-bot` and `agent-langgraph` work it out from the tools + * each run offers them. + */ +export function computerGuidance(holding: ComputerHolding): string { + return asParagraphs([ + ...OPENING_A_PAGE_LINES, + "", + ...(holding.search ? WITH_SEARCH_LINES : WITHOUT_SEARCH_LINES), + "", + ...ACTING_LINES, + ]); +} + +/** How a deployment with a computer words it for a run. See {@link computerGuidance}. */ +export type ComputerGuidance = typeof computerGuidance; /** * Where an answer came from, said out loud. @@ -133,18 +218,7 @@ const PROVENANCE_GUIDANCE_LINES = [ "honest, and a search that never ends is a Bot that never answers.", ]; -export const PROVENANCE_GUIDANCE = PROVENANCE_GUIDANCE_LINES.reduce( - (paragraphs, line) => { - if (line === "") { - paragraphs.push(""); - return paragraphs; - } - const last = paragraphs.length - 1; - paragraphs[last] = paragraphs[last] ? `${paragraphs[last]} ${line}` : line; - return paragraphs; - }, - [""], -).join("\n\n"); +export const PROVENANCE_GUIDANCE = asParagraphs(PROVENANCE_GUIDANCE_LINES); /** * What a tool call is given when its answer never came. diff --git a/shared/research.ts b/shared/research.ts index 2763697691..6b3d31d081 100644 --- a/shared/research.ts +++ b/shared/research.ts @@ -91,3 +91,35 @@ export function researchRoleOf(ref: string): ResearchRole | null { if (prefix !== RESEARCH_ROLE_PREFIX || extra !== undefined) return null; return role === "search" || role === "fetch" ? role : null; } + +/** + * The research role a tool plays, from the name a model is offered it under, or null. + * + * Granted tools reach a model as `mcp____`. A Bot running its own loop (`agent-bot`, + * `agent-langgraph`) sees only those names, not the refs the server holds, so this is how it learns + * whether it was offered a search tool on this run. + */ +export function researchRoleOfToolName(name: string): ResearchRole | null { + if (!name.startsWith("mcp__")) return null; + const rest = name.slice("mcp__".length); + const separator = rest.indexOf("__"); + if (separator <= 0) return null; + return researchRoleOfTool( + rest.slice(0, separator), + rest.slice(separator + 2), + ); +} + +/** + * The search apps a Bot holding none may offer the person, by a name the app card resolves. + * + * Titles of the free catalogue entries, because `show_app_access` resolves an app's key or title and + * nothing more general: there is no app called "search". All of them rather than one, because which + * search app a deployment uses is the administrator's choice. A name here that is not a research + * entry's title is a test failure, so the list cannot drift from the catalogue. + */ +export const RESEARCH_APP_NAMES: readonly string[] = Object.freeze([ + "Parallel Search", + "Exa Search", + "Firecrawl", +]);