Send a search to the search tool, never to the browser

CHANGELOG.md

@@ -40,6 +40,14 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged.
 - Migration `0062_action_policy_labels` adds a `labels` column to `action_policies`, defaulting to
   empty. Existing boundary rules keep working and show as "Custom rule" until somebody names them.
 
+### A Bot searches with its search tool, never its browser
+
+Asked to search for something, a Bot could drive its browser to Google or another search engine
+instead of using the search tool it held. With a search tool it now searches with that, and keeps
+the browser for signing in, clicking through a site, filling in a form or opening a page the person
+named. Without one it answers from what it knows, says it has not checked, and shows a card for a
+search app (Parallel Search, Exa Search or Firecrawl) that the person can ask an administrator for.
+
 ### Screens share one spacing, row and empty-state design
 
 Every configuration screen now takes its spacing from the page frame: 32px under the title, 48px

agent-bot/src/history.ts

@@ -7,7 +7,8 @@
  */
 import type { RunAgentInput } from "@ag-ui/core";
 import type OpenAI from "openai";
-import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { computerGuidance, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { researchRoleOfToolName } from "../../shared/research";
 import { userContent } from "../../shared/user-content";
 
 export { NO_ANSWER_CAME };
@@ -17,7 +18,19 @@ export function toProviderMessages(
   input: RunAgentInput,
 ): OpenAI.Chat.ChatCompletionMessageParam[] {
   const messages: OpenAI.Chat.ChatCompletionMessageParam[] = [
-    { role: "system", content: COMPUTER_GUIDANCE },
+    /*
+     * Worded for the tools this run offers, which is all this Bot can see of its grants: the names
+     * say whether one of them is a search tool, and that decides whether the browser is told to
+     * leave searching to it or never to search at all.
+     */
+    {
+      role: "system",
+      content: computerGuidance({
+        search: (input.tools ?? []).some(
+          (tool) => researchRoleOfToolName(tool.name) === "search",
+        ),
+      }),
+    },
     // AG-UI application context is separate from history. The A2UI catalog and tool instructions
     // arrive here; omitting them leaves the model guessing component names and action schemas.
     ...(input.context ?? []).map(({ description, value }) => ({

agent-bot/tests/history.test.ts

@@ -1,7 +1,44 @@
 import { describe, expect, test } from "bun:test";
 import type { RunAgentInput } from "@ag-ui/core";
+import { computerGuidance } from "../../shared/bot-prompt";
 import { NO_ANSWER_CAME, toProviderMessages } from "../src/history";
 
+/**
+ * Whether the browser is told to leave searching to a search tool is decided by the tools this run
+ * offers, since the names are all this Bot sees of its grants. "search for X" made a Bot holding
+ * a search tool drive its browser to a search engine instead.
+ */
+describe("the computer guidance this Bot is given", () => {
+  const offered = (...names: string[]) =>
+    ({
+      messages: [{ id: "u1", role: "user", content: "Search for X." }],
+      tools: names.map((name) => ({ name, description: "", parameters: {} })),
+    }) as unknown as RunAgentInput;
+
+  test("leaves searching to a search tool the run offers", () => {
+    const [system] = toProviderMessages(
+      offered("show_app_access", "mcp__exa__web_search_exa"),
+    );
+    expect(system).toEqual({
+      role: "system",
+      content: computerGuidance({ search: true }),
+    });
+  });
+
+  test("never searches in the browser when no search tool is offered", () => {
+    for (const run of [
+      offered("show_app_access", "mcp__google-drive__search_files"),
+      // A reader alone does not find sources.
+      offered("mcp__exa__web_fetch_exa"),
+      { messages: [] } as unknown as RunAgentInput,
+    ])
+      expect(toProviderMessages(run)[0]).toEqual({
+        role: "system",
+        content: computerGuidance({ search: false }),
+      });
+  });
+});
+
 /**
  * The Bot that ships in the box, and the conversation a declined handover used to end.
  *

agent-langgraph/src/history.ts

@@ -13,7 +13,8 @@ import {
   SystemMessage,
   ToolMessage,
 } from "@langchain/core/messages";
-import { COMPUTER_GUIDANCE, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { computerGuidance, NO_ANSWER_CAME } from "../../shared/bot-prompt";
+import { researchRoleOfToolName } from "../../shared/research";
 import { userContent } from "../../shared/user-content";
 
 /*
@@ -39,7 +40,14 @@ export function toLangChainMessages(
   provider = "openai",
 ): BaseMessage[] {
   const messages: BaseMessage[] = [
-    new SystemMessage(COMPUTER_GUIDANCE),
+    // Worded for the tools this run offers; see `computerGuidance` and agent-bot's history.ts.
+    new SystemMessage(
+      computerGuidance({
+        search: (input.tools ?? []).some(
+          (tool) => researchRoleOfToolName(tool.name) === "search",
+        ),
+      }),
+    ),
     // AG-UI carries application context separately from conversation history. CopilotKit puts
     // the A2UI catalog and tool instructions here; dropping it leaves the model guessing the
     // component schema and can strand the renderer on an invalid, never-painted surface.

agent-langgraph/tests/history.test.ts

@@ -7,7 +7,7 @@ import {
   ToolMessage,
 } from "@langchain/core/messages";
 import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
-import { COMPUTER_GUIDANCE } from "../../shared/bot-prompt";
+import { computerGuidance } from "../../shared/bot-prompt";
 import { NO_ANSWER_CAME, toLangChainMessages } from "../src/history";
 
 /**
@@ -26,6 +26,27 @@ import { NO_ANSWER_CAME, toLangChainMessages } from "../src/history";
 const input = (messages: unknown[]): RunAgentInput =>
   ({ messages }) as unknown as RunAgentInput;
 
+describe("the computer guidance this Bot is given", () => {
+  // Decided by the tool names the run offers, the only view this Bot has of its grants.
+  const offered = (...names: string[]) =>
+    ({
+      messages: [{ id: "u1", role: "user", content: "Search for X." }],
+      tools: names.map((name) => ({ name, description: "", parameters: {} })),
+    }) as unknown as RunAgentInput;
+
+  test("leaves searching to a search tool the run offers", () => {
+    const [system] = toLangChainMessages(
+      offered("show_app_access", "mcp__parallel__web_search"),
+    );
+    expect(String(system?.content)).toBe(computerGuidance({ search: true }));
+  });
+
+  test("never searches in the browser when no search tool is offered", () => {
+    const [system] = toLangChainMessages(offered("show_app_access"));
+    expect(String(system?.content)).toBe(computerGuidance({ search: false }));
+  });
+});
+
 const assistantAsking = {
   role: "assistant",
   content: "",
@@ -273,7 +294,7 @@ describe("a provider that takes one system prompt", () => {
       expect(system).toHaveLength(1);
       expect(messages[0]).toBe(system[0] as SystemMessage);
       const prompt = String(system[0]?.content);
-      expect(prompt.startsWith(COMPUTER_GUIDANCE)).toBe(true);
+      expect(prompt.startsWith(computerGuidance({ search: false }))).toBe(true);
       const at = instructions.map((text) => prompt.indexOf(text));
       expect(at.every((index) => index > 0)).toBe(true);
       expect(at).toEqual([...at].sort((a, b) => a - b));

app/src/lib/copilot/computer-tools.tsx

@@ -199,9 +199,10 @@ export function ComputerTools() {
     available: offered.navigate,
     name: "computer_navigate",
     description:
-      "Open a web page on your own computer so the person can watch. Use this when asked to look " +
-      "at, visit, open or check a website. Returns the page title and its readable text, so answer " +
-      "from what comes back rather than telling the person to go and look.",
+      "Open a web page on your own computer so the person can watch. Use this when asked to open, " +
+      "visit or read a particular website. It is not a web search: never open a search engine with " +
+      "it to look something up. Returns the page title and its readable text, so answer from what " +
+      "comes back rather than telling the person to go and look.",
     parameters: z.object({
       url: z.string().describe("Full web address to open, including https://"),
     }),

server/src/agents/agent-resolver.ts

@@ -1,4 +1,5 @@
 import type { AbstractAgent } from "@ag-ui/client";
+import type { ComputerGuidance } from "@openbot/shared/bot-prompt";
 import type { AuditInitiator } from "../audit";
 import type { AgentFetch, StallGuard } from "../channels/stall-guard";
 import {
@@ -49,7 +50,7 @@ export type ActorAgentResolverDependencies = {
     initiator?: AuditInitiator,
     depth?: number,
   ) => SignRun;
-  computerGuidance?: string;
+  computerGuidance?: ComputerGuidance;
   loadVendors?: () => Promise<readonly string[]>;
   selectionForActor?: (actorId: string) => ToolSelection;
   agentFetch?: AgentFetch;

server/src/copilot.ts

@@ -6,7 +6,10 @@ import {
   CopilotRuntime,
 } from "@copilotkit/runtime/v2";
 import { createCopilotHonoHandler } from "@copilotkit/runtime/v2/hono";
-import { PROVENANCE_GUIDANCE } from "@openbot/shared/bot-prompt";
+import {
+  type ComputerGuidance,
+  PROVENANCE_GUIDANCE,
+} from "@openbot/shared/bot-prompt";
 import { defaultModelFor } from "@openbot/shared/model-providers";
 import {
   BuiltInAgentWithSaneHistory,
@@ -65,7 +68,7 @@ import {
   SELECTION_FLOOR,
   selectTools,
 } from "./plugins/selection";
-import { grantedToolGuidance } from "./plugins/tools";
+import { grantedToolGuidance, holdsWebSearch } from "./plugins/tools";
 import { UNTRUSTED_GUIDANCE } from "./untrusted-content";
 
 /**
@@ -375,8 +378,11 @@ export function builtInAgentConfiguration(
    * what its hands are. Absent leaves the role alone, which is right for a deployment with no
    * computer configured, where the browser routes are not mounted and a Bot promised a browser would
    * be promising something that does not exist.
+   *
+   * Worded for the tools passed here, which are what this run is offered: whether one of them finds
+   * sources decides whether the browser is told to leave searching to it or never to search at all.
    */
-  computerGuidance?: string,
+  computerGuidance?: ComputerGuidance,
   /**
    * Vendors this deployment connects to, whether or not this Bot holds any of their tools.
    *
@@ -445,7 +451,9 @@ export function builtInAgentConfiguration(
       ...(grantedToolGuidance(tools, connectedVendors)
         ? [grantedToolGuidance(tools, connectedVendors)]
         : []),
-      ...(computerGuidance ? [computerGuidance] : []),
+      ...(computerGuidance
+        ? [computerGuidance({ search: holdsWebSearch(tools) })]
+        : []),
       ...(learnedSkills?.catalog ? [learnedSkills.catalog] : []),
     ].join("\n\n"),
     ...(planModel ? {} : { apiKey: apiKey ?? undefined }),
@@ -501,7 +509,7 @@ export async function buildAgents(
   loadTools: LoadToolsForBot = async () => [],
   signRun?: SignRun,
   /** What every built-in Bot is told about the computer. Absent means this deployment has none. */
-  computerGuidance?: string,
+  computerGuidance?: ComputerGuidance,
   /**
    * Which vendors this deployment connects to. Asked once per build rather than per Bot, because it
    * is a fact about the deployment; what differs per Bot is which of them it holds.
@@ -847,7 +855,7 @@ async function buildAgent(
   stallGuard: StallGuard | undefined,
   loadTools: LoadToolsForBot,
   signRun?: SignRun,
-  computerGuidance?: string,
+  computerGuidance?: ComputerGuidance,
   connectedVendors: readonly string[] = [],
   selection?: ToolSelection,
   agentFetch?: AgentFetch,
@@ -1285,7 +1293,7 @@ function remoteAgentWithStandingRole(
    * the same reason the tools are, so switching a connector on reaches the next run.
    *
    * The remote path needs this more than the built-in one, not less. A framework Bot is handed the
-   * tools as an offer and decides for itself what to call, with `COMPUTER_GUIDANCE` as its whole
+   * tools as an offer and decides for itself what to call, with `computerGuidance` as its whole
    * prompt — a page about the browser that mentions connectors nowhere. That is the Bot that browsed
    * to drive.google.com holding four Drive tools.
    *
@@ -1566,7 +1574,7 @@ export async function resolveRuntimeAgents(
   stallGuard?: StallGuard,
   loadTools?: LoadToolsForBot,
   signRun?: SignRun,
-  computerGuidance?: string,
+  computerGuidance?: ComputerGuidance,
   loadVendors?: () => Promise<readonly string[]>,
   selection?: ToolSelection,
   agentFetch?: AgentFetch,

server/src/index.ts

@@ -3,7 +3,7 @@ import {
   CopilotKitIntelligence,
   IntelligenceAgentRunner,
 } from "@copilotkit/runtime/v2";
-import { COMPUTER_GUIDANCE } from "@openbot/shared/bot-prompt";
+import { computerGuidance } from "@openbot/shared/bot-prompt";
 import { workOwner } from "@openbot/shared/work-owner";
 import { createTurnRunner } from "@openbot/toolkit/run-turn";
 import { and, eq, isNull, ne, sql } from "drizzle-orm";
@@ -1704,7 +1704,7 @@ const actorAgentResolver = createActorAgentResolver({
       proactiveReadOnlyRefs,
     ),
   signRunForActor,
-  computerGuidance: config.computer ? COMPUTER_GUIDANCE : undefined,
+  computerGuidance: config.computer ? computerGuidance : undefined,
   loadVendors,
   selectionForActor,
   agentFetch,

server/src/plugins/tools.ts

@@ -80,7 +80,7 @@ export function vendorAnswer(result: { text: string; isError: boolean }) {
  * A tool array tells a model a tool exists. It does not tell it that the tool is the right way to
  * reach that system, and it competes with a page of prose about the browser that every Bot is given
  * whether or not it has any connectors at all. The browser prose wins: it is emphatic, it is about
- * capability, and it says "never claim you cannot browse".
+ * capability, and it used to say "never claim you cannot browse".
  *
  * So a Bot holding four Google Drive tools browsed to drive.google.com, met a sign-in page its
  * container could never satisfy, and asked its person to sign in to a vendor that person had already
@@ -204,6 +204,17 @@ type ResearchProvider = {
   fetch: GrantedTool;
 };
 
+/**
+ * Whether these tools include one that finds sources, from any research provider.
+ *
+ * The search role alone, not the full pair `researchProviders` wants: a Bot holding a search tool and
+ * no reader can still search, and telling it to drive its browser to a search engine instead would
+ * be the very routing mistake this decides against. See `computerGuidance`.
+ */
+export function holdsWebSearch(tools: readonly GrantedTool[]): boolean {
+  return tools.some((tool) => researchRoleFor(tool.ref) === "search");
+}
+
 /**
  * The research providers this Bot can actually use: a search tool and a fetch tool from one server.
  *