Add native Decision commerce automation and shopping mode settings

ts/packages/agents/browser/package.json

@@ -38,8 +38,9 @@
     "setup:aisearch": "node dist/agent/lookup/aiSearchSetup.mjs",
     "test": "npm run test:local",
     "test:crossword": "node --test test/crosswordDecisionHandler.spec.mjs test/crosswordDecisionIntegrated.spec.mjs test/crosswordHybrid2.spec.mjs",
-    "test:local": "jest && npm run test:crossword && npm run test:views:gateway && npm run test:views:client",
+    "test:local": "jest && npm run test:crossword && npm run test:ui-decision && npm run test:views:gateway && npm run test:views:client",
     "test:pdf": "node --test test/pdf*.spec.mjs test/pdfExtraction/*.spec.mjs test/memoryImportHandlers.spec.mjs test/agentConnectionBoundary.spec.mjs",
+    "test:ui-decision": "node --test test/uiDecision.spec.mjs test/shoppingMode.spec.mjs",
     "test:views:client": "jest --runInBand --testMatch \"**/views/client/**/*.test.ts\"",
     "test:views:gateway": "node --test test/viewGateway.test.mjs test/viewDraftGateway.test.mjs test/viewLifecycle.test.mjs test/memoryHub.test.mjs test/memoryHubQuery.test.mjs test/memoryHubGraph.test.mjs test/memoryHubScopedKnowledge.test.mjs test/memoryHubSourceDependencies.test.mjs test/memoryHubCapture.test.mjs test/browserCaptureRpc.test.mjs test/memoryHubRunbookImports.test.mjs test/memoryHubWebAnalytics.test.mjs test/schemaQuestionRpc.test.mjs test/knowledgeRpc.test.mjs",
     "tsc:agent": "tsc -p src/agent/tsconfig.json",

ts/packages/agents/browser/src/agent/discovery/translator.mts

@@ -240,9 +240,14 @@ export class SchemaDiscoveryAgent<T extends object> {
         userRequest?: string,
         fragments?: HtmlFragments[],
         screenshots?: string[],
+        targetSchema?: string,
     ) {
         const componentsSchema =
-            await getSchemaFileContents("pageComponents.mts");
+            targetSchema === undefined
+                ? await getSchemaFileContents("pageComponents.mts")
+                : targetSchema.trimStart().startsWith("export")
+                  ? targetSchema
+                  : `export type ${componentTypeName} = ${targetSchema};`;
         const bootstrapTranslator = this.getBootstrapTranslator(
             componentTypeName,
             componentsSchema,

ts/packages/agents/browser/src/agent/rpc/externalBrowserControlClient.mts

@@ -34,6 +34,21 @@ export function createExternalBrowserClient(
         }
     }
 
+    async function invokeGroundedUi<T>(invoke: () => Promise<T>): Promise<T> {
+        try {
+            return await invoke();
+        } catch (error) {
+            if (
+                error instanceof Error &&
+                error.message.includes("No invoke handler")
+            )
+                throw new Error(
+                    "unsupported: this browser host does not support grounded UI decisions",
+                );
+            throw error;
+        }
+    }
+
     const control: BrowserControl = {
         openWebPage: async (...args) =>
             getActiveRpc().invoke("openWebPage", ...args),
@@ -51,6 +66,18 @@ export function createExternalBrowserClient(
             invokeCapture(() =>
                 getActiveRpc().invoke("capturePageSnapshot", pageId),
             ),
+        captureUiObservation: (frameScope) =>
+            invokeGroundedUi(() =>
+                getActiveRpc().invoke("captureUiObservation", frameScope),
+            ),
+        validateUiObservation: (snapshotId) =>
+            invokeGroundedUi(() =>
+                getActiveRpc().invoke("validateUiObservation", snapshotId),
+            ),
+        executeUiAction: (...args) =>
+            invokeGroundedUi(() =>
+                getActiveRpc().invoke("executeUiAction", ...args),
+            ),
         setAgentStatus: (...args) =>
             getActiveRpc().send("setAgentStatus", ...args),
         scrollUp: async () => getActiveRpc().invoke("scrollUp"),

ts/packages/agents/browser/src/agent/webFlows/actionHandler.mts

@@ -20,6 +20,7 @@ import { normalizeRecording, RecordingData } from "./recordingNormalizer.mjs";
 import { loadSampleFlows } from "./sampleFlowLoader.mjs";
 import { sendWebFlowRefreshToClient } from "../browserActionHandler.mjs";
 import registerDebug from "debug";
+import { isShoppingMode } from "@typeagent/browser-control-rpc/uiDecisionTypes";
 
 const debug = registerDebug("typeagent:browser:webflows:handler");
 
@@ -133,6 +134,7 @@ function createExtractComponentFn(
             userRequest,
             htmlFragments,
             screenshots,
+            componentDef.schema,
         );
 
         if (!response.success) {
@@ -294,7 +296,7 @@ async function handleDynamicFlowExecution(
     context: SessionContext<BrowserActionContext>,
 ): Promise<WebFlowActionResult> {
     const store = await getStore(context);
-    const flow = await store.get(actionName);
+    let flow = await store.get(actionName);
 
     if (!flow) {
         return {
@@ -304,6 +306,34 @@ async function handleDynamicFlowExecution(
     }
 
     const browser = getBrowserControl(context.agentContext);
+    const decisionName =
+        actionName === "searchForProduct"
+            ? "decisionSearchForProduct"
+            : actionName === "navigateToProductPage"
+              ? "decisionNavigateToProductPage"
+              : undefined;
+    let decisionFlow =
+        actionName === "decisionSearchForProduct" ||
+        actionName === "decisionNavigateToProductPage";
+    if (decisionName) {
+        const { shoppingMode } = await browser.getBrowserSettings();
+        if (shoppingMode !== undefined && !isShoppingMode(shoppingMode)) {
+            throw new Error("Invalid Shopping Mode in browser settings.");
+        }
+        if (shoppingMode === "decision") {
+            const decisionSample = loadSampleFlows().find(
+                (sample) => sample.name === decisionName,
+            );
+            if (!decisionSample) {
+                throw new Error(
+                    `Decision shopping flow "${decisionName}" is unavailable.`,
+                );
+            }
+            // Select an implementation without rewriting saved flows or their scope.
+            flow = { ...decisionSample, scope: flow.scope };
+            decisionFlow = true;
+        }
+    }
 
     // Validate scope against current page
     if (flow.scope.type === "site" && flow.scope.domains?.length) {
@@ -371,12 +401,16 @@ async function handleDynamicFlowExecution(
             displayText: `WebFlow "${actionName}" failed: ${errorMsg}`,
             data: result,
             error: errorMsg,
-            fallbackToReasoning: true,
-            fallbackContext: {
-                failedFlow: actionName,
-                errorMessage: errorMsg,
-                hint: `Use editWebFlow to fix the script for "${actionName}", or regenerate the flow with a clearer description.`,
-            },
+            fallbackToReasoning: !decisionFlow,
+            ...(decisionFlow
+                ? {}
+                : {
+                      fallbackContext: {
+                          failedFlow: actionName,
+                          errorMessage: errorMsg,
+                          hint: `Use editWebFlow to fix the script for "${actionName}", or regenerate the flow with a clearer description.`,
+                      },
+                  }),
         };
     }
 
@@ -393,7 +427,11 @@ async function handleStartGoalDrivenTask(
     const browser = getBrowserControl(context.agentContext);
     const { goal, startUrl, maxSteps } = action.parameters;
 
-    const browserApi = new WebFlowBrowserAPIImpl(browser);
+    const browserApi = new WebFlowBrowserAPIImpl(
+        browser,
+        undefined,
+        createExtractComponentFn(context),
+    );
     const agent = new BrowserReasoningAgent(browserApi, {
         onThinking: (text) => debug(`[thinking] ${text.slice(0, 100)}...`),
         onToolCall: (tool, args) => debug(`[tool_call] ${tool}`),

ts/packages/agents/browser/src/agent/webFlows/samples/decisionNavigateToProductPage.json

@@ -0,0 +1,26 @@
+{
+  "name": "decisionNavigateToProductPage",
+  "description": "Opt-in native decision pilot: select a main-document product detail link and verify the main-document destination. Iframe content is outside this flow's evidence. Does not add items or purchase.",
+  "version": 3,
+  "parameters": {
+    "productName": {
+      "type": "string",
+      "required": true,
+      "description": "The product and required variant to open"
+    },
+    "navigationTimeoutMs": {
+      "type": "number",
+      "required": false,
+      "description": "Optional navigation wait budget in milliseconds; omitted preserves the browser host default."
+    }
+  },
+  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    const link = await browser.selectElement({\n        condition: 'The main-document product detail link for ' + JSON.stringify(params.productName) + ', not an add-to-cart, checkout, sponsored alternative or unrelated recommendation control',\n        capability: 'activate',\n        frameScope: 'main-document'\n    });\n    if (link.role !== 'link') {\n        throw new Error('The selected product detail control is not a link; no action attempted');\n    }\n    await browser.executeUiAction(link.targetId, { type: 'activate' });\n    await browser.awaitPageLoad(params.navigationTimeoutMs);\n    const result = await browser.evaluatePageCondition('The main document shows the primary product detail page for ' + JSON.stringify(params.productName) + ', including its requested variant, rather than a search listing or recommendation.', 'main-document');\n    if (result.status !== 'matched') {\n        throw new Error('Product destination was not verified (' + result.status + '). Navigation may have occurred; no retry.');\n    }\n    return { success: true, message: 'Verified main-document product page for ' + params.productName };\n}",
+  "grammarPatterns": ["decision open product $(productName:wildcard)"],
+  "scope": {
+    "type": "global"
+  },
+  "source": {
+    "type": "manual",
+    "timestamp": "2026-10-25T00:00:00.000Z"
+  }
+}

ts/packages/agents/browser/src/agent/webFlows/samples/decisionSearchForProduct.json

@@ -0,0 +1,31 @@
+{
+  "name": "decisionSearchForProduct",
+  "description": "Opt-in native decision pilot: find a main-document product search input, submit through its native form, and verify main-document results. Iframe content is outside this flow's evidence. Does not add items or purchase.",
+  "version": 4,
+  "parameters": {
+    "productName": {
+      "type": "string",
+      "required": true,
+      "description": "The exact product search query to enter"
+    },
+    "selectionCriteria": {
+      "type": "string",
+      "required": false,
+      "description": "Optional search context, such as price range or brand, used when locating the search input"
+    },
+    "navigationTimeoutMs": {
+      "type": "number",
+      "required": false,
+      "description": "Optional navigation wait budget in milliseconds; omitted preserves the browser host default."
+    }
+  },
+  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    const input = await browser.selectElement({\n        condition: 'The product search input in the main document, not a newsletter, account or address field' + (params.selectionCriteria ? '. Search request context: ' + JSON.stringify(params.productName + ' ' + params.selectionCriteria) : ''),\n        capability: 'fill',\n        frameScope: 'main-document'\n    });\n    await browser.executeUiAction(input.targetId, { type: 'fill', text: params.productName, submit: true });\n    await browser.awaitPageLoad(params.navigationTimeoutMs);\n    const result = await browser.evaluatePageCondition('The main document displays search results for the query ' + JSON.stringify(params.productName) + ', or an explicit no-results message for that query.', 'main-document');\n    if (result.status !== 'matched') {\n        throw new Error('Search result was not verified (' + result.status + '). The form may have been submitted; no retry.');\n    }\n    return { success: true, message: 'Verified main-document search results for ' + params.productName };\n}",
+  "grammarPatterns": ["decision search for $(productName:wildcard)"],
+  "scope": {
+    "type": "global"
+  },
+  "source": {
+    "type": "manual",
+    "timestamp": "2026-10-25T00:00:00.000Z"
+  }
+}

ts/packages/agents/browser/src/agent/webFlows/samples/navigateToProductPage.json

@@ -1,20 +1,30 @@
 {
   "name": "navigateToProductPage",
   "description": "Navigate to a product details page from a listing or search results",
-  "version": 1,
+  "version": 2,
   "parameters": {
     "productName": {
       "type": "string",
       "required": true,
       "description": "The name of the product to view"
+    },
+    "navigationTimeoutMs": {
+      "type": "number",
+      "required": false,
+      "description": "Optional navigation wait budget in milliseconds; omitted preserves the browser host default."
     }
   },
-  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    const productTile = await browser.extractComponent<{ name: string; price: string; detailsLinkSelector: string }>(\n        {\n            typeName: 'ProductTile',\n            schema: '{ name: string; price: string; detailsLinkSelector: string; }'\n        },\n        params.productName\n    );\n    if (!productTile || !productTile.detailsLinkSelector) {\n        throw new Error('Could not find product: ' + params.productName);\n    }\n    await browser.click(productTile.detailsLinkSelector);\n    await browser.awaitPageLoad();\n    return { success: true, message: 'Navigated to ' + productTile.name };\n}",
+  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    const productTile = await browser.extractComponent<{ name: string; price: string; detailsLinkSelector: string }>(\n        {\n            typeName: 'ProductTile',\n            schema: '{ name: string; price: string; detailsLinkSelector: string; }'\n        },\n        params.productName\n    );\n    if (!productTile || !productTile.detailsLinkSelector) {\n        throw new Error('Could not find product: ' + params.productName);\n    }\n    await browser.click(productTile.detailsLinkSelector);\n    await browser.awaitPageLoad(params.navigationTimeoutMs);\n    return { success: true, message: 'Navigated to ' + productTile.name };\n}",
   "grammarPatterns": [
     "view $(productName:wildcard) details",
     "go to $(productName:wildcard) product page",
     "open $(productName:wildcard)"
   ],
-  "scope": { "type": "global" },
-  "source": { "type": "manual", "timestamp": "2026-03-13T00:00:00.000Z" }
+  "scope": {
+    "type": "global"
+  },
+  "source": {
+    "type": "manual",
+    "timestamp": "2026-03-13T00:00:00.000Z"
+  }
 }

ts/packages/agents/browser/src/agent/webFlows/samples/searchForProduct.json

@@ -1,7 +1,7 @@
 {
   "name": "searchForProduct",
   "description": "Search for a product on an e-commerce website",
-  "version": 1,
+  "version": 2,
   "parameters": {
     "productName": {
       "type": "string",
@@ -12,15 +12,25 @@
       "type": "string",
       "required": false,
       "description": "Optional criteria like price range or brand"
+    },
+    "navigationTimeoutMs": {
+      "type": "number",
+      "required": false,
+      "description": "Optional navigation wait budget in milliseconds; omitted preserves the browser host default."
     }
   },
-  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    const searchInput = await browser.extractComponent<{ cssSelector: string; submitButtonCssSelector: string }>({\n        typeName: 'SearchInput',\n        schema: '{ cssSelector: string; submitButtonCssSelector: string; }'\n    }, params.selectionCriteria ? params.productName + ' ' + params.selectionCriteria : params.productName);\n    if (!searchInput || !searchInput.cssSelector) {\n        throw new Error('Could not find search input on this page');\n    }\n    await browser.clearAndType(searchInput.cssSelector, params.productName);\n    if (searchInput.submitButtonCssSelector) {\n        await browser.click(searchInput.submitButtonCssSelector);\n    } else {\n        await browser.pressKey('Enter');\n    }\n    await browser.awaitPageLoad();\n    return { success: true, message: 'Searched for ' + params.productName };\n}",
+  "script": "async function execute(browser: WebFlowBrowserAPI, params: FlowParams): Promise<WebFlowResult> {\n    const searchInput = await browser.extractComponent<{ cssSelector: string; submitButtonCssSelector: string }>({\n        typeName: 'SearchInput',\n        schema: '{ cssSelector: string; submitButtonCssSelector: string; }'\n    }, params.selectionCriteria ? params.productName + ' ' + params.selectionCriteria : params.productName);\n    if (!searchInput || !searchInput.cssSelector) {\n        throw new Error('Could not find search input on this page');\n    }\n    await browser.clearAndType(searchInput.cssSelector, params.productName);\n    if (searchInput.submitButtonCssSelector) {\n        await browser.click(searchInput.submitButtonCssSelector);\n    } else {\n        await browser.pressKey('Enter');\n    }\n    await browser.awaitPageLoad(params.navigationTimeoutMs);\n    return { success: true, message: 'Searched for ' + params.productName };\n}",
   "grammarPatterns": [
     "search for $(productName:wildcard)",
     "find $(productName:wildcard)",
     "look for $(productName:wildcard)",
     "search $(productName:wildcard)"
   ],
-  "scope": { "type": "global" },
-  "source": { "type": "manual", "timestamp": "2026-03-13T00:00:00.000Z" }
+  "scope": {
+    "type": "global"
+  },
+  "source": {
+    "type": "manual",
+    "timestamp": "2026-03-13T00:00:00.000Z"
+  }
 }

ts/packages/agents/browser/src/agent/webFlows/schema/browserApi.mts

@@ -39,8 +39,42 @@ export interface WebFlowBrowserMethods {
         componentDef: ComponentDef,
         userRequest?: string,
     ): Promise<T>;
+
+    // Opt-in native decisions; require extension support and never fall back to chat.
+    selectElement(request: {
+        condition: string;
+        capability: "activate" | "fill" | "select";
+        frameScope?: "all-frames" | "main-document";
+    }): Promise<UiTarget>;
+    evaluatePageCondition(
+        condition: string,
+        frameScope?: "all-frames" | "main-document",
+    ): Promise<{
+        status: "matched" | "not_matched" | "insufficient_evidence";
+        explanation: string;
+    }>;
+    // A target is single-use. Dispatch is not proof of a business postcondition.
+    executeUiAction(
+        targetId: string,
+        action: UiAction,
+    ): Promise<{
+        dispatched: true;
+        valueVerified?: true;
+    }>;
 }
 
+export type UiTarget = {
+    targetId: string;
+    name: string;
+    role: string;
+    options?: { id: string; text: string; disabled: boolean }[];
+};
+
+export type UiAction =
+    | { type: "activate" }
+    | { type: "fill"; text: string; submit?: boolean }
+    | { type: "select"; optionId: string };
+
 export type ComponentDef = {
     typeName: string;
     schema: string;