diff --git a/CHANGELOG.md b/CHANGELOG.md index bd80d60..fd5250d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Mobile `click` never reports a tap that hit something else: device-mcp refuses a tap when another element covers the target, and the click fails with the new `MM_TARGET_OBSCURED` (`MM_TARGET_NOT_VISIBLE` when the target is off screen). `click --dismiss-keyboard` closes a covering keyboard and taps once more (`keyboardDismissed: true`). With an observation, a mobile click result has `screenChanged`. The hit-test needs a device-mcp release with MetaMask/device-mcp#51. - Start the CLI faster: find the project root without running `git` and do not load zod. Load `@metamask/device-mcp` only when a mobile session uses it, so browser daemons start without it. - **BREAKING:** `scroll_to_element` `direction` is now the content direction (`down` reveals content below; it was the finger direction) and defaults to `auto`. The result adds `swipes`, `inViewport` and `bounds`; a target that is found but not brought into view fails with the new `MM_TARGET_NOT_VISIBLE`, and one never found with `MM_TARGET_NOT_FOUND`. The viewport check needs a device-mcp release with MetaMask/device-mcp#50; with device-mcp 0.7.0 only `bounds` is returned. - Mobile `click` and `type` tap the element that the target wait found, so a tap takes one device snapshot instead of two (needs a device-mcp release with `tapElement(query, { element })`; older releases ignore the option). `describe_screen` reads the app state and the screen at the same time on mobile. diff --git a/README.md b/README.md index 66d0425..31348eb 100644 --- a/README.md +++ b/README.md @@ -624,16 +624,16 @@ mm describe-screen ### Interaction -| Command | Description | -| --------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `mm click [--timeout ] [--selector ] [--testid ] [--within ]` | Clicks an element by its accessibility reference (e.g., `e3`). The ref comes from a prior `describe-screen` call. Waits for the element to be visible before clicking. The `--timeout` covers the entire operation (visibility wait + click action). Default: 15s. Use `--within` to scope the target inside a parent element (`testid:`, `selector:`, or a bare a11y ref). | -| `mm type [--timeout ] [--selector ] [--testid ] [--within ]` | Types text into an input element identified by its accessibility reference. Clears the field first, then sets the new value (uses Playwright's `fill()`). Accepts `--timeout` for the total time budget (default: 15s). Use `--within` to scope the target inside a parent element. | -| `mm get-text [--timeout ] [--selector ] [--testid ] [--within ]` | Reads the text content of an element. Returns the inner text, target descriptor, and character length. Accepts `--timeout` for the total time budget (default: 15s). Useful for asserting visible values without screenshots. | -| `mm describe-screen` | Captures the full screen state: extension info, visible test IDs, and a trimmed accessibility tree with deterministic refs (`e1`, `e2`, ...). This is the primary command for understanding what's on screen before interacting. | -| `mm screenshot [--name ]` | Takes a full-page screenshot of the current page. Saves to the artifacts directory. Use `--name` to set a descriptive filename. | -| `mm wait-for [--timeout ] [--selector ] [--testid ] [--within ]` | Blocks until an element identified by its accessibility reference becomes visible, or the timeout expires. Default timeout is 15 seconds. Use `--within` to scope the target inside a parent element. | -| `mm wait-for-notification [--timeout ]` | Waits for the extension notification popup to appear within a timeout. Returns the notification page URL. | -| `mm clipboard [text]` | Reads from or writes to the system clipboard via Chrome DevTools Protocol. Useful for pasting seed phrases or copying addresses. | +| Command | Description | +| ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `mm click [--timeout ] [--selector ] [--testid ] [--within ] [--dismiss-keyboard]` | Clicks an element by its accessibility reference (e.g., `e3`). The ref comes from a prior `describe-screen` call. Waits for the element to be visible before clicking. The `--timeout` covers the entire operation (visibility wait + click action). Default: 15s. Use `--within` to scope the target inside a parent element (`testid:`, `selector:`, or a bare a11y ref). Mobile: fails with `MM_TARGET_OBSCURED` when something covers the target; `--dismiss-keyboard` closes a covering keyboard and taps again. | +| `mm type [--timeout ] [--selector ] [--testid ] [--within ]` | Types text into an input element identified by its accessibility reference. Clears the field first, then sets the new value (uses Playwright's `fill()`). Accepts `--timeout` for the total time budget (default: 15s). Use `--within` to scope the target inside a parent element. | +| `mm get-text [--timeout ] [--selector ] [--testid ] [--within ]` | Reads the text content of an element. Returns the inner text, target descriptor, and character length. Accepts `--timeout` for the total time budget (default: 15s). Useful for asserting visible values without screenshots. | +| `mm describe-screen` | Captures the full screen state: extension info, visible test IDs, and a trimmed accessibility tree with deterministic refs (`e1`, `e2`, ...). This is the primary command for understanding what's on screen before interacting. | +| `mm screenshot [--name ]` | Takes a full-page screenshot of the current page. Saves to the artifacts directory. Use `--name` to set a descriptive filename. | +| `mm wait-for [--timeout ] [--selector ] [--testid ] [--within ]` | Blocks until an element identified by its accessibility reference becomes visible, or the timeout expires. Default timeout is 15 seconds. Use `--within` to scope the target inside a parent element. | +| `mm wait-for-notification [--timeout ]` | Waits for the extension notification popup to appear within a timeout. Returns the notification page URL. | +| `mm clipboard [text]` | Reads from or writes to the system clipboard via Chrome DevTools Protocol. Useful for pasting seed phrases or copying addresses. | ### Navigation @@ -801,6 +801,8 @@ Tool errors are classified into specific error codes for structured handling: | `MM_TARGET_NOT_FOUND` | Element not found by ref, testId, or selector | | `MM_WAIT_TIMEOUT` | Timeout waiting for element visibility | | `MM_CLICK_FAILED` | Click operation failed | +| `MM_TARGET_OBSCURED` | Mobile: another element (keyboard, sheet, overlay) covers the target; nothing was tapped | +| `MM_TARGET_NOT_VISIBLE` | Mobile: the target is outside the viewport (tap or `scroll_to_element`) | | `MM_CLICK_TIMEOUT` | Click action timed out (element found, click hung) | | `MM_TYPE_FAILED` | Type operation failed | | `MM_TYPE_TIMEOUT` | Fill action timed out | diff --git a/SKILL.md b/SKILL.md index 75e867a..f5a6811 100644 --- a/SKILL.md +++ b/SKILL.md @@ -302,6 +302,12 @@ Use `--within` to scope the target inside a parent element. Values use the forma If the page closes after clicking (e.g., confirmation popup), the response includes `pageClosedAfterClick: true` — this is normal, not an error. +**Mobile:** before tapping, iOS hit-tests the tap point. When another element is on top of the target (the keyboard, a sheet, an overlay), nothing is tapped and the click fails with `MM_TARGET_OBSCURED`; the message names the covering element. A target outside the screen fails with `MM_TARGET_NOT_VISIBLE` (run `mm scroll-to-element` first). `--dismiss-keyboard` closes the keyboard when it is the cover and taps once more (the result has `keyboardDismissed: true`). With an observation (not `--observe none`) the result has `screenChanged`: `false` means the screen looks the same as before the click; that is normal for some taps (the open tab) but a hint for a button that should navigate. + +``` +mm click --testid log-in-button --dismiss-keyboard +``` + **Timeout behavior:** If the click hangs (e.g., element found but click never resolves due to a side effect), `MM_CLICK_TIMEOUT` is returned. The click may still complete in the background — run `mm describe-screen` to verify current state before retrying. #### `mm type ` @@ -681,49 +687,51 @@ Every interaction command (`click`, `type`, `get-text`, `wait-for`) needs a targ When a command fails, the response includes `error.code`. Use this to decide what to do: -| Code | Meaning | Recovery | -| ----------------------------------- | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| `MM_NO_ACTIVE_SESSION` | No browser session running | Run `mm launch` first | -| `MM_SESSION_ALREADY_RUNNING` | Session already exists | Run `mm cleanup` first, or use `--force` | -| `MM_LAUNCH_FAILED` | Browser session launch failed | Check extension path and config; retry | -| `MM_PAGE_CLOSED` | Page was closed unexpectedly | Normal after some confirmations; run describe-screen | -| `MM_BUILD_FAILED` | Extension build failed | Check build logs; fix build errors and retry | -| `MM_DEPENDENCIES_MISSING` | Required build dependencies not installed | Run dependency install (npm/yarn) and retry build | -| `MM_TARGET_NOT_FOUND` | Element ref/testId/selector not found | Run `mm describe-screen` to get fresh refs | -| `MM_WAIT_TIMEOUT` | Element didn't appear in time | Increase timeout or verify you're on the right screen | -| `MM_CLICK_FAILED` | Click failed after finding element | Element may be obscured; try waiting or scrolling | -| `MM_CLICK_TIMEOUT` | Click action timed out (element found, click hung) | Run `mm describe-screen` to verify if click completed; retry with `--timeout` or different approach | -| `MM_TYPE_FAILED` | Type failed after finding element | Element may not be an input; verify with describe-screen | -| `MM_TYPE_TIMEOUT` | Fill action timed out | Run `mm describe-screen` to verify state; retry with `--timeout` | -| `MM_GETTEXT_FAILED` | getText operational failure (non-timeout) | Element may be detached; run `mm describe-screen` and re-target | -| `MM_GETTEXT_TIMEOUT` | textContent action timed out | Retry with `--timeout` | -| `MM_CLIPBOARD_PERMISSION_DENIED` | Clipboard permission denied by browser | Check browser permissions; try CDP approach | -| `MM_CLIPBOARD_LAVAMOAT_BLOCKED` | Clipboard blocked by LavaMoat policy | Extension security policy blocks clipboard; use alternative input method | -| `MM_CLIPBOARD_FAILED` | Clipboard operation failed | Retry; check if page is still active | -| `MM_NAVIGATION_FAILED` | Navigation error or network failure | Check URL validity; retry once | -| `MM_NOTIFICATION_TIMEOUT` | Extension notification popup didn't appear | Action may not have triggered a notification; check state | -| `MM_TAB_NOT_FOUND` | Tab role/URL not found | Run `mm get-state` to see available tabs | -| `MM_DISCOVERY_FAILED` | Discovery tool failure | Page may be loading; wait and retry | -| `MM_SCREENSHOT_FAILED` | Screenshot capture failure | Page may be in unstable state; retry after describe-screen | -| `MM_STATE_FAILED` | State retrieval failed | Session may be unstable; run `mm describe-screen` | -| `MM_CONTRACT_NOT_FOUND` | Unknown contract name for seeding | See available contracts below | -| `MM_SEED_FAILED` | Contract deployment failure | Check Anvil chain is running; verify contract name | -| `MM_CAPABILITY_NOT_AVAILABLE` | Feature requires a capability not configured | Check environment mode (e2e vs prod) | -| `MM_CONTEXT_SWITCH_BLOCKED` | Can't switch context with active session | Run `mm cleanup` first | -| `MM_SET_CONTEXT_FAILED` | Context switch operation failed | Retry; check session state | -| `MM_INVALID_INPUT` | Bad parameters | Fix input and retry | -| `MM_INVALID_CONFIG` | Invalid configuration | Check config file format and required fields | -| `MM_PORT_IN_USE` | Port already in use | Stop conflicting process or let the daemon auto-allocate | -| `MM_UNKNOWN_TOOL` | Unknown tool name | Check tool name spelling | -| `MM_INTERNAL_ERROR` | Internal server error | Retry; if persistent, restart daemon with `mm stop && mm serve` | -| `MM_BATCH_TIMEOUT` | `batchTimeoutMs` deadline exceeded | Remaining steps were skipped; check partial results | -| `MM_CDP_BLOCKED` | CDP method is blocked (destructive) | Use a different CDP method; blocked list differs by platform (see `cdp`) | -| `MM_CDP_FAILED` | CDP command failed or timed out (mobile: `HERMES_*` code in message) | Check method/params; on mobile ensure a DEBUG build with Metro is running, then `mm hermes-targets --all` | -| `MM_HERMES_FAILED` | `hermes_targets` discovery failed (`HERMES_*` code in message) | Ensure a DEBUG build with Metro is running; run `mm hermes-targets --all` to diagnose | -| `MM_HERMES_NOT_AVAILABLE` | `hermes_targets` used outside a mobile session | Launch a mobile (iOS/Android) session first | -| `MM_DEVICE_ACTION_FAILED` | A mobile device action failed | Check the target/args; ensure the device/emulator is responsive | -| `MM_DEVICE_NOT_AVAILABLE` | Mobile-only tool invoked without a mobile driver | Launch a mobile (iOS/Android) session first | -| `MM_TOOL_NOT_SUPPORTED_ON_PLATFORM` | Tool gated off the active platform | Use the tool on its supported platform (browser vs mobile) | +| Code | Meaning | Recovery | +| ----------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| `MM_NO_ACTIVE_SESSION` | No browser session running | Run `mm launch` first | +| `MM_SESSION_ALREADY_RUNNING` | Session already exists | Run `mm cleanup` first, or use `--force` | +| `MM_LAUNCH_FAILED` | Browser session launch failed | Check extension path and config; retry | +| `MM_PAGE_CLOSED` | Page was closed unexpectedly | Normal after some confirmations; run describe-screen | +| `MM_BUILD_FAILED` | Extension build failed | Check build logs; fix build errors and retry | +| `MM_DEPENDENCIES_MISSING` | Required build dependencies not installed | Run dependency install (npm/yarn) and retry build | +| `MM_TARGET_NOT_FOUND` | Element ref/testId/selector not found | Run `mm describe-screen` to get fresh refs | +| `MM_WAIT_TIMEOUT` | Element didn't appear in time | Increase timeout or verify you're on the right screen | +| `MM_CLICK_FAILED` | Click failed after finding element | Element may be obscured; try waiting or scrolling | +| `MM_TARGET_OBSCURED` | Mobile: another element (keyboard, sheet, overlay) covers the target; nothing was tapped | Keyboard: `mm dismiss-keyboard` or `mm click ... --dismiss-keyboard`. Overlay: close it first | +| `MM_TARGET_NOT_VISIBLE` | Mobile: target found but outside the screen | `mm scroll-to-element `, then click | +| `MM_CLICK_TIMEOUT` | Click action timed out (element found, click hung) | Run `mm describe-screen` to verify if click completed; retry with `--timeout` or different approach | +| `MM_TYPE_FAILED` | Type failed after finding element | Element may not be an input; verify with describe-screen | +| `MM_TYPE_TIMEOUT` | Fill action timed out | Run `mm describe-screen` to verify state; retry with `--timeout` | +| `MM_GETTEXT_FAILED` | getText operational failure (non-timeout) | Element may be detached; run `mm describe-screen` and re-target | +| `MM_GETTEXT_TIMEOUT` | textContent action timed out | Retry with `--timeout` | +| `MM_CLIPBOARD_PERMISSION_DENIED` | Clipboard permission denied by browser | Check browser permissions; try CDP approach | +| `MM_CLIPBOARD_LAVAMOAT_BLOCKED` | Clipboard blocked by LavaMoat policy | Extension security policy blocks clipboard; use alternative input method | +| `MM_CLIPBOARD_FAILED` | Clipboard operation failed | Retry; check if page is still active | +| `MM_NAVIGATION_FAILED` | Navigation error or network failure | Check URL validity; retry once | +| `MM_NOTIFICATION_TIMEOUT` | Extension notification popup didn't appear | Action may not have triggered a notification; check state | +| `MM_TAB_NOT_FOUND` | Tab role/URL not found | Run `mm get-state` to see available tabs | +| `MM_DISCOVERY_FAILED` | Discovery tool failure | Page may be loading; wait and retry | +| `MM_SCREENSHOT_FAILED` | Screenshot capture failure | Page may be in unstable state; retry after describe-screen | +| `MM_STATE_FAILED` | State retrieval failed | Session may be unstable; run `mm describe-screen` | +| `MM_CONTRACT_NOT_FOUND` | Unknown contract name for seeding | See available contracts below | +| `MM_SEED_FAILED` | Contract deployment failure | Check Anvil chain is running; verify contract name | +| `MM_CAPABILITY_NOT_AVAILABLE` | Feature requires a capability not configured | Check environment mode (e2e vs prod) | +| `MM_CONTEXT_SWITCH_BLOCKED` | Can't switch context with active session | Run `mm cleanup` first | +| `MM_SET_CONTEXT_FAILED` | Context switch operation failed | Retry; check session state | +| `MM_INVALID_INPUT` | Bad parameters | Fix input and retry | +| `MM_INVALID_CONFIG` | Invalid configuration | Check config file format and required fields | +| `MM_PORT_IN_USE` | Port already in use | Stop conflicting process or let the daemon auto-allocate | +| `MM_UNKNOWN_TOOL` | Unknown tool name | Check tool name spelling | +| `MM_INTERNAL_ERROR` | Internal server error | Retry; if persistent, restart daemon with `mm stop && mm serve` | +| `MM_BATCH_TIMEOUT` | `batchTimeoutMs` deadline exceeded | Remaining steps were skipped; check partial results | +| `MM_CDP_BLOCKED` | CDP method is blocked (destructive) | Use a different CDP method; blocked list differs by platform (see `cdp`) | +| `MM_CDP_FAILED` | CDP command failed or timed out (mobile: `HERMES_*` code in message) | Check method/params; on mobile ensure a DEBUG build with Metro is running, then `mm hermes-targets --all` | +| `MM_HERMES_FAILED` | `hermes_targets` discovery failed (`HERMES_*` code in message) | Ensure a DEBUG build with Metro is running; run `mm hermes-targets --all` to diagnose | +| `MM_HERMES_NOT_AVAILABLE` | `hermes_targets` used outside a mobile session | Launch a mobile (iOS/Android) session first | +| `MM_DEVICE_ACTION_FAILED` | A mobile device action failed | Check the target/args; ensure the device/emulator is responsive | +| `MM_DEVICE_NOT_AVAILABLE` | Mobile-only tool invoked without a mobile driver | Launch a mobile (iOS/Android) session first | +| `MM_TOOL_NOT_SUPPORTED_ON_PLATFORM` | Tool gated off the active platform | Use the tool on its supported platform (browser vs mobile) | ## Available Contracts (E2E only) diff --git a/src/cli/mm.test.ts b/src/cli/mm.test.ts index 8c063ed..8db49fa 100644 --- a/src/cli/mm.test.ts +++ b/src/cli/mm.test.ts @@ -1737,6 +1737,33 @@ describe('routeCommand', () => { ); }); + it('routes click with --dismiss-keyboard', async () => { + await routeCommand( + 'click', + ['--testid', 'log-in-button', '--dismiss-keyboard'], + 3000, + ); + expect(globalThis.fetch).toHaveBeenCalledWith( + 'http://127.0.0.1:3000/tool/click', + expect.objectContaining({ + body: JSON.stringify({ + testId: 'log-in-button', + dismissKeyboard: true, + }), + }), + ); + }); + + it('does not read --dismiss-keyboard as the click target', async () => { + await routeCommand('click', ['--dismiss-keyboard', 'e3'], 3000); + expect(globalThis.fetch).toHaveBeenCalledWith( + 'http://127.0.0.1:3000/tool/click', + expect.objectContaining({ + body: JSON.stringify({ a11yRef: 'e3', dismissKeyboard: true }), + }), + ); + }); + it('routes click with --selector', async () => { await routeCommand('click', ['--selector', '.btn'], 3000); expect(globalThis.fetch).toHaveBeenCalledWith( diff --git a/src/cli/mm.ts b/src/cli/mm.ts index ff0b4cd..e3cfc51 100644 --- a/src/cli/mm.ts +++ b/src/cli/mm.ts @@ -579,11 +579,18 @@ export async function routeCommand( !args.includes('--testid') ) { process.stderr.write( - 'Usage: mm click [--timeout ] [--selector ] [--testid ] [--within ]\n', + 'Usage: mm click [--timeout ] [--selector ] [--testid ] [--within ] [--dismiss-keyboard]\n', ); process.exit(1); } - await sendInteractionRequest(port, '/tool/click', args); + await sendInteractionRequest( + port, + '/tool/click', + args.filter((arg) => arg !== '--dismiss-keyboard'), + args.includes('--dismiss-keyboard') + ? { dismissKeyboard: true } + : undefined, + ); break; } case 'type': { diff --git a/src/platform/mobile-platform-driver.test.ts b/src/platform/mobile-platform-driver.test.ts index e97eeca..2cf4bb1 100644 --- a/src/platform/mobile-platform-driver.test.ts +++ b/src/platform/mobile-platform-driver.test.ts @@ -464,6 +464,126 @@ describe('MobilePlatformDriver', () => { }); describe('click', () => { + const obscured = (keyboard: boolean): Error => + Object.assign(new Error('TARGET_OBSCURED: covered'), { + code: 'TARGET_OBSCURED', + keyboard, + }); + + it('dismisses the keyboard and taps again with dismissKeyboard', async () => { + const before = makeElement({ identifier: 'log-in-button' }); + const after = makeElement({ + identifier: 'log-in-button', + frame: { x: 16, y: 700, width: 370, height: 48 }, + }); + const backend = createMockBackend({ + waitForElement: vi + .fn() + .mockResolvedValueOnce(before) + .mockResolvedValueOnce(after), + tapElement: vi + .fn() + .mockRejectedValueOnce(obscured(true)) + .mockResolvedValueOnce({ + success: true, + x: 1, + y: 2, + targetDescription: '', + }), + }); + const driver = new MobilePlatformDriver(backend); + + const result = await driver.click( + 'testId', + 'log-in-button', + new Map(), + 5000, + undefined, + undefined, + { dismissKeyboard: true }, + ); + + expect(result).toStrictEqual({ + clicked: true, + target: 'testId:log-in-button', + keyboardDismissed: true, + }); + expect(backend.dismissKeyboard).toHaveBeenCalledTimes(1); + // The second tap uses the element read after the keyboard closed. + expect(vi.mocked(backend.tapElement).mock.calls).toStrictEqual([ + [{ identifier: 'log-in-button' }, { element: before }], + [{ identifier: 'log-in-button' }, { element: after }], + ]); + }); + + it('does not dismiss the keyboard without the option', async () => { + const backend = createMockBackend({ + tapElement: vi.fn().mockRejectedValue(obscured(true)), + }); + const driver = new MobilePlatformDriver(backend); + + await expect( + driver.click('testId', 'log-in-button', new Map(), 5000), + ).rejects.toMatchObject({ code: 'TARGET_OBSCURED' }); + expect(backend.dismissKeyboard).not.toHaveBeenCalled(); + expect(backend.tapElement).toHaveBeenCalledTimes(1); + }); + + it('does not dismiss the keyboard when an overlay (not the keyboard) covers the target', async () => { + const backend = createMockBackend({ + tapElement: vi.fn().mockRejectedValue(obscured(false)), + }); + const driver = new MobilePlatformDriver(backend); + + await expect( + driver.click('testId', 'x', new Map(), 5000, undefined, undefined, { + dismissKeyboard: true, + }), + ).rejects.toMatchObject({ code: 'TARGET_OBSCURED', keyboard: false }); + expect(backend.dismissKeyboard).not.toHaveBeenCalled(); + }); + + it('fails when the target is still covered after the keyboard is dismissed', async () => { + const backend = createMockBackend({ + tapElement: vi.fn().mockRejectedValue(obscured(true)), + }); + const driver = new MobilePlatformDriver(backend); + + await expect( + driver.click('testId', 'x', new Map(), 5000, undefined, undefined, { + dismissKeyboard: true, + }), + ).rejects.toMatchObject({ code: 'TARGET_OBSCURED' }); + expect(backend.dismissKeyboard).toHaveBeenCalledTimes(1); + expect(backend.tapElement).toHaveBeenCalledTimes(2); + }); + + it('does not tap again when the request is aborted while dismissing', async () => { + const controller = new AbortController(); + const backend = createMockBackend({ + tapElement: vi.fn().mockRejectedValue(obscured(true)), + dismissKeyboard: vi.fn(async () => { + controller.abort(); + }), + }); + const driver = new MobilePlatformDriver(backend); + + await expect( + driver.click( + 'testId', + 'x', + new Map(), + 5000, + undefined, + controller.signal, + { + dismissKeyboard: true, + }, + ), + ).rejects.toThrowError('This operation was aborted'); + expect(backend.tapElement).toHaveBeenCalledTimes(1); + }); + it('taps the element that waitForElement returned (one snapshot)', async () => { const found = makeElement({ identifier: 'submit-btn', diff --git a/src/platform/mobile-platform-driver.ts b/src/platform/mobile-platform-driver.ts index 626d251..5efb52a 100644 --- a/src/platform/mobile-platform-driver.ts +++ b/src/platform/mobile-platform-driver.ts @@ -17,6 +17,7 @@ import type { IPlatformDriver, TargetType, ClickActionResult, + ClickOptions, TypeActionResult, GetTextActionResult, PlatformScreenshotOptions, @@ -143,6 +144,11 @@ export class MobilePlatformDriver implements IPlatformDriver { * @param signal - When aborted during the target wait, throws instead of * tapping. device-mcp takes no signal: a wait or tap that is already * running inside the backend is not cancelled. + * @param options - Click options. + * @param options.dismissKeyboard - When device-mcp refuses the tap because + * the keyboard covers the target (`TARGET_OBSCURED`, `keyboard: true`), + * dismiss the keyboard and tap once more. The element is looked up again, + * because closing the keyboard can move the layout. * @returns Click result with success status and target info. */ async click( @@ -152,6 +158,7 @@ export class MobilePlatformDriver implements IPlatformDriver { timeoutMs: number, within?: WithinScope, signal?: AbortSignal, + options?: ClickOptions, ): Promise { rejectWithinScope(within); const query = resolveTargetToQuery(targetType, targetValue, refMap); @@ -159,13 +166,37 @@ export class MobilePlatformDriver implements IPlatformDriver { const element = await this.#backend.waitForElement(query, timeoutMs); // The request may have timed out while we waited for the target. signal?.throwIfAborted(); - const remaining = deadline - Date.now(); + const target = `${targetType}:${targetValue}`; + try { + await withTimeout( + this.#tapResolved(query, element), + deadline - Date.now(), + 'tapElement', + ); + return { clicked: true, target }; + } catch (error) { + if (!options?.dismissKeyboard || !isKeyboardObscured(error)) { + throw error; + } + } + // Nothing was tapped: the keyboard was on top of the target. await withTimeout( - this.#tapResolved(query, element), - remaining, + this.#backend.dismissKeyboard(), + deadline - Date.now(), + 'dismissKeyboard', + ); + signal?.throwIfAborted(); + const moved = await this.#backend.waitForElement( + query, + Math.max(0, deadline - Date.now()), + ); + signal?.throwIfAborted(); + await withTimeout( + this.#tapResolved(query, moved), + deadline - Date.now(), 'tapElement', ); - return { clicked: true, target: `${targetType}:${targetValue}` }; + return { clicked: true, target, keyboardDismissed: true }; } /** @@ -867,6 +898,18 @@ function isMetroDownError(error: unknown): boolean { return message.includes('ECONNREFUSED') || message.includes('fetch failed'); } +/** + * Whether device-mcp refused a tap because the software keyboard covers the + * target (`TargetObscuredError` with `keyboard: true`). + * + * @param error - The thrown error. + * @returns True for a keyboard-obscured tap. + */ +function isKeyboardObscured(error: unknown): boolean { + const value = error as { code?: unknown; keyboard?: unknown } | null; + return value?.code === 'TARGET_OBSCURED' && value.keyboard === true; +} + /** * Read device-mcp's `scrollToElement` result: `{ element, swipes, * inViewport }` from releases with the viewport check, or the bare element diff --git a/src/platform/types.ts b/src/platform/types.ts index bdaea08..6578676 100644 --- a/src/platform/types.ts +++ b/src/platform/types.ts @@ -22,6 +22,16 @@ export type ClickActionResult = { clicked: boolean; target: string; pageClosedAfterClick?: boolean; + /** Mobile: the keyboard covered the target and was dismissed first. */ + keyboardDismissed?: boolean; +}; + +export type ClickOptions = { + /** + * Mobile: when the software keyboard covers the target, dismiss it and tap + * once more. Ignored by the browser driver. + */ + dismissKeyboard?: boolean; }; export type TypeActionResult = { @@ -89,6 +99,7 @@ export type IPlatformDriver = { timeoutMs: number, within?: WithinScope, signal?: AbortSignal, + options?: ClickOptions, ): Promise; /** diff --git a/src/server/create-server.test.ts b/src/server/create-server.test.ts index 8a51306..afc29af 100644 --- a/src/server/create-server.test.ts +++ b/src/server/create-server.test.ts @@ -11,10 +11,12 @@ import { buildResponseBody, resolveObserveLevel, screenAfterResponse, + withScreenChanged, } from './create-server.js'; import { readDaemonState } from './daemon-state.js'; import pkg from '../../package.json'; import type { PortMap, WorkflowContext } from '../capabilities/context.js'; +import type { IPlatformDriver } from '../platform/types.js'; import { toolRegistry } from '../tools/registry.js'; import type { StepRecordObservation } from '../tools/types/observation.js'; import { waitForAnimationsToSettle } from '../tools/utils/discovery.js'; @@ -3270,6 +3272,74 @@ describe('createServer abort, ref generations and observations', () => { }); }); +describe('withScreenChanged', () => { + const before = [{ ref: 'e1', role: 'button', name: 'Go', path: [] }]; + const same = [{ ref: 'e7', role: 'button', name: 'Go', path: [] }]; + const other = [{ ref: 'e1', role: 'button', name: 'Back', path: [] }]; + const obs = (nodes: unknown[]): StepRecordObservation => + ({ testIds: [], a11y: { nodes } }) as unknown as StepRecordObservation; + const mobile = { getPlatform: () => 'ios' } as unknown as IPlatformDriver; + const browser = { + getPlatform: () => 'browser', + } as unknown as IPlatformDriver; + const ok = { + ok: true as const, + result: { clicked: true, target: 'testId:x' }, + }; + + it('reports a changed screen', () => { + expect( + withScreenChanged('click', ok, obs(other), before, mobile), + ).toStrictEqual({ + ok: true, + result: { clicked: true, target: 'testId:x', screenChanged: true }, + }); + }); + + it('reports an unchanged screen, ignoring refs', () => { + expect( + withScreenChanged('click', ok, obs(same), before, mobile), + ).toMatchObject({ result: { screenChanged: false } }); + }); + + it.each([ + ['another tool', 'type', ok, obs(other), before, mobile], + [ + 'a failed click', + 'click', + { ok: false as const, error: { code: 'X', message: 'm' } }, + obs(other), + before, + mobile, + ], + ['no observation', 'click', ok, undefined, before, mobile], + ['no earlier screen', 'click', ok, obs(other), null, mobile], + ['the browser', 'click', ok, obs(other), before, browser], + ['no driver', 'click', ok, obs(other), before, undefined], + [ + 'a non-object result', + 'click', + { ok: true as const, result: 'x' }, + obs(other), + before, + mobile, + ], + ])( + 'leaves the result unchanged for %s', + (_name, tool, result, observations, screen, driver) => { + expect( + withScreenChanged( + tool, + result as never, + observations, + screen as never, + driver, + ), + ).toBe(result); + }, + ); +}); + describe('screenAfterResponse', () => { const tree = [{ ref: 'e1', role: 'button', name: 'Go', path: [] }]; const other = [{ ref: 'e1', role: 'button', name: 'Back', path: [] }]; diff --git a/src/server/create-server.ts b/src/server/create-server.ts index 71f412b..38dfd0b 100644 --- a/src/server/create-server.ts +++ b/src/server/create-server.ts @@ -157,6 +157,49 @@ export function screenAfterResponse( return previous; } +/** + * Adds `screenChanged` to a successful mobile `click` result when an + * observation was collected: whether the observed screen differs (refs + * ignored) from the last screen the agent received before the click. Not + * added without an observation or without an earlier screen to compare. + * + * @param toolName - The tool name. + * @param result - The tool response. + * @param observations - The observation collected after the tool, if any. + * @param screenBefore - The last screen the agent received before the tool. + * @param driver - The platform driver of the request. + * @returns The response, with `screenChanged` when it applies. + */ +export function withScreenChanged( + toolName: string, + result: ToolResponse, + observations: StepRecordObservation | undefined, + screenBefore: A11yNodeTrimmed[] | null, + driver: IPlatformDriver | undefined, +): ToolResponse { + if ( + toolName !== 'click' || + !result.ok || + !observations || + !screenBefore || + !driver || + driver.getPlatform() === 'browser' || + typeof result.result !== 'object' || + result.result === null + ) { + return result; + } + return { + ok: true, + result: { + ...(result.result as Record), + screenChanged: + screenSignature(observations.a11y.nodes) !== + screenSignature(screenBefore), + }, + }; +} + /** * Merges a tool result with observation data into the HTTP response body. * @@ -708,6 +751,9 @@ export function createServer(config: ServerConfig): ServerInstance { timer.markToolEnd(); const level = resolveObserveLevel(category, inputRecord, result); + // Captured now: the observation below must compare with the + // screen the agent had before this click. + const screenBefore = lastScreen; let obs: StepRecordObservation | undefined; if ( level !== 'none' && @@ -726,7 +772,13 @@ export function createServer(config: ServerConfig): ServerInstance { } return { - toolResult: result, + toolResult: withScreenChanged( + toolName, + result, + obs, + screenBefore, + context.driver, + ), observeLevel: level, observations: obs, }; diff --git a/src/tools/driver-delegation.test.ts b/src/tools/driver-delegation.test.ts index f499701..2e993cf 100644 --- a/src/tools/driver-delegation.test.ts +++ b/src/tools/driver-delegation.test.ts @@ -214,7 +214,39 @@ describe('tool delegation to context.driver', () => { expect.any(Number), undefined, undefined, + undefined, + ); + }); + + it('clickTool passes dismissKeyboard to the driver', async () => { + const driver = createMockDriver(); + const context = createContextWithDriver(driver); + + await clickTool({ testId: 'btn', dismissKeyboard: true }, context); + + expect(vi.mocked(driver.click).mock.calls[0]?.[6]).toStrictEqual({ + dismissKeyboard: true, + }); + }); + + it('clickTool maps TARGET_OBSCURED to MM_TARGET_OBSCURED', async () => { + const driver = createMockDriver(); + vi.mocked(driver.click).mockRejectedValue( + Object.assign(new Error('TARGET_OBSCURED: covered by the keyboard'), { + code: 'TARGET_OBSCURED', + }), ); + const context = createContextWithDriver(driver); + + const result = await clickTool({ testId: 'btn' }, context); + + expect(result).toStrictEqual({ + ok: false, + error: { + code: 'MM_TARGET_OBSCURED', + message: 'TARGET_OBSCURED: covered by the keyboard', + }, + }); }); it('clickTool and typeTool pass the request signal to the driver', async () => { diff --git a/src/tools/error-classification.test.ts b/src/tools/error-classification.test.ts index c58f6d6..d6fab04 100644 --- a/src/tools/error-classification.test.ts +++ b/src/tools/error-classification.test.ts @@ -121,6 +121,33 @@ describe('error-classification', () => { }); describe('classifyInteractionError', () => { + describe('with a typed device-mcp error', () => { + it('maps TARGET_OBSCURED to MM_TARGET_OBSCURED even when the text says "not found"', () => { + const error = Object.assign( + new Error('TARGET_OBSCURED: Button id="not found" is covered'), + { code: 'TARGET_OBSCURED' }, + ); + + expect( + classifyInteractionError(error, ErrorCodes.MM_CLICK_FAILED), + ).toStrictEqual({ + code: ErrorCodes.MM_TARGET_OBSCURED, + message: 'TARGET_OBSCURED: Button id="not found" is covered', + }); + }); + + it('maps ELEMENT_NOT_IN_VIEWPORT to MM_TARGET_NOT_VISIBLE', () => { + const error = Object.assign( + new Error('Element is outside the 1080x2400 viewport'), + { code: 'ELEMENT_NOT_IN_VIEWPORT' }, + ); + + expect( + classifyInteractionError(error, ErrorCodes.MM_CLICK_FAILED).code, + ).toBe(ErrorCodes.MM_TARGET_NOT_VISIBLE); + }); + }); + describe('when error indicates target not found', () => { it('returns TARGET_NOT_FOUND for "Unknown a11yRef"', () => { // Arrange diff --git a/src/tools/error-classification.ts b/src/tools/error-classification.ts index 2f9e171..2d289e9 100644 --- a/src/tools/error-classification.ts +++ b/src/tools/error-classification.ts @@ -54,6 +54,16 @@ export function classifyInteractionError( } { const message = extractErrorMessage(error); + // Typed device-mcp errors first: their messages can contain "not found" + // or "timed out" text that the patterns below would misread. + const deviceCode = (error as { code?: unknown } | null | undefined)?.code; + if (deviceCode === 'TARGET_OBSCURED') { + return { code: ErrorCodes.MM_TARGET_OBSCURED, message }; + } + if (deviceCode === 'ELEMENT_NOT_IN_VIEWPORT') { + return { code: ErrorCodes.MM_TARGET_NOT_VISIBLE, message }; + } + // Page-closed takes priority — the browser/context is gone, so no other // classification is meaningful. for (const pattern of ERROR_PATTERNS.pageClosed) { diff --git a/src/tools/interaction.ts b/src/tools/interaction.ts index 1f60c75..c24d965 100644 --- a/src/tools/interaction.ts +++ b/src/tools/interaction.ts @@ -150,6 +150,7 @@ export async function clickTool( timeoutMs, resolveWithinScope(input.within), context.signal, + input.dismissKeyboard ? { dismissKeyboard: true } : undefined, ); return createToolSuccess(result); } catch (error) { diff --git a/src/tools/types/errors.ts b/src/tools/types/errors.ts index 26f2b8e..60e9219 100644 --- a/src/tools/types/errors.ts +++ b/src/tools/types/errors.ts @@ -15,7 +15,8 @@ export const ErrorCodes = { MM_NOTIFICATION_TIMEOUT: 'MM_NOTIFICATION_TIMEOUT', MM_TARGET_NOT_FOUND: 'MM_TARGET_NOT_FOUND', - MM_TARGET_NOT_VISIBLE: 'MM_TARGET_NOT_VISIBLE', // found, but not brought into the viewport (scroll_to_element) + MM_TARGET_NOT_VISIBLE: 'MM_TARGET_NOT_VISIBLE', // found, but not in the viewport (scroll_to_element, mobile tap) + MM_TARGET_OBSCURED: 'MM_TARGET_OBSCURED', // another element (keyboard, sheet, overlay) is on top of the target; nothing was tapped MM_CLICK_FAILED: 'MM_CLICK_FAILED', MM_TYPE_FAILED: 'MM_TYPE_FAILED', MM_GETTEXT_FAILED: 'MM_GETTEXT_FAILED', diff --git a/src/tools/types/tool-inputs.ts b/src/tools/types/tool-inputs.ts index c0d1d06..8c2f15a 100644 --- a/src/tools/types/tool-inputs.ts +++ b/src/tools/types/tool-inputs.ts @@ -103,6 +103,11 @@ export type WithinTarget = { export type ClickInput = TargetSelection & { timeoutMs?: number; within?: WithinTarget; + /** + * Mobile: when the software keyboard covers the target, dismiss it and + * tap once more. + */ + dismissKeyboard?: boolean; }; export type TypeInput = TargetSelection & { diff --git a/src/tools/types/tool-outputs.ts b/src/tools/types/tool-outputs.ts index 4663271..cf503ca 100644 --- a/src/tools/types/tool-outputs.ts +++ b/src/tools/types/tool-outputs.ts @@ -84,6 +84,14 @@ export type ClickResult = { clicked: boolean; target: string; pageClosedAfterClick?: boolean; + /** Mobile: the keyboard covered the target and was dismissed first. */ + keyboardDismissed?: boolean; + /** + * Mobile, with an observation: whether the screen after the click differs + * from the last screen the agent received. `false` is a hint (many taps + * change nothing, e.g. the open tab), not an error. + */ + screenChanged?: boolean; }; export type TypeResult = { diff --git a/src/validation/schemas.test.ts b/src/validation/schemas.test.ts index 254d8c9..b973748 100644 --- a/src/validation/schemas.test.ts +++ b/src/validation/schemas.test.ts @@ -18,6 +18,7 @@ import { mockNetworkInputSchema, launchInputSchema, scrollToElementInputSchema, + clickInputSchema, deviceSwipeInputSchema, tapCoordinatesInputSchema, dismissAlertInputSchema, @@ -593,6 +594,22 @@ describe('launchInputSchema', () => { }); describe('device tool schemas', () => { + describe('clickInputSchema dismissKeyboard', () => { + it('accepts dismissKeyboard', () => { + expect( + clickInputSchema.safeParse({ testId: 'x', dismissKeyboard: true }) + .success, + ).toBe(true); + }); + + it('rejects a non-boolean dismissKeyboard', () => { + expect( + clickInputSchema.safeParse({ testId: 'x', dismissKeyboard: 'yes' }) + .success, + ).toBe(false); + }); + }); + describe('scrollToElementInputSchema', () => { it('accepts a target with direction and maxAttempts', () => { const result = scrollToElementInputSchema.safeParse({ diff --git a/src/validation/schemas.ts b/src/validation/schemas.ts index 876b2f4..d9a453a 100644 --- a/src/validation/schemas.ts +++ b/src/validation/schemas.ts @@ -471,6 +471,13 @@ export const clickInputSchema = targetSelectionSchema.and( .default(15000) .describe('Timeout to wait for element to become visible'), within: withinTargetSchema.optional(), + dismissKeyboard: z + .boolean() + .optional() + .describe( + 'Mobile: when the software keyboard covers the target ' + + '(MM_TARGET_OBSCURED), dismiss it and tap once more', + ), }), );