diff --git a/CHANGELOG.md b/CHANGELOG.md index f126408..bd80d60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Start the CLI faster: find the project root without running `git` and do not load zod. Load `@metamask/device-mcp` only when a mobile session uses it, so browser daemons start without it. +- **BREAKING:** `scroll_to_element` `direction` is now the content direction (`down` reveals content below; it was the finger direction) and defaults to `auto`. The result adds `swipes`, `inViewport` and `bounds`; a target that is found but not brought into view fails with the new `MM_TARGET_NOT_VISIBLE`, and one never found with `MM_TARGET_NOT_FOUND`. The viewport check needs a device-mcp release with MetaMask/device-mcp#50; with device-mcp 0.7.0 only `bounds` is returned. - Mobile `click` and `type` tap the element that the target wait found, so a tap takes one device snapshot instead of two (needs a device-mcp release with `tapElement(query, { element })`; older releases ignore the option). `describe_screen` reads the app state and the screen at the same time on mobile. ### Fixed diff --git a/README.md b/README.md index 283dca7..66d0425 100644 --- a/README.md +++ b/README.md @@ -432,7 +432,7 @@ The daemon routes `POST /tool/:name` requests through the registry, applies Zod | **Hermes (mobile only)** | | | `hermes_targets` | Lists and diagnoses the debuggable Hermes targets Metro exposes (iOS and Android), reporting which target would be chosen or why selection is ambiguous. Use to confirm Metro is running and the app is registered. Pass `all` to bypass the appId filter and discover the real appId. Mobile only — there is no browser equivalent. | | **Mobile (iOS/Android only)** | | -| `scroll_to_element` | Scrolls the screen until an element (a11y ref, test ID, or selector) becomes visible. Supports `direction` (`up`/`down`) and `maxAttempts`. Mobile only. | +| `scroll_to_element` | Swipes until the target (a11y ref or test ID) is inside the viewport, not only in the UI tree, and returns `swipes`, `inViewport` and `bounds`. `direction` is the content direction (`down` reveals content below); default `auto` picks it from the element's position. Fails with `MM_TARGET_NOT_VISIBLE` (found, not in view) or `MM_TARGET_NOT_FOUND`. `maxAttempts` = maximum swipes (default 10). Mobile only. | | `device_swipe` | Swipes the screen in a `direction` (`up`/`down`/`left`/`right`), with optional `startX`/`startY`/`distance`. Mobile only. | | `long_press` | Long-presses an element (a11y ref, test ID, or selector) for an optional `durationMs`. Mobile only. | | `tap_coordinates` | Taps raw `x`/`y` screen coordinates. Mobile only. | @@ -754,26 +754,26 @@ reach the agent intact. These commands require a mobile session (`mm launch --platform ios|android`). Run on a browser session, they return `MM_TOOL_NOT_SUPPORTED_ON_PLATFORM`. -| Command | Description | -| ---------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | -| `mm scroll-to-element [--direction up\|down] [--maxAttempts ] [--selector ] [--testid ]` | Scrolls until the target element is visible. | -| `mm device-swipe --direction [--startX ] [--startY ] [--distance ]` | Swipes the screen in a direction. | -| `mm long-press [--duration ] [--selector ] [--testid ]` | Long-presses the target element. | -| `mm tap-coordinates ` | Taps raw screen coordinates. | -| `mm dismiss-keyboard` | Dismisses the on-screen keyboard. | -| `mm dismiss-alert [--accept]` | Dismisses a native alert; `--accept` accepts it. | -| `mm get-alert-text` | Prints the text of a visible native alert. | -| `mm open-app ` | Launches or foregrounds an app. | -| `mm close-app ` | Terminates an app. | -| `mm press-button ` | Presses a hardware/system button. | -| `mm device-context list` / `mm device-context switch ` | Lists or switches native/webview contexts. | -| `mm device-clipboard read` / `mm device-clipboard write ` | Reads or writes the device clipboard. | -| `mm screen-recording start [--output ]` / `mm screen-recording stop` | Starts or stops a screen recording; `--output` is sandboxed to `artifactsDir`. | -| `mm device-logs [--duration ] [--filter ]` | Fetches device logs. | -| `mm generate-locators` | Lists ranked selector suggestions for on-screen elements. | +| Command | Description | +| ---------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `mm scroll-to-element [--direction auto\|down\|up] [--maxAttempts ] [--selector ] [--testid ]` | Swipes until the target element is inside the viewport. | +| `mm device-swipe --direction [--startX ] [--startY ] [--distance ]` | Swipes the screen in a direction. | +| `mm long-press [--duration ] [--selector ] [--testid ]` | Long-presses the target element. | +| `mm tap-coordinates ` | Taps raw screen coordinates. | +| `mm dismiss-keyboard` | Dismisses the on-screen keyboard. | +| `mm dismiss-alert [--accept]` | Dismisses a native alert; `--accept` accepts it. | +| `mm get-alert-text` | Prints the text of a visible native alert. | +| `mm open-app ` | Launches or foregrounds an app. | +| `mm close-app ` | Terminates an app. | +| `mm press-button ` | Presses a hardware/system button. | +| `mm device-context list` / `mm device-context switch ` | Lists or switches native/webview contexts. | +| `mm device-clipboard read` / `mm device-clipboard write ` | Reads or writes the device clipboard. | +| `mm screen-recording start [--output ]` / `mm screen-recording stop` | Starts or stops a screen recording; `--output` is sandboxed to `artifactsDir`. | +| `mm device-logs [--duration ] [--filter ]` | Fetches device logs. | +| `mm generate-locators` | Lists ranked selector suggestions for on-screen elements. | ```bash -mm scroll-to-element e12 --direction down +mm scroll-to-element e12 # auto: scrolls toward the element mm device-swipe --direction up --distance 400 mm open-app io.metamask mm device-context switch WEBVIEW_1 diff --git a/SKILL.md b/SKILL.md index 94b08ea..75e867a 100644 --- a/SKILL.md +++ b/SKILL.md @@ -591,9 +591,15 @@ These commands require a mobile session (`mm launch --platform ios|android`). On session they return `MM_TOOL_NOT_SUPPORTED_ON_PLATFORM`. Element-targeting commands accept the same `` / `--testid` / `--selector` targeting as `mm click`. -#### `mm scroll-to-element [--direction up|down] [--maxAttempts ]` +#### `mm scroll-to-element [--direction auto|down|up] [--maxAttempts ]` -Scrolls the screen until the target element becomes visible. +Swipes until the target is inside the viewport (being in the UI tree is not +enough; React Native keeps off-screen rows in the tree), so a following `click` +can tap it. Returns `{scrolled, swipes, inViewport, bounds}`. `--direction` is +the content direction: `down` reveals content below. The default `auto` picks +it from where the element is. Errors: `MM_TARGET_NOT_VISIBLE` (found but still +outside the viewport after `--maxAttempts` swipes or at the end of the list; +the message has its last frame), `MM_TARGET_NOT_FOUND` (never in the UI tree). #### `mm device-swipe --direction [--startX ] [--startY ] [--distance ]` @@ -652,7 +658,7 @@ for every interactive element on the current screen. Read-only — useful when a mobile automation to pick stable selectors. ```bash -mm scroll-to-element e12 --direction down +mm scroll-to-element e12 mm device-swipe --direction up --distance 400 mm open-app io.metamask mm device-context switch WEBVIEW_1 diff --git a/src/cli/mm.ts b/src/cli/mm.ts index 53d2e5f..ff0b4cd 100644 --- a/src/cli/mm.ts +++ b/src/cli/mm.ts @@ -917,7 +917,7 @@ export async function routeCommand( !args.includes('--testid') ) { process.stderr.write( - 'Usage: mm scroll-to-element [--selector ] [--testid ] [--direction up|down] [--maxAttempts ]\n', + 'Usage: mm scroll-to-element [--selector ] [--testid ] [--direction auto|down|up] [--maxAttempts ]\n', ); process.exit(1); } diff --git a/src/platform/mobile-platform-driver.test.ts b/src/platform/mobile-platform-driver.test.ts index a0f372f..e97eeca 100644 --- a/src/platform/mobile-platform-driver.test.ts +++ b/src/platform/mobile-platform-driver.test.ts @@ -1889,35 +1889,106 @@ describe('MobilePlatformDriver', () => { }); describe('scrollToElement', () => { - it('resolves target and delegates', async () => { + it('maps the content direction to the opposite finger direction', async () => { const backend = createMockBackend({ scrollToElement: vi.fn().mockResolvedValue(makeElement()), }); const driver = new MobilePlatformDriver(backend); const refMap = new Map([['e1', 'identifier:footer']]); + // Content down (reveal below) = finger up. await driver.scrollToElement('a11yRef', 'e1', refMap, 'down', 5); + await driver.scrollToElement('a11yRef', 'e1', refMap, 'up', 5); - expect(backend.scrollToElement).toHaveBeenCalledWith( - { identifier: 'footer' }, - 'down', - 5, - ); + expect(vi.mocked(backend.scrollToElement).mock.calls).toStrictEqual([ + [{ identifier: 'footer' }, 'up', 5], + [{ identifier: 'footer' }, 'down', 5], + ]); }); - it('resolves testId target', async () => { + it('leaves the direction to device-mcp by default (auto)', async () => { const backend = createMockBackend({ scrollToElement: vi.fn().mockResolvedValue(makeElement()), }); const driver = new MobilePlatformDriver(backend); await driver.scrollToElement('testId', 'terms-section', new Map()); - - expect(backend.scrollToElement).toHaveBeenCalledWith( - { identifier: 'terms-section' }, - undefined, - undefined, + await driver.scrollToElement( + 'testId', + 'terms-section', + new Map(), + 'auto', ); + + expect(vi.mocked(backend.scrollToElement).mock.calls).toStrictEqual([ + [{ identifier: 'terms-section' }, undefined, undefined], + [{ identifier: 'terms-section' }, undefined, undefined], + ]); + }); + + it('returns swipes, inViewport and bounds from a viewport-checking device-mcp', async () => { + const element = makeElement({ + frame: { x: 16, y: 646, width: 370, height: 24 }, + }); + const backend = createMockBackend({ + scrollToElement: vi.fn().mockResolvedValue({ + element, + swipes: 1, + inViewport: true, + }) as unknown as DeviceBackend['scrollToElement'], + }); + const driver = new MobilePlatformDriver(backend); + + const outcome = await driver.scrollToElement('testId', 'row', new Map()); + + expect(outcome).toStrictEqual({ + swipes: 1, + inViewport: true, + bounds: { x: 16, y: 646, width: 370, height: 24 }, + }); + }); + + it('returns only the bounds from device-mcp 0.7.0 (element only)', async () => { + const backend = createMockBackend({ + scrollToElement: vi + .fn() + .mockResolvedValue( + makeElement({ frame: { x: 1, y: 996, width: 2, height: 3 } }), + ), + }); + const driver = new MobilePlatformDriver(backend); + + expect( + await driver.scrollToElement('testId', 'row', new Map()), + ).toStrictEqual({ bounds: { x: 1, y: 996, width: 2, height: 3 } }); + }); + + it('returns nothing for an object without a frame', async () => { + const backend = createMockBackend({ + scrollToElement: vi + .fn() + .mockResolvedValue({}) as unknown as DeviceBackend['scrollToElement'], + }); + const driver = new MobilePlatformDriver(backend); + + expect( + await driver.scrollToElement('testId', 'row', new Map()), + ).toStrictEqual({}); + }); + + it('returns nothing for an unexpected backend result', async () => { + const backend = createMockBackend({ + scrollToElement: vi + .fn() + .mockResolvedValue( + undefined, + ) as unknown as DeviceBackend['scrollToElement'], + }); + const driver = new MobilePlatformDriver(backend); + + expect( + await driver.scrollToElement('testId', 'row', new Map()), + ).toStrictEqual({}); }); }); diff --git a/src/platform/mobile-platform-driver.ts b/src/platform/mobile-platform-driver.ts index a04977c..626d251 100644 --- a/src/platform/mobile-platform-driver.ts +++ b/src/platform/mobile-platform-driver.ts @@ -22,6 +22,7 @@ import type { PlatformScreenshotOptions, PlatformType, ScreenCapture, + ScrollToElementOutcome, WithinScope, } from './types.js'; import type { @@ -458,15 +459,45 @@ export class MobilePlatformDriver implements IPlatformDriver { * @param direction - Scroll direction (default: down). * @param maxAttempts - Maximum scroll attempts. */ + /** + * Swipe until the element is in the viewport. + * + * `direction` is the **content** direction (`down` reveals content below), + * so it maps to the opposite finger direction in device-mcp. Default + * `auto`: device-mcp picks it from the element's position. device-mcp + * releases before the viewport check (0.7.0) return the element as soon + * as it is in the UI tree and do not know `auto`, so `auto` is not sent + * to them. + * + * @param targetType - Target identifier type. + * @param targetValue - Target value for element lookup. + * @param refMap - Map of a11y refs to stable identifiers. + * @param direction - Content direction, or `auto`. + * @param maxAttempts - Maximum number of swipes. + * @returns Swipe count, whether the element is in view, and its frame. + */ async scrollToElement( targetType: TargetType, targetValue: string, refMap: Map, - direction?: 'up' | 'down', + direction: 'up' | 'down' | 'auto' = 'auto', maxAttempts?: number, - ): Promise { + ): Promise { const query = resolveTargetToQuery(targetType, targetValue, refMap); - await this.#backend.scrollToElement(query, direction, maxAttempts); + const finger = { auto: undefined, down: 'up', up: 'down' } as const; + const backend = this.#backend as Omit & { + scrollToElement( + query: ElementQuery, + direction?: 'up' | 'down' | 'auto', + maxAttempts?: number, + ): Promise; + }; + const outcome = await backend.scrollToElement( + query, + finger[direction], + maxAttempts, + ); + return readScrollOutcome(outcome); } /** @@ -836,6 +867,30 @@ function isMetroDownError(error: unknown): boolean { return message.includes('ECONNREFUSED') || message.includes('fetch failed'); } +/** + * Read device-mcp's `scrollToElement` result: `{ element, swipes, + * inViewport }` from releases with the viewport check, or the bare element + * from older ones (which only know it is in the UI tree). + * + * @param outcome - The backend result. + * @returns Swipe count, viewport flag and frame, when known. + */ +function readScrollOutcome(outcome: unknown): ScrollToElementOutcome { + if (!outcome || typeof outcome !== 'object') { + return {}; + } + if ('element' in outcome) { + const { element, swipes, inViewport } = outcome as { + element: UIElement; + swipes: number; + inViewport: boolean; + }; + return { swipes, inViewport, bounds: { ...element.frame } }; + } + const { frame } = outcome as Partial; + return frame ? { bounds: { ...frame } } : {}; +} + /** * Resolves a target selector to a DeviceBackend ElementQuery. * diff --git a/src/platform/types.ts b/src/platform/types.ts index 02c808d..bdaea08 100644 --- a/src/platform/types.ts +++ b/src/platform/types.ts @@ -30,6 +30,16 @@ export type TypeActionResult = { textLength: number; }; +/** + * What `scrollToElement` did. `swipes` and `inViewport` come from device-mcp + * releases that check the viewport; older releases report only the element. + */ +export type ScrollToElementOutcome = { + swipes?: number; + inViewport?: boolean; + bounds?: { x: number; y: number; width: number; height: number }; +}; + export type GetTextActionResult = { text: string; target: string; @@ -148,9 +158,9 @@ export type IPlatformDriver = { targetType: TargetType, targetValue: string, refMap: Map, - direction?: 'up' | 'down', + direction?: 'up' | 'down' | 'auto', maxAttempts?: number, - ): Promise; + ): Promise; longPress?( targetType: TargetType, diff --git a/src/tools/device.test.ts b/src/tools/device.test.ts index ff3d81a..f54d980 100644 --- a/src/tools/device.test.ts +++ b/src/tools/device.test.ts @@ -165,7 +165,7 @@ describe('scrollToElementTool', () => { }); it('delegates to driver.scrollToElement and returns the target', async () => { - const scrollToElement = vi.fn().mockResolvedValue(undefined); + const scrollToElement = vi.fn().mockResolvedValue({}); const context = createContext({ driver: { scrollToElement } }); const result = await scrollToElementTool( @@ -202,6 +202,50 @@ describe('scrollToElementTool', () => { } }); + it('returns swipes, inViewport and bounds', async () => { + const bounds = { x: 16, y: 646, width: 370, height: 24 }; + const scrollToElement = vi + .fn() + .mockResolvedValue({ swipes: 2, inViewport: true, bounds }); + const context = createContext({ driver: { scrollToElement } }); + + const result = await scrollToElementTool({ testId: 'foo' }, context); + + expect(result).toStrictEqual({ + ok: true, + result: { + scrolled: true, + target: 'testId:foo', + swipes: 2, + inViewport: true, + bounds, + }, + }); + }); + + it.each([ + ['ELEMENT_NOT_IN_VIEWPORT', ErrorCodes.MM_TARGET_NOT_VISIBLE], + ['ELEMENT_NOT_FOUND', ErrorCodes.MM_TARGET_NOT_FOUND], + ])('maps device-mcp %s to %s', async (code, expected) => { + const error = Object.assign( + new Error('Element is outside the 402x874 viewport after 3 swipe(s)'), + { code }, + ); + const scrollToElement = vi.fn().mockRejectedValue(error); + const context = createContext({ driver: { scrollToElement } }); + + const result = await scrollToElementTool({ testId: 'foo' }, context); + + expect(result).toStrictEqual({ + ok: false, + error: { + code: expected, + message: + 'Scrolling to element failed: Element is outside the 402x874 viewport after 3 swipe(s)', + }, + }); + }); + it('returns MM_DEVICE_ACTION_FAILED when the driver throws', async () => { const scrollToElement = vi.fn().mockRejectedValue(new Error('scroll boom')); const context = createContext({ driver: { scrollToElement } }); diff --git a/src/tools/device.ts b/src/tools/device.ts index 730b7d4..360ec40 100644 --- a/src/tools/device.ts +++ b/src/tools/device.ts @@ -148,7 +148,7 @@ export async function scrollToElementTool( } try { - await driver.scrollToElement( + const outcome = await driver.scrollToElement( resolved.targetType, resolved.targetValue, context.refMap, @@ -158,16 +158,37 @@ export async function scrollToElementTool( return createToolSuccess({ scrolled: true, target: `${resolved.targetType}:${resolved.targetValue}`, + ...outcome, }); } catch (error) { const message = error instanceof Error ? error.message : String(error); return createToolError( - ErrorCodes.MM_DEVICE_ACTION_FAILED, + scrollErrorCode(error), `Scrolling to element failed: ${message}`, ); } } +/** + * Error code for a failed scroll: device-mcp's `ScrollToElementError` codes + * map to `MM_TARGET_NOT_FOUND` (never in the UI tree) and + * `MM_TARGET_NOT_VISIBLE` (found, not brought into view). Anything else is a + * device failure. + * + * @param error - The thrown error. + * @returns The tool error code. + */ +function scrollErrorCode(error: unknown): string { + const code = (error as { code?: unknown } | undefined)?.code; + if (code === 'ELEMENT_NOT_FOUND') { + return ErrorCodes.MM_TARGET_NOT_FOUND; + } + if (code === 'ELEMENT_NOT_IN_VIEWPORT') { + return ErrorCodes.MM_TARGET_NOT_VISIBLE; + } + return ErrorCodes.MM_DEVICE_ACTION_FAILED; +} + /** * Performs a swipe gesture on the device screen. Mobile only (iOS/Android) — * the browser platform has no device-level swipe gesture. diff --git a/src/tools/types/errors.ts b/src/tools/types/errors.ts index ed42a79..26f2b8e 100644 --- a/src/tools/types/errors.ts +++ b/src/tools/types/errors.ts @@ -15,6 +15,7 @@ export const ErrorCodes = { MM_NOTIFICATION_TIMEOUT: 'MM_NOTIFICATION_TIMEOUT', MM_TARGET_NOT_FOUND: 'MM_TARGET_NOT_FOUND', + MM_TARGET_NOT_VISIBLE: 'MM_TARGET_NOT_VISIBLE', // found, but not brought into the viewport (scroll_to_element) MM_CLICK_FAILED: 'MM_CLICK_FAILED', MM_TYPE_FAILED: 'MM_TYPE_FAILED', MM_GETTEXT_FAILED: 'MM_GETTEXT_FAILED', diff --git a/src/tools/types/tool-inputs.ts b/src/tools/types/tool-inputs.ts index ef05426..c0d1d06 100644 --- a/src/tools/types/tool-inputs.ts +++ b/src/tools/types/tool-inputs.ts @@ -227,7 +227,8 @@ export type SetContextInput = { }; export type ScrollToElementInput = TargetSelection & { - direction?: 'up' | 'down'; + /** Content direction: `down` reveals content below. Default `auto`. */ + direction?: 'up' | 'down' | 'auto'; maxAttempts?: number; }; diff --git a/src/tools/types/tool-outputs.ts b/src/tools/types/tool-outputs.ts index 98d5920..4663271 100644 --- a/src/tools/types/tool-outputs.ts +++ b/src/tools/types/tool-outputs.ts @@ -245,6 +245,12 @@ export type GetContextResult = { export type ScrollToElementResult = { scrolled: boolean; target: string; + /** Swipes performed (0 when the element was already in view). */ + swipes?: number; + /** Always true on success (the element is inside the viewport). */ + inViewport?: boolean; + /** The element's frame after scrolling, in device points. */ + bounds?: { x: number; y: number; width: number; height: number }; }; export type DeviceSwipeResult = { diff --git a/src/validation/schemas.test.ts b/src/validation/schemas.test.ts index c340a73..254d8c9 100644 --- a/src/validation/schemas.test.ts +++ b/src/validation/schemas.test.ts @@ -612,6 +612,15 @@ describe('device tool schemas', () => { expect(result.success).toBe(false); }); + it('accepts direction auto', () => { + expect( + scrollToElementInputSchema.safeParse({ + testId: 'foo', + direction: 'auto', + }).success, + ).toBe(true); + }); + it('rejects an invalid direction', () => { const result = scrollToElementInputSchema.safeParse({ testId: 'foo', diff --git a/src/validation/schemas.ts b/src/validation/schemas.ts index 5a7d17c..876b2f4 100644 --- a/src/validation/schemas.ts +++ b/src/validation/schemas.ts @@ -746,9 +746,12 @@ export const getWindowSizeInputSchema = z.object({}); export const scrollToElementInputSchema = targetSelectionSchema.and( z.object({ direction: z - .enum(['up', 'down']) + .enum(['auto', 'up', 'down']) .describe( - 'Finger swipe direction to perform while searching (swipe up scrolls content down)', + 'Content direction to search: "down" reveals content below (finger ' + + 'swipes up), "up" reveals content above. Default "auto": from the ' + + "element's position, searching below first when it is not in the " + + 'UI tree yet.', ) .optional(), maxAttempts: z @@ -756,7 +759,7 @@ export const scrollToElementInputSchema = targetSelectionSchema.and( .int() .min(1) .max(50) - .describe('Maximum number of scroll attempts before giving up') + .describe('Maximum number of swipes before giving up (default 10)') .optional(), }), );