diff --git a/CHANGELOG.md b/CHANGELOG.md index b7242e5..04be89e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `ScrollToElementError` (`ELEMENT_NOT_FOUND` / `ELEMENT_NOT_IN_VIEWPORT`, with `swipes` and the last `frame`) and the `ScrollDirection` / `ScrollToElementResult` types ([#50](https://github.com/MetaMask/device-mcp/pull/50)) - `tapElement(query, { element })` taps an element the caller already resolved (for example from `waitForElement`) without taking a second snapshot ### Changed +- **BREAKING:** fix: `scrollToElement` succeeds only when the element is in the viewport ([#50](https://github.com/MetaMask/device-mcp/pull/50)) + - It returns `{ element, swipes, inViewport: true }` instead of the element. Before, it returned as soon as the element was in the UI tree; React Native keeps off-screen rows in the tree, so it often returned after zero swipes with the element below the fold. + - After a swipe it reads the screen until it stops moving. It fails with `ScrollToElementError` after `maxAttempts` swipes, or when the screen stops changing. + - New default `direction: 'auto'`: the finger direction comes from the element's position (searching toward content below first while it is not in the tree). Explicit `up` / `down` stay finger directions. Before, the default `down` revealed content above. + - Android and Appium swipe from the middle of the real screen (40% of its height), not from fixed coordinates. - **BREAKING:** fix: iOS key presses, `back` and text clearing ([#48](https://github.com/MetaMask/device-mcp/pull/48)) - iOS `pressButton('enter')` and `dismissKeyboard()` send the integer HID key code (`idb ui key 40`); `home` and `lock` use `idb ui button`. Both failed before because `idb ui key` takes only integer codes. - iOS `pressButton('back')` now throws `UNSUPPORTED_ON_IOS` instead of pressing Home (which put the app in the background). diff --git a/README.md b/README.md index 10ffa0e..c2950d1 100644 --- a/README.md +++ b/README.md @@ -176,15 +176,16 @@ The `.device-session` file is typically written by the test runner when it creat ### Interaction -| Tool | Description | -| ------------------------ | ------------------------------------------------------------------ | -| `device_tap_element` | Find an element by label/identifier/text/type and tap its center. | -| `device_tap_coordinates` | Tap at exact screen coordinates. Last resort when queries fail. | -| `device_type` | Type text into the currently focused input field. | -| `device_swipe` | Swipe in a direction with optional start coordinates and distance. | -| `device_long_press` | Long press an element for context menus or drag initiation. | -| `device_wait_for` | Poll until an element matching a query appears. | -| `device_press_button` | Press a device button (home/back/enter/lock; no back on iOS). | +| Tool | Description | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `device_tap_element` | Find an element by label/identifier/text/type and tap its center. | +| `device_tap_coordinates` | Tap at exact screen coordinates. Last resort when queries fail. | +| `device_type` | Type text into the currently focused input field. | +| `device_swipe` | Swipe in a direction with optional start coordinates and distance. | +| `device_long_press` | Long press an element for context menus or drag initiation. | +| `device_scroll_to_element` | Swipe until an element is inside the viewport (not only in the UI tree); reports the swipe count. `direction` is the finger direction (`up` reveals content below); default `auto` picks it from the element's position. | +| `device_wait_for` | Poll until an element matching a query appears. | +| `device_press_button` | Press a device button (home/back/enter/lock; no back on iOS). | ### App & Device Control diff --git a/src/backends/adb-backend.ts b/src/backends/adb-backend.ts index dc176a6..b0c7225 100644 --- a/src/backends/adb-backend.ts +++ b/src/backends/adb-backend.ts @@ -43,6 +43,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -63,6 +65,7 @@ import { hardenArtifactFile, resolveArtifactPath, } from '../utils/output-path.js'; +import { scrollUntilInViewport } from '../utils/scroll.js'; // `uiautomator dump` exits 0 and prints its success banner even when it fails, so // the XML payload is the only trustworthy signal. @@ -900,26 +903,40 @@ export class AdbBackend implements DeviceBackend { async scrollToElement( query: ElementQuery, - direction: 'up' | 'down' = 'down', + direction: ScrollDirection = 'auto', maxAttempts = 10, - ): Promise { - let previousRaw = ''; + ): Promise { + const windowSize = await this.getWindowSize(); + return scrollUntilInViewport( + { + snapshot: async () => this.snapshot(), + windowSize, + swipe: async (finger) => this.#scrollSwipe(finger, windowSize), + }, + query, + direction, + maxAttempts, + ); + } - for (let i = 0; i < maxAttempts; i++) { - const snap = await this.snapshot(); - const element = findElement(snap.hierarchy, query); - if (element) { - return element; - } - if (snap.raw === previousRaw) { - break; - } - previousRaw = snap.raw; - await this.swipe(direction); - } - throw new Error( - `Element not found after scrolling: ${JSON.stringify(query)}\n` + - 'Use device_snapshot to inspect the current UI hierarchy.', + /** + * One scroll gesture from the middle of the screen over 40% of its height + * (the iOS backend's default), so the swipe matches the real screen size. + * + * @param finger - Finger direction. + * @param windowSize - The screen size, in the frame units. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + */ + async #scrollSwipe( + finger: 'up' | 'down', + { width, height }: WindowSize, + ): Promise { + await this.swipe( + finger, + Math.round(width / 2), + Math.round(height / 2), + Math.round(height * 0.4), ); } diff --git a/src/backends/appium-backend.ts b/src/backends/appium-backend.ts index eec2700..9e28106 100644 --- a/src/backends/appium-backend.ts +++ b/src/backends/appium-backend.ts @@ -10,6 +10,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -26,6 +28,7 @@ import { resolveArtifactPath, writeArtifactFile, } from '../utils/output-path.js'; +import { scrollUntilInViewport } from '../utils/scroll.js'; export class AppiumBackend implements DeviceBackend { readonly kind = 'appium' as const; @@ -425,26 +428,40 @@ export class AppiumBackend implements DeviceBackend { async scrollToElement( query: ElementQuery, - direction: 'up' | 'down' = 'down', + direction: ScrollDirection = 'auto', maxAttempts = 10, - ): Promise { - let previousRaw = ''; + ): Promise { + const windowSize = await this.getWindowSize(); + return scrollUntilInViewport( + { + snapshot: async () => this.snapshot(), + windowSize, + swipe: async (finger) => this.#scrollSwipe(finger, windowSize), + }, + query, + direction, + maxAttempts, + ); + } - for (let i = 0; i < maxAttempts; i++) { - const snap = await this.snapshot(); - const element = findElement(snap.hierarchy, query); - if (element) { - return element; - } - if (snap.raw === previousRaw) { - break; - } - previousRaw = snap.raw; - await this.swipe(direction); - } - throw new Error( - `Element not found after scrolling: ${JSON.stringify(query)}\n` + - 'Use device_snapshot to inspect the current UI hierarchy.', + /** + * One scroll gesture from the middle of the screen over 40% of its height + * (the iOS backend's default), so the swipe matches the real screen size. + * + * @param finger - Finger direction. + * @param windowSize - The screen size, in the frame units. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + */ + async #scrollSwipe( + finger: 'up' | 'down', + { width, height }: WindowSize, + ): Promise { + await this.swipe( + finger, + Math.round(width / 2), + Math.round(height / 2), + Math.round(height * 0.4), ); } diff --git a/src/backends/idb-backend.test.ts b/src/backends/idb-backend.test.ts index 6c335c9..d3a56e3 100644 --- a/src/backends/idb-backend.test.ts +++ b/src/backends/idb-backend.test.ts @@ -701,9 +701,9 @@ describe('IdbBackend.swipe and getWindowSize', () => { snapshotCount += 1; return JSON.stringify([ { - type: snapshotCount === 3 ? 'Button' : 'StaticText', - AXLabel: snapshotCount === 3 ? 'Settings' : `Page ${snapshotCount}`, - frame: { x: 10, y: 20, width: 100, height: 44 }, + type: snapshotCount >= 3 ? 'Button' : 'StaticText', + AXLabel: snapshotCount >= 3 ? 'Settings' : `Page ${snapshotCount}`, + frame: { x: 10, y: 200, width: 100, height: 44 }, enabled: true, }, ]); diff --git a/src/backends/idb-backend.ts b/src/backends/idb-backend.ts index c646cee..4b728d9 100644 --- a/src/backends/idb-backend.ts +++ b/src/backends/idb-backend.ts @@ -14,6 +14,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -38,6 +40,7 @@ import { idbSupportsNestedDescribe, resolveIdbPath, } from '../utils/platform.js'; +import { scrollUntilInViewport } from '../utils/scroll.js'; import { stopChildProcess } from '../utils/stop-child-process.js'; /** @@ -909,28 +912,20 @@ export class IdbBackend implements DeviceBackend { async scrollToElement( query: ElementQuery, - direction: 'up' | 'down' = 'down', + direction: ScrollDirection = 'auto', maxAttempts = 10, - ): Promise { + ): Promise { await this.ensureConnected(); const windowSize = await this.getWindowSize(); - let previousRaw = ''; - - for (let i = 0; i < maxAttempts; i++) { - const snap = await this.snapshot(); - const element = findElement(snap.hierarchy, query); - if (element) { - return element; - } - if (snap.raw === previousRaw) { - break; - } - previousRaw = snap.raw; - await this.#swipe(direction, windowSize); - } - throw new Error( - `Element not found after scrolling: ${JSON.stringify(query)}\n` + - 'Use device_snapshot to inspect the current UI hierarchy.', + return scrollUntilInViewport( + { + snapshot: async () => this.snapshot(), + windowSize, + swipe: async (finger) => this.#swipe(finger, windowSize), + }, + query, + direction, + maxAttempts, ); } diff --git a/src/backends/index.ts b/src/backends/index.ts index 8d4e98a..562e6b0 100644 --- a/src/backends/index.ts +++ b/src/backends/index.ts @@ -15,6 +15,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -304,9 +306,9 @@ export function createLazyBackend( async scrollToElement( query: ElementQuery, - direction?: 'up' | 'down', + direction?: ScrollDirection, maxAttempts?: number, - ): Promise { + ): Promise { return (await resolve()).scrollToElement(query, direction, maxAttempts); }, diff --git a/src/backends/types.ts b/src/backends/types.ts index 4fb3452..7fa5af4 100644 --- a/src/backends/types.ts +++ b/src/backends/types.ts @@ -100,6 +100,17 @@ export type ElementQuery = { export type DeviceButton = 'home' | 'back' | 'enter' | 'lock'; +/** Finger direction of a scroll gesture, or `auto` (from the target's position). */ +export type ScrollDirection = 'up' | 'down' | 'auto'; + +export type ScrollToElementResult = { + /** The element as last read, with its in-view frame. */ + element: UIElement; + /** Swipes performed (0 when the element was already in view). */ + swipes: number; + inViewport: true; +}; + export type WindowSize = { width: number; height: number; @@ -191,11 +202,23 @@ export type DeviceBackend = { longPress(query: ElementQuery, durationMs?: number): Promise; + /** + * Swipe until the element is in the viewport (not only in the UI tree). + * + * @param query - The element to bring into view. + * @param direction - Finger direction (`up` reveals content below), or + * `auto` (default): from the element's position, searching downward first + * when it is not in the tree yet. + * @param maxAttempts - Maximum number of swipes (default 10). + * @returns The element as last read and the number of swipes. + * @throws {ScrollToElementError} `ELEMENT_NOT_FOUND` or + * `ELEMENT_NOT_IN_VIEWPORT` (with the last frame). + */ scrollToElement( query: ElementQuery, - direction?: 'up' | 'down', + direction?: ScrollDirection, maxAttempts?: number, - ): Promise; + ): Promise; getAlertText(): Promise; diff --git a/src/index.ts b/src/index.ts index b3f4a90..21ea0c3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,8 @@ export { createMcpServer } from './server.js'; export { detectAllDevices, MultipleDevicesError } from './utils/platform.js'; export { IdbUnavailableError } from './backends/idb-errors.js'; +export { ScrollToElementError } from './utils/scroll.js'; +export type { ScrollToElementErrorCode } from './utils/scroll.js'; export type { IdbErrorCode } from './backends/idb-errors.js'; export type { DetectedDevice } from './utils/platform.js'; @@ -23,6 +25,8 @@ export type { ScreenshotResult, ScreenshotFileResult, ScreenshotOptions, + ScrollDirection, + ScrollToElementResult, LogsResult, TapElementOptions, TapResult, diff --git a/src/tools/scroll-to-element.ts b/src/tools/scroll-to-element.ts index 0046db1..a8726f9 100644 --- a/src/tools/scroll-to-element.ts +++ b/src/tools/scroll-to-element.ts @@ -13,8 +13,10 @@ export function registerScrollToElementTool( { title: 'Scroll To Element', description: - 'Scroll the screen until an element matching the query becomes visible. ' + - 'Combines scrolling and element search into one atomic operation. ' + + 'Swipe until an element matching the query is inside the viewport ' + + '(being in the UI tree is not enough), and report how many swipes ' + + 'it took. Fails when the element is not in view after maxAttempts ' + + 'swipes or the screen stops moving. ' + 'Specify at least one of: label, identifier, text, type.', inputSchema: { label: z.string().optional().describe('Accessibility label to match'), @@ -25,18 +27,22 @@ export function registerScrollToElementTool( text: z.string().optional().describe('Text content to match'), type: z.string().optional().describe('Element type to match'), direction: z - .enum(['up', 'down']) + .enum(['auto', 'up', 'down']) .optional() - .describe('Scroll direction (default: down)'), + .describe( + 'Finger direction: "up" reveals content below, "down" content ' + + 'above. Default "auto": from where the element is, searching ' + + 'downward first when it is not in the UI tree yet.', + ), maxAttempts: z .number() .optional() - .describe('Maximum scroll attempts (default: 10)'), + .describe('Maximum number of swipes (default: 10)'), }, }, async ({ label, identifier, text, type, direction, maxAttempts }) => { try { - const element = await backend.scrollToElement( + const { element, swipes } = await backend.scrollToElement( { label, identifier, text, type }, direction, maxAttempts, @@ -46,7 +52,7 @@ export function registerScrollToElementTool( { type: 'text' as const, text: [ - `Found element after scrolling:`, + `Element in viewport after ${swipes} swipe(s):`, ` type: ${element.type}`, element.label ? ` label: "${element.label}"` : null, element.identifier ? ` id: "${element.identifier}"` : null, diff --git a/src/utils/scroll.test.ts b/src/utils/scroll.test.ts new file mode 100644 index 0000000..b886b0c --- /dev/null +++ b/src/utils/scroll.test.ts @@ -0,0 +1,309 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + isInViewport, + scrollUntilInViewport, + ScrollToElementError, + swipeToward, +} from './scroll.js'; +import type { SnapshotResult, UIElement } from '../backends/types.js'; + +const windowSize = { width: 402, height: 874 }; + +const node = (label: string, y: number, height = 24): UIElement => ({ + type: 'StaticText', + label, + frame: { x: 16, y, width: 370, height }, + enabled: true, +}); + +const snap = (elements: UIElement[]): SnapshotResult => ({ + platform: 'ios', + hierarchy: elements, + raw: JSON.stringify(elements), + timestamp: 0, +}); + +/** + * A scrolling list: rows at fixed content offsets, and a scroll offset that a + * finger-up swipe increases by `step` (clamped to [0, maxOffset]). + * + * @param rows - Content rows: label and content y. + * @param options - Scroll step, end of the list, fixed (non-scrolling) rows. + * @param options.step - Points moved per swipe. + * @param options.maxOffset - Largest scroll offset (end of the list). + * @param options.fixed - Rows that do not scroll (e.g. a footer button). + * @returns Backend stubs and the swipe log. + */ +function list( + rows: [string, number][], + { + step = 350, + maxOffset = 2000, + fixed = [] as UIElement[], + }: { step?: number; maxOffset?: number; fixed?: UIElement[] } = {}, +): { + deps: Parameters[0]; + swipes: string[]; +} { + let offset = 0; + const swipes: string[] = []; + return { + swipes, + deps: { + windowSize, + snapshot: vi.fn(async () => + snap([...rows.map(([label, y]) => node(label, y - offset)), ...fixed]), + ), + swipe: vi.fn(async (finger: 'up' | 'down') => { + swipes.push(finger); + offset = Math.min( + maxOffset, + Math.max(0, offset + (finger === 'up' ? step : -step)), + ); + }), + }, + }; +} + +describe('isInViewport', () => { + it('rejects an element that is in the tree but below the fold', () => { + expect(isInViewport(node('Account icon', 996), windowSize)).toBe(false); + }); + + it('rejects an element hidden under the bottom inset', () => { + // 874 * 0.96 = 839: only 9 of 24 pt are above the inset. + expect(isInViewport(node('Row', 830), windowSize)).toBe(false); + }); + + it('accepts an element that is fully in view', () => { + expect(isInViewport(node('Row', 400), windowSize)).toBe(true); + }); + + it('accepts a tall element once 44 pt of it are in view', () => { + expect(isInViewport(node('Card', 780, 600), windowSize)).toBe(true); + expect(isInViewport(node('Card', 800, 600), windowSize)).toBe(false); + }); + + it('rejects a zero-height frame (Android clips hidden rows to the list edge)', () => { + expect(isInViewport(node('Currency', 210, 0), windowSize)).toBe(false); + }); + + it('rejects an element outside the viewport horizontally', () => { + const element = { + ...node('Row', 400), + frame: { x: 500, y: 400, width: 100, height: 24 }, + }; + expect(isInViewport(element, windowSize)).toBe(false); + }); +}); + +describe('swipeToward', () => { + it('swipes up (reveals content below) for an element below the middle', () => { + expect(swipeToward(node('Row', 996), windowSize)).toBe('up'); + }); + + it('swipes down (reveals content above) for an element above the middle', () => { + expect(swipeToward(node('Row', -300), windowSize)).toBe('down'); + }); +}); + +describe('scrollUntilInViewport', () => { + it('does not swipe when the element is already in view', async () => { + const { deps, swipes } = list([['Account icon', 400]]); + + const result = await scrollUntilInViewport( + deps, + { label: 'Account icon' }, + 'auto', + 10, + ); + + expect(result).toMatchObject({ swipes: 0, inViewport: true }); + expect(swipes).toStrictEqual([]); + }); + + it('swipes when the element is in the tree but off-screen, and returns it in view', async () => { + // The 2026-10-28 bug: found in the tree at y=996, returned after 0 swipes. + const { deps, swipes } = list([['Account icon', 996]]); + + const result = await scrollUntilInViewport( + deps, + { label: 'Account icon' }, + 'auto', + 10, + ); + + expect(swipes).toStrictEqual(['up']); + expect(result.swipes).toBe(1); + expect(result.inViewport).toBe(true); + expect(result.element.frame.y).toBe(646); + expect(isInViewport(result.element, windowSize)).toBe(true); + }); + + it('auto swipes down for an element above the viewport', async () => { + const { deps, swipes } = list([['Header', 400]]); + // Start scrolled down by 700 pt: Header is at y=-300, above the viewport. + await deps.swipe('up'); + await deps.swipe('up'); + swipes.length = 0; + + const result = await scrollUntilInViewport( + deps, + { label: 'Header' }, + 'auto', + 10, + ); + + expect(swipes[0]).toBe('down'); + expect(result.inViewport).toBe(true); + }); + + it('auto searches below first, then turns around when the list ends', async () => { + // The element is not in the tree until scrolled back to the top. + let offset = 700; + const swipes: string[] = []; + const deps = { + windowSize, + snapshot: vi.fn(async () => + snap(offset === 0 ? [node('Top', 200)] : [node(`at ${offset}`, 400)]), + ), + swipe: vi.fn(async (finger: 'up' | 'down') => { + swipes.push(finger); + offset = Math.min( + 700, + Math.max(0, offset + (finger === 'up' ? 350 : -350)), + ); + }), + }; + + const result = await scrollUntilInViewport( + deps, + { label: 'Top' }, + 'auto', + 10, + ); + + // up (already at the end: no change) -> turn -> down, down. + expect(swipes).toStrictEqual(['up', 'down', 'down']); + expect(result).toMatchObject({ swipes: 3, inViewport: true }); + }); + + it('honours an explicit finger direction', async () => { + const { deps, swipes } = list([['Account icon', 996]]); + + await expect( + scrollUntilInViewport(deps, { label: 'Account icon' }, 'down', 3), + ).rejects.toMatchObject({ + code: 'ELEMENT_NOT_IN_VIEWPORT', + frame: { y: 996 }, + }); + // Finger down at the top of the list moves nothing: stop after one swipe. + expect(swipes).toStrictEqual(['down']); + }); + + it('fails with the last frame when the list ends before the element is in view', async () => { + const { deps, swipes } = list([['Footer note', 2000]], { maxOffset: 500 }); + + const error = await scrollUntilInViewport( + deps, + { label: 'Footer note' }, + 'auto', + 10, + ).catch((e: unknown) => e); + + expect(error).toBeInstanceOf(ScrollToElementError); + expect(error).toMatchObject({ + code: 'ELEMENT_NOT_IN_VIEWPORT', + frame: { y: 1500 }, + }); + expect((error as Error).message).toContain('after 3 swipe(s)'); + expect((error as Error).message).toContain('at (16, 1500 370x24)'); + expect(swipes).toStrictEqual(['up', 'up', 'up']); + }); + + it('fails with ELEMENT_NOT_FOUND when the element never appears', async () => { + const { deps } = list([['Row', 400]], { maxOffset: 0 }); + + await expect( + scrollUntilInViewport(deps, { label: 'Missing' }, 'auto', 10), + ).rejects.toMatchObject({ code: 'ELEMENT_NOT_FOUND', swipes: 2 }); + }); + + it('stops after maxAttempts swipes', async () => { + const { deps, swipes } = list([['Far', 20000]], { maxOffset: 100000 }); + + await expect( + scrollUntilInViewport(deps, { label: 'Far' }, 'auto', 4), + ).rejects.toMatchObject({ code: 'ELEMENT_NOT_IN_VIEWPORT', swipes: 4 }); + expect(swipes).toHaveLength(4); + }); + + it('accepts a fixed element that the swipe does not move', async () => { + // A footer button under the bottom inset never scrolls into the inset area. + const footer = node('Continue', 840, 30); + const { deps, swipes } = list([['Row', 400]], { fixed: [footer] }); + + const result = await scrollUntilInViewport( + deps, + { label: 'Continue' }, + 'auto', + 10, + ); + + expect(swipes).toStrictEqual(['up']); + expect(result).toMatchObject({ swipes: 1, inViewport: true }); + }); + + it('waits for the scroll to settle before returning', async () => { + // After the swipe, momentum moves the row 100 pt more over two reads. + const ys = [996, 646, 546, 546]; + let read = 0; + const deps = { + windowSize, + snapshot: vi.fn(async () => + snap([ + node('Account icon', ys[Math.min((read += 1) - 1, ys.length - 1)]), + ]), + ), + swipe: vi.fn(async () => undefined), + }; + + const result = await scrollUntilInViewport( + deps, + { label: 'Account icon' }, + 'auto', + 10, + ); + + expect(result.element.frame.y).toBe(546); + expect(deps.snapshot).toHaveBeenCalledTimes(4); + }); + + it('keeps scrolling when momentum carries the element out of view', async () => { + // In view right after the swipe, but the scroll keeps going past it. + const ys = [996, 400, -200, -200, 400, 400]; + let read = 0; + const swipes: string[] = []; + const deps = { + windowSize, + snapshot: vi.fn(async () => + snap([node('Row', ys[Math.min((read += 1) - 1, ys.length - 1)])]), + ), + swipe: vi.fn(async (finger: 'up' | 'down') => { + swipes.push(finger); + }), + }; + + const result = await scrollUntilInViewport( + deps, + { label: 'Row' }, + 'auto', + 10, + ); + + expect(swipes).toStrictEqual(['up', 'down']); + expect(result).toMatchObject({ swipes: 2, inViewport: true }); + expect(result.element.frame.y).toBe(400); + }); +}); diff --git a/src/utils/scroll.ts b/src/utils/scroll.ts new file mode 100644 index 0000000..a17e402 --- /dev/null +++ b/src/utils/scroll.ts @@ -0,0 +1,273 @@ +import { describeElement, findElement } from './element.js'; +import type { + ElementQuery, + ScrollDirection, + ScrollToElementResult, + SnapshotResult, + UIElement, + WindowSize, +} from '../backends/types.js'; + +/** + * Part of the viewport height kept clear at the top and at the bottom when + * deciding whether an element is in view (about 35 pt on an 874 pt iPhone): + * the status bar and the iOS home indicator overlay the scrolled content. + */ +const VIEWPORT_INSET = 0.04; + +/** A tall element counts as in view once this many points of it are. */ +const MIN_VISIBLE = 44; + +/** Extra reads, after the target is in view, to let the scroll momentum stop. */ +const MAX_SETTLE_READS = 3; + +export type ScrollToElementErrorCode = + | 'ELEMENT_NOT_FOUND' + | 'ELEMENT_NOT_IN_VIEWPORT'; + +/** + * `scrollToElement` could not bring the element into view. `code` tells + * whether the element was never in the UI tree (`ELEMENT_NOT_FOUND`) or was + * found but stayed outside the viewport (`ELEMENT_NOT_IN_VIEWPORT`, with its + * last `frame`). + */ +export class ScrollToElementError extends Error { + readonly code: ScrollToElementErrorCode; + + readonly swipes: number; + + readonly frame: UIElement['frame'] | undefined; + + constructor( + code: ScrollToElementErrorCode, + message: string, + details: { swipes: number; frame?: UIElement['frame'] }, + ) { + super(message); + this.name = 'ScrollToElementError'; + this.code = code; + this.swipes = details.swipes; + this.frame = details.frame; + } +} + +/** + * Whether enough of the element is inside the viewport, after removing + * {@link VIEWPORT_INSET} at the top and bottom, to see and tap it: half of + * its height, or {@link MIN_VISIBLE} points for a tall element. A zero-size + * frame is never in view. + * + * @param element - The element. + * @param windowSize - The viewport size, in the same units as the frame. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + * @returns True when the element is in view. + */ +export function isInViewport( + element: UIElement, + { width, height }: WindowSize, +): boolean { + const { x, y, width: w, height: h } = element.frame; + const top = Math.max(y, height * VIEWPORT_INSET); + const bottom = Math.min(y + h, height * (1 - VIEWPORT_INSET)); + const visibleWidth = Math.min(x + w, width) - Math.max(x, 0); + const visibleHeight = bottom - top; + // Android clips an off-screen node's bounds to the edge of its scroll + // container, so a hidden row can be reported with zero height. + return ( + visibleWidth > 0 && + visibleHeight > 0 && + visibleHeight >= Math.min(h / 2, MIN_VISIBLE) + ); +} + +/** + * Whether any part of the element is inside the viewport (the rule a tap + * uses). + * + * @param element - The element. + * @param windowSize - The viewport size. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + * @returns True when the frame intersects the viewport. + */ +function intersectsViewport( + element: UIElement, + { width, height }: WindowSize, +): boolean { + const { x, y, width: w, height: h } = element.frame; + return ( + Math.min(x + w, width) > Math.max(x, 0) && + Math.min(y + h, height) > Math.max(y, 0) + ); +} + +function sameFrame(a: UIElement, b: UIElement): boolean { + return ( + a.frame.x === b.frame.x && + a.frame.y === b.frame.y && + a.frame.width === b.frame.width && + a.frame.height === b.frame.height + ); +} + +/** + * The finger direction that brings an off-screen element toward the middle + * of the viewport: up for an element below it (reveals content below), down + * for one above it. + * + * @param element - The element. + * @param windowSize - The viewport size. + * @returns The swipe direction. + */ +export function swipeToward( + element: UIElement, + windowSize: WindowSize, +): 'up' | 'down' { + const centerY = element.frame.y + element.frame.height / 2; + return centerY > windowSize.height / 2 ? 'up' : 'down'; +} + +export type ScrollDeps = { + snapshot: () => Promise; + windowSize: WindowSize; + /** One scroll gesture; `direction` is the finger direction. */ + swipe: (direction: 'up' | 'down') => Promise; +}; + +/** + * Swipe until the element matching `query` is in the viewport. + * + * Finding the element in the UI tree is not enough: React Native keeps + * off-screen rows in the accessibility tree, so the element must also pass + * {@link isInViewport}. An element that a swipe did not move while the screen + * changed is fixed (for example a footer button) and is accepted when it + * intersects the viewport. After a swipe, the screen is read again until it + * stops changing (scroll momentum) before the result is returned. + * + * `direction` is the finger direction. With `auto`, the direction comes from + * the element's position; while the element is not in the tree, the search + * goes toward content below (finger up) and turns around once when the + * screen stops changing. + * + * @param deps - Backend operations. + * @param query - The element to bring into view. + * @param direction - Finger direction, or `auto`. + * @param maxSwipes - Maximum number of swipes. + * @returns The element as last read, the number of swipes, `inViewport: true`. + * @throws {ScrollToElementError} When the element is not in view after + * `maxSwipes`, or the screen stops changing first. + */ +export async function scrollUntilInViewport( + deps: ScrollDeps, + query: ElementQuery, + direction: ScrollDirection, + maxSwipes: number, +): Promise { + const { windowSize } = deps; + let search: 'up' | 'down' = direction === 'down' ? 'down' : 'up'; + let turned = direction !== 'auto'; + let swipes = 0; + let snap = await deps.snapshot(); + let previousRaw: string | undefined; + let last: UIElement | null = null; + + for (;;) { + const element = findElement(snap.hierarchy, query); + const before = last; + if (element) { + last = element; + } + const fixed = + element !== null && + before !== null && + swipes > 0 && + snap.raw !== previousRaw && + sameFrame(element, before) && + intersectsViewport(element, windowSize); + if (element && fixed) { + // Does not scroll with the content, so momentum cannot move it. + return { element, swipes, inViewport: true }; + } + if (element && isInViewport(element, windowSize)) { + const settled = swipes === 0 ? null : await settle(deps, snap, query); + if (!settled) { + return { element, swipes, inViewport: true }; + } + if (settled.element && isInViewport(settled.element, windowSize)) { + return { element: settled.element, swipes, inViewport: true }; + } + // Momentum carried it out of view: judge again from the settled screen. + snap = settled.snap; + continue; + } + if (snap.raw === previousRaw) { + // The last swipe moved nothing: the end of the list in this direction. + if (direction === 'auto' && !element && !turned) { + turned = true; + search = search === 'up' ? 'down' : 'up'; + } else { + break; + } + } + if (swipes >= maxSwipes) { + break; + } + const next = + direction === 'auto' && element + ? swipeToward(element, windowSize) + : search; + previousRaw = snap.raw; + await deps.swipe(next); + swipes += 1; + snap = await deps.snapshot(); + } + + if (!last) { + throw new ScrollToElementError( + 'ELEMENT_NOT_FOUND', + `Element not found after ${swipes} swipe(s): ${JSON.stringify(query)}\n` + + 'Use device_snapshot to inspect the current UI hierarchy.', + { swipes }, + ); + } + const { frame } = last; + const [x, y, w, h] = [frame.x, frame.y, frame.width, frame.height].map( + Math.round, + ); + throw new ScrollToElementError( + 'ELEMENT_NOT_IN_VIEWPORT', + `Element is outside the ${windowSize.width}x${windowSize.height} viewport ` + + `after ${swipes} swipe(s): ${describeElement(last)} at ` + + `(${x}, ${y} ${w}x${h}).`, + { swipes, frame }, + ); +} + +/** + * Read the screen until two reads are equal, so a scroll that is still moving + * is not judged mid-flight. + * + * @param deps - Backend operations. + * @param first - The read that found the element in view. + * @param query - The element. + * @returns The settled read and the element in it, or null when the first + * read was already settled. + */ +async function settle( + deps: ScrollDeps, + first: SnapshotResult, + query: ElementQuery, +): Promise<{ snap: SnapshotResult; element: UIElement | null } | null> { + let previous = first; + for (let i = 0; i < MAX_SETTLE_READS; i++) { + const snap = await deps.snapshot(); + if (snap.raw === previous.raw) { + return i === 0 + ? null + : { snap, element: findElement(snap.hierarchy, query) }; + } + previous = snap; + } + return { snap: previous, element: findElement(previous.hierarchy, query) }; +}