From 4946b922bb0c4ba76b0e6e595c80c813e869f5a0 Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Thu, 29 Oct 2026 23:55:50 +0000 Subject: [PATCH 1/4] fix!: scrollToElement succeeds only when the element is in the viewport React Native keeps off-screen rows in the accessibility tree, so scrollToElement returned as soon as findElement matched, often after zero swipes with the element below the fold, and the next tap failed with 'Element is outside the viewport'. - success = enough of the frame inside the viewport (4% top/bottom inset; half the height, or 44 pt for tall elements); a fixed element that a swipe did not move is accepted when it intersects the viewport - read the screen until it settles after a swipe (momentum) - return { element, swipes, inViewport: true }; throw ScrollToElementError ELEMENT_NOT_FOUND / ELEMENT_NOT_IN_VIEWPORT (swipes + last frame) - direction 'auto' (default): finger direction from the element's position; searches toward content below first, turns once at the end - Android/Appium: scroll swipes from the real screen middle BREAKING CHANGE: scrollToElement returns ScrollToElementResult instead of UIElement; the default direction is 'auto' instead of 'down'. --- CHANGELOG.md | 6 + README.md | 19 +- src/backends/adb-backend.ts | 53 ++++-- src/backends/appium-backend.ts | 53 ++++-- src/backends/idb-backend.test.ts | 6 +- src/backends/idb-backend.ts | 33 ++-- src/backends/index.ts | 6 +- src/backends/types.ts | 27 ++- src/index.ts | 4 + src/tools/scroll-to-element.ts | 20 +- src/utils/scroll.test.ts | 302 +++++++++++++++++++++++++++++++ src/utils/scroll.ts | 262 +++++++++++++++++++++++++++ 12 files changed, 713 insertions(+), 78 deletions(-) create mode 100644 src/utils/scroll.test.ts create mode 100644 src/utils/scroll.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b7242e5..04be89e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `ScrollToElementError` (`ELEMENT_NOT_FOUND` / `ELEMENT_NOT_IN_VIEWPORT`, with `swipes` and the last `frame`) and the `ScrollDirection` / `ScrollToElementResult` types ([#50](https://github.com/MetaMask/device-mcp/pull/50)) - `tapElement(query, { element })` taps an element the caller already resolved (for example from `waitForElement`) without taking a second snapshot ### Changed +- **BREAKING:** fix: `scrollToElement` succeeds only when the element is in the viewport ([#50](https://github.com/MetaMask/device-mcp/pull/50)) + - It returns `{ element, swipes, inViewport: true }` instead of the element. Before, it returned as soon as the element was in the UI tree; React Native keeps off-screen rows in the tree, so it often returned after zero swipes with the element below the fold. + - After a swipe it reads the screen until it stops moving. It fails with `ScrollToElementError` after `maxAttempts` swipes, or when the screen stops changing. + - New default `direction: 'auto'`: the finger direction comes from the element's position (searching toward content below first while it is not in the tree). Explicit `up` / `down` stay finger directions. Before, the default `down` revealed content above. + - Android and Appium swipe from the middle of the real screen (40% of its height), not from fixed coordinates. - **BREAKING:** fix: iOS key presses, `back` and text clearing ([#48](https://github.com/MetaMask/device-mcp/pull/48)) - iOS `pressButton('enter')` and `dismissKeyboard()` send the integer HID key code (`idb ui key 40`); `home` and `lock` use `idb ui button`. Both failed before because `idb ui key` takes only integer codes. - iOS `pressButton('back')` now throws `UNSUPPORTED_ON_IOS` instead of pressing Home (which put the app in the background). diff --git a/README.md b/README.md index 10ffa0e..c2950d1 100644 --- a/README.md +++ b/README.md @@ -176,15 +176,16 @@ The `.device-session` file is typically written by the test runner when it creat ### Interaction -| Tool | Description | -| ------------------------ | ------------------------------------------------------------------ | -| `device_tap_element` | Find an element by label/identifier/text/type and tap its center. | -| `device_tap_coordinates` | Tap at exact screen coordinates. Last resort when queries fail. | -| `device_type` | Type text into the currently focused input field. | -| `device_swipe` | Swipe in a direction with optional start coordinates and distance. | -| `device_long_press` | Long press an element for context menus or drag initiation. | -| `device_wait_for` | Poll until an element matching a query appears. | -| `device_press_button` | Press a device button (home/back/enter/lock; no back on iOS). | +| Tool | Description | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `device_tap_element` | Find an element by label/identifier/text/type and tap its center. | +| `device_tap_coordinates` | Tap at exact screen coordinates. Last resort when queries fail. | +| `device_type` | Type text into the currently focused input field. | +| `device_swipe` | Swipe in a direction with optional start coordinates and distance. | +| `device_long_press` | Long press an element for context menus or drag initiation. | +| `device_scroll_to_element` | Swipe until an element is inside the viewport (not only in the UI tree); reports the swipe count. `direction` is the finger direction (`up` reveals content below); default `auto` picks it from the element's position. | +| `device_wait_for` | Poll until an element matching a query appears. | +| `device_press_button` | Press a device button (home/back/enter/lock; no back on iOS). | ### App & Device Control diff --git a/src/backends/adb-backend.ts b/src/backends/adb-backend.ts index dc176a6..b0c7225 100644 --- a/src/backends/adb-backend.ts +++ b/src/backends/adb-backend.ts @@ -43,6 +43,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -63,6 +65,7 @@ import { hardenArtifactFile, resolveArtifactPath, } from '../utils/output-path.js'; +import { scrollUntilInViewport } from '../utils/scroll.js'; // `uiautomator dump` exits 0 and prints its success banner even when it fails, so // the XML payload is the only trustworthy signal. @@ -900,26 +903,40 @@ export class AdbBackend implements DeviceBackend { async scrollToElement( query: ElementQuery, - direction: 'up' | 'down' = 'down', + direction: ScrollDirection = 'auto', maxAttempts = 10, - ): Promise { - let previousRaw = ''; + ): Promise { + const windowSize = await this.getWindowSize(); + return scrollUntilInViewport( + { + snapshot: async () => this.snapshot(), + windowSize, + swipe: async (finger) => this.#scrollSwipe(finger, windowSize), + }, + query, + direction, + maxAttempts, + ); + } - for (let i = 0; i < maxAttempts; i++) { - const snap = await this.snapshot(); - const element = findElement(snap.hierarchy, query); - if (element) { - return element; - } - if (snap.raw === previousRaw) { - break; - } - previousRaw = snap.raw; - await this.swipe(direction); - } - throw new Error( - `Element not found after scrolling: ${JSON.stringify(query)}\n` + - 'Use device_snapshot to inspect the current UI hierarchy.', + /** + * One scroll gesture from the middle of the screen over 40% of its height + * (the iOS backend's default), so the swipe matches the real screen size. + * + * @param finger - Finger direction. + * @param windowSize - The screen size, in the frame units. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + */ + async #scrollSwipe( + finger: 'up' | 'down', + { width, height }: WindowSize, + ): Promise { + await this.swipe( + finger, + Math.round(width / 2), + Math.round(height / 2), + Math.round(height * 0.4), ); } diff --git a/src/backends/appium-backend.ts b/src/backends/appium-backend.ts index eec2700..9e28106 100644 --- a/src/backends/appium-backend.ts +++ b/src/backends/appium-backend.ts @@ -10,6 +10,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -26,6 +28,7 @@ import { resolveArtifactPath, writeArtifactFile, } from '../utils/output-path.js'; +import { scrollUntilInViewport } from '../utils/scroll.js'; export class AppiumBackend implements DeviceBackend { readonly kind = 'appium' as const; @@ -425,26 +428,40 @@ export class AppiumBackend implements DeviceBackend { async scrollToElement( query: ElementQuery, - direction: 'up' | 'down' = 'down', + direction: ScrollDirection = 'auto', maxAttempts = 10, - ): Promise { - let previousRaw = ''; + ): Promise { + const windowSize = await this.getWindowSize(); + return scrollUntilInViewport( + { + snapshot: async () => this.snapshot(), + windowSize, + swipe: async (finger) => this.#scrollSwipe(finger, windowSize), + }, + query, + direction, + maxAttempts, + ); + } - for (let i = 0; i < maxAttempts; i++) { - const snap = await this.snapshot(); - const element = findElement(snap.hierarchy, query); - if (element) { - return element; - } - if (snap.raw === previousRaw) { - break; - } - previousRaw = snap.raw; - await this.swipe(direction); - } - throw new Error( - `Element not found after scrolling: ${JSON.stringify(query)}\n` + - 'Use device_snapshot to inspect the current UI hierarchy.', + /** + * One scroll gesture from the middle of the screen over 40% of its height + * (the iOS backend's default), so the swipe matches the real screen size. + * + * @param finger - Finger direction. + * @param windowSize - The screen size, in the frame units. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + */ + async #scrollSwipe( + finger: 'up' | 'down', + { width, height }: WindowSize, + ): Promise { + await this.swipe( + finger, + Math.round(width / 2), + Math.round(height / 2), + Math.round(height * 0.4), ); } diff --git a/src/backends/idb-backend.test.ts b/src/backends/idb-backend.test.ts index 6c335c9..d3a56e3 100644 --- a/src/backends/idb-backend.test.ts +++ b/src/backends/idb-backend.test.ts @@ -701,9 +701,9 @@ describe('IdbBackend.swipe and getWindowSize', () => { snapshotCount += 1; return JSON.stringify([ { - type: snapshotCount === 3 ? 'Button' : 'StaticText', - AXLabel: snapshotCount === 3 ? 'Settings' : `Page ${snapshotCount}`, - frame: { x: 10, y: 20, width: 100, height: 44 }, + type: snapshotCount >= 3 ? 'Button' : 'StaticText', + AXLabel: snapshotCount >= 3 ? 'Settings' : `Page ${snapshotCount}`, + frame: { x: 10, y: 200, width: 100, height: 44 }, enabled: true, }, ]); diff --git a/src/backends/idb-backend.ts b/src/backends/idb-backend.ts index c646cee..4b728d9 100644 --- a/src/backends/idb-backend.ts +++ b/src/backends/idb-backend.ts @@ -14,6 +14,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -38,6 +40,7 @@ import { idbSupportsNestedDescribe, resolveIdbPath, } from '../utils/platform.js'; +import { scrollUntilInViewport } from '../utils/scroll.js'; import { stopChildProcess } from '../utils/stop-child-process.js'; /** @@ -909,28 +912,20 @@ export class IdbBackend implements DeviceBackend { async scrollToElement( query: ElementQuery, - direction: 'up' | 'down' = 'down', + direction: ScrollDirection = 'auto', maxAttempts = 10, - ): Promise { + ): Promise { await this.ensureConnected(); const windowSize = await this.getWindowSize(); - let previousRaw = ''; - - for (let i = 0; i < maxAttempts; i++) { - const snap = await this.snapshot(); - const element = findElement(snap.hierarchy, query); - if (element) { - return element; - } - if (snap.raw === previousRaw) { - break; - } - previousRaw = snap.raw; - await this.#swipe(direction, windowSize); - } - throw new Error( - `Element not found after scrolling: ${JSON.stringify(query)}\n` + - 'Use device_snapshot to inspect the current UI hierarchy.', + return scrollUntilInViewport( + { + snapshot: async () => this.snapshot(), + windowSize, + swipe: async (finger) => this.#swipe(finger, windowSize), + }, + query, + direction, + maxAttempts, ); } diff --git a/src/backends/index.ts b/src/backends/index.ts index 8d4e98a..562e6b0 100644 --- a/src/backends/index.ts +++ b/src/backends/index.ts @@ -15,6 +15,8 @@ import type { ScreenshotFileResult, ScreenshotOptions, ScreenshotResult, + ScrollDirection, + ScrollToElementResult, SnapshotResult, TapElementOptions, TapResult, @@ -304,9 +306,9 @@ export function createLazyBackend( async scrollToElement( query: ElementQuery, - direction?: 'up' | 'down', + direction?: ScrollDirection, maxAttempts?: number, - ): Promise { + ): Promise { return (await resolve()).scrollToElement(query, direction, maxAttempts); }, diff --git a/src/backends/types.ts b/src/backends/types.ts index 4fb3452..7fa5af4 100644 --- a/src/backends/types.ts +++ b/src/backends/types.ts @@ -100,6 +100,17 @@ export type ElementQuery = { export type DeviceButton = 'home' | 'back' | 'enter' | 'lock'; +/** Finger direction of a scroll gesture, or `auto` (from the target's position). */ +export type ScrollDirection = 'up' | 'down' | 'auto'; + +export type ScrollToElementResult = { + /** The element as last read, with its in-view frame. */ + element: UIElement; + /** Swipes performed (0 when the element was already in view). */ + swipes: number; + inViewport: true; +}; + export type WindowSize = { width: number; height: number; @@ -191,11 +202,23 @@ export type DeviceBackend = { longPress(query: ElementQuery, durationMs?: number): Promise; + /** + * Swipe until the element is in the viewport (not only in the UI tree). + * + * @param query - The element to bring into view. + * @param direction - Finger direction (`up` reveals content below), or + * `auto` (default): from the element's position, searching downward first + * when it is not in the tree yet. + * @param maxAttempts - Maximum number of swipes (default 10). + * @returns The element as last read and the number of swipes. + * @throws {ScrollToElementError} `ELEMENT_NOT_FOUND` or + * `ELEMENT_NOT_IN_VIEWPORT` (with the last frame). + */ scrollToElement( query: ElementQuery, - direction?: 'up' | 'down', + direction?: ScrollDirection, maxAttempts?: number, - ): Promise; + ): Promise; getAlertText(): Promise; diff --git a/src/index.ts b/src/index.ts index b3f4a90..21ea0c3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,8 @@ export { createMcpServer } from './server.js'; export { detectAllDevices, MultipleDevicesError } from './utils/platform.js'; export { IdbUnavailableError } from './backends/idb-errors.js'; +export { ScrollToElementError } from './utils/scroll.js'; +export type { ScrollToElementErrorCode } from './utils/scroll.js'; export type { IdbErrorCode } from './backends/idb-errors.js'; export type { DetectedDevice } from './utils/platform.js'; @@ -23,6 +25,8 @@ export type { ScreenshotResult, ScreenshotFileResult, ScreenshotOptions, + ScrollDirection, + ScrollToElementResult, LogsResult, TapElementOptions, TapResult, diff --git a/src/tools/scroll-to-element.ts b/src/tools/scroll-to-element.ts index 0046db1..a8726f9 100644 --- a/src/tools/scroll-to-element.ts +++ b/src/tools/scroll-to-element.ts @@ -13,8 +13,10 @@ export function registerScrollToElementTool( { title: 'Scroll To Element', description: - 'Scroll the screen until an element matching the query becomes visible. ' + - 'Combines scrolling and element search into one atomic operation. ' + + 'Swipe until an element matching the query is inside the viewport ' + + '(being in the UI tree is not enough), and report how many swipes ' + + 'it took. Fails when the element is not in view after maxAttempts ' + + 'swipes or the screen stops moving. ' + 'Specify at least one of: label, identifier, text, type.', inputSchema: { label: z.string().optional().describe('Accessibility label to match'), @@ -25,18 +27,22 @@ export function registerScrollToElementTool( text: z.string().optional().describe('Text content to match'), type: z.string().optional().describe('Element type to match'), direction: z - .enum(['up', 'down']) + .enum(['auto', 'up', 'down']) .optional() - .describe('Scroll direction (default: down)'), + .describe( + 'Finger direction: "up" reveals content below, "down" content ' + + 'above. Default "auto": from where the element is, searching ' + + 'downward first when it is not in the UI tree yet.', + ), maxAttempts: z .number() .optional() - .describe('Maximum scroll attempts (default: 10)'), + .describe('Maximum number of swipes (default: 10)'), }, }, async ({ label, identifier, text, type, direction, maxAttempts }) => { try { - const element = await backend.scrollToElement( + const { element, swipes } = await backend.scrollToElement( { label, identifier, text, type }, direction, maxAttempts, @@ -46,7 +52,7 @@ export function registerScrollToElementTool( { type: 'text' as const, text: [ - `Found element after scrolling:`, + `Element in viewport after ${swipes} swipe(s):`, ` type: ${element.type}`, element.label ? ` label: "${element.label}"` : null, element.identifier ? ` id: "${element.identifier}"` : null, diff --git a/src/utils/scroll.test.ts b/src/utils/scroll.test.ts new file mode 100644 index 0000000..20c5b59 --- /dev/null +++ b/src/utils/scroll.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + isInViewport, + scrollUntilInViewport, + ScrollToElementError, + swipeToward, +} from './scroll.js'; +import type { SnapshotResult, UIElement } from '../backends/types.js'; + +const windowSize = { width: 402, height: 874 }; + +const node = (label: string, y: number, height = 24): UIElement => ({ + type: 'StaticText', + label, + frame: { x: 16, y, width: 370, height }, + enabled: true, +}); + +const snap = (elements: UIElement[]): SnapshotResult => ({ + hierarchy: elements, + raw: JSON.stringify(elements), +}); + +/** + * A scrolling list: rows at fixed content offsets, and a scroll offset that a + * finger-up swipe increases by `step` (clamped to [0, maxOffset]). + * + * @param rows - Content rows: label and content y. + * @param options - Scroll step, end of the list, fixed (non-scrolling) rows. + * @param options.step - Points moved per swipe. + * @param options.maxOffset - Largest scroll offset (end of the list). + * @param options.fixed - Rows that do not scroll (e.g. a footer button). + * @returns Backend stubs and the swipe log. + */ +function list( + rows: [string, number][], + { + step = 350, + maxOffset = 2000, + fixed = [] as UIElement[], + }: { step?: number; maxOffset?: number; fixed?: UIElement[] } = {}, +): { + deps: Parameters[0]; + swipes: string[]; +} { + let offset = 0; + const swipes: string[] = []; + return { + swipes, + deps: { + windowSize, + snapshot: vi.fn(async () => + snap([...rows.map(([label, y]) => node(label, y - offset)), ...fixed]), + ), + swipe: vi.fn(async (finger: 'up' | 'down') => { + swipes.push(finger); + offset = Math.min( + maxOffset, + Math.max(0, offset + (finger === 'up' ? step : -step)), + ); + }), + }, + }; +} + +describe('isInViewport', () => { + it('rejects an element that is in the tree but below the fold', () => { + expect(isInViewport(node('Account icon', 996), windowSize)).toBe(false); + }); + + it('rejects an element hidden under the bottom inset', () => { + // 874 * 0.96 = 839: only 9 of 24 pt are above the inset. + expect(isInViewport(node('Row', 830), windowSize)).toBe(false); + }); + + it('accepts an element that is fully in view', () => { + expect(isInViewport(node('Row', 400), windowSize)).toBe(true); + }); + + it('accepts a tall element once 44 pt of it are in view', () => { + expect(isInViewport(node('Card', 780, 600), windowSize)).toBe(true); + expect(isInViewport(node('Card', 800, 600), windowSize)).toBe(false); + }); + + it('rejects an element outside the viewport horizontally', () => { + const element = { + ...node('Row', 400), + frame: { x: 500, y: 400, width: 100, height: 24 }, + }; + expect(isInViewport(element, windowSize)).toBe(false); + }); +}); + +describe('swipeToward', () => { + it('swipes up (reveals content below) for an element below the middle', () => { + expect(swipeToward(node('Row', 996), windowSize)).toBe('up'); + }); + + it('swipes down (reveals content above) for an element above the middle', () => { + expect(swipeToward(node('Row', -300), windowSize)).toBe('down'); + }); +}); + +describe('scrollUntilInViewport', () => { + it('does not swipe when the element is already in view', async () => { + const { deps, swipes } = list([['Account icon', 400]]); + + const result = await scrollUntilInViewport( + deps, + { label: 'Account icon' }, + 'auto', + 10, + ); + + expect(result).toMatchObject({ swipes: 0, inViewport: true }); + expect(swipes).toStrictEqual([]); + }); + + it('swipes when the element is in the tree but off-screen, and returns it in view', async () => { + // The 2026-10-28 bug: found in the tree at y=996, returned after 0 swipes. + const { deps, swipes } = list([['Account icon', 996]]); + + const result = await scrollUntilInViewport( + deps, + { label: 'Account icon' }, + 'auto', + 10, + ); + + expect(swipes).toStrictEqual(['up']); + expect(result.swipes).toBe(1); + expect(result.inViewport).toBe(true); + expect(result.element.frame.y).toBe(646); + expect(isInViewport(result.element, windowSize)).toBe(true); + }); + + it('auto swipes down for an element above the viewport', async () => { + const { deps, swipes } = list([['Header', 400]]); + // Start scrolled down by 700 pt: Header is at y=-300, above the viewport. + await deps.swipe('up'); + await deps.swipe('up'); + swipes.length = 0; + + const result = await scrollUntilInViewport( + deps, + { label: 'Header' }, + 'auto', + 10, + ); + + expect(swipes[0]).toBe('down'); + expect(result.inViewport).toBe(true); + }); + + it('auto searches below first, then turns around when the list ends', async () => { + // The element is not in the tree until scrolled back to the top. + let offset = 700; + const swipes: string[] = []; + const deps = { + windowSize, + snapshot: vi.fn(async () => + snap(offset === 0 ? [node('Top', 200)] : [node(`at ${offset}`, 400)]), + ), + swipe: vi.fn(async (finger: 'up' | 'down') => { + swipes.push(finger); + offset = Math.min( + 700, + Math.max(0, offset + (finger === 'up' ? 350 : -350)), + ); + }), + }; + + const result = await scrollUntilInViewport( + deps, + { label: 'Top' }, + 'auto', + 10, + ); + + // up (already at the end: no change) -> turn -> down, down. + expect(swipes).toStrictEqual(['up', 'down', 'down']); + expect(result).toMatchObject({ swipes: 3, inViewport: true }); + }); + + it('honours an explicit finger direction', async () => { + const { deps, swipes } = list([['Account icon', 996]]); + + await expect( + scrollUntilInViewport(deps, { label: 'Account icon' }, 'down', 3), + ).rejects.toMatchObject({ + code: 'ELEMENT_NOT_IN_VIEWPORT', + frame: { y: 996 }, + }); + // Finger down at the top of the list moves nothing: stop after one swipe. + expect(swipes).toStrictEqual(['down']); + }); + + it('fails with the last frame when the list ends before the element is in view', async () => { + const { deps, swipes } = list([['Footer note', 2000]], { maxOffset: 500 }); + + const error = await scrollUntilInViewport( + deps, + { label: 'Footer note' }, + 'auto', + 10, + ).catch((e: unknown) => e); + + expect(error).toBeInstanceOf(ScrollToElementError); + expect(error).toMatchObject({ + code: 'ELEMENT_NOT_IN_VIEWPORT', + frame: { y: 1500 }, + }); + expect((error as Error).message).toContain('after 3 swipe(s)'); + expect(swipes).toStrictEqual(['up', 'up', 'up']); + }); + + it('fails with ELEMENT_NOT_FOUND when the element never appears', async () => { + const { deps } = list([['Row', 400]], { maxOffset: 0 }); + + await expect( + scrollUntilInViewport(deps, { label: 'Missing' }, 'auto', 10), + ).rejects.toMatchObject({ code: 'ELEMENT_NOT_FOUND', swipes: 2 }); + }); + + it('stops after maxAttempts swipes', async () => { + const { deps, swipes } = list([['Far', 20000]], { maxOffset: 100000 }); + + await expect( + scrollUntilInViewport(deps, { label: 'Far' }, 'auto', 4), + ).rejects.toMatchObject({ code: 'ELEMENT_NOT_IN_VIEWPORT', swipes: 4 }); + expect(swipes).toHaveLength(4); + }); + + it('accepts a fixed element that the swipe does not move', async () => { + // A footer button under the bottom inset never scrolls into the inset area. + const footer = node('Continue', 840, 30); + const { deps, swipes } = list([['Row', 400]], { fixed: [footer] }); + + const result = await scrollUntilInViewport( + deps, + { label: 'Continue' }, + 'auto', + 10, + ); + + expect(swipes).toStrictEqual(['up']); + expect(result).toMatchObject({ swipes: 1, inViewport: true }); + }); + + it('waits for the scroll to settle before returning', async () => { + // After the swipe, momentum moves the row 100 pt more over two reads. + const ys = [996, 646, 546, 546]; + let read = 0; + const deps = { + windowSize, + snapshot: vi.fn(async () => + snap([ + node('Account icon', ys[Math.min((read += 1) - 1, ys.length - 1)]), + ]), + ), + swipe: vi.fn(async () => undefined), + }; + + const result = await scrollUntilInViewport( + deps, + { label: 'Account icon' }, + 'auto', + 10, + ); + + expect(result.element.frame.y).toBe(546); + expect(deps.snapshot).toHaveBeenCalledTimes(4); + }); + + it('keeps scrolling when momentum carries the element out of view', async () => { + // In view right after the swipe, but the scroll keeps going past it. + const ys = [996, 400, -200, -200, 400, 400]; + let read = 0; + const swipes: string[] = []; + const deps = { + windowSize, + snapshot: vi.fn(async () => + snap([node('Row', ys[Math.min((read += 1) - 1, ys.length - 1)])]), + ), + swipe: vi.fn(async (finger: 'up' | 'down') => { + swipes.push(finger); + }), + }; + + const result = await scrollUntilInViewport( + deps, + { label: 'Row' }, + 'auto', + 10, + ); + + expect(swipes).toStrictEqual(['up', 'down']); + expect(result).toMatchObject({ swipes: 2, inViewport: true }); + expect(result.element.frame.y).toBe(400); + }); +}); diff --git a/src/utils/scroll.ts b/src/utils/scroll.ts new file mode 100644 index 0000000..aa59de8 --- /dev/null +++ b/src/utils/scroll.ts @@ -0,0 +1,262 @@ +import { describeElement, findElement } from './element.js'; +import type { + ElementQuery, + ScrollDirection, + ScrollToElementResult, + SnapshotResult, + UIElement, + WindowSize, +} from '../backends/types.js'; + +/** + * Part of the viewport height kept clear at the top and at the bottom when + * deciding whether an element is in view (about 35 pt on an 874 pt iPhone): + * the status bar and the iOS home indicator overlay the scrolled content. + */ +const VIEWPORT_INSET = 0.04; + +/** A tall element counts as in view once this many points of it are. */ +const MIN_VISIBLE = 44; + +/** Extra reads, after the target is in view, to let the scroll momentum stop. */ +const MAX_SETTLE_READS = 3; + +export type ScrollToElementErrorCode = + | 'ELEMENT_NOT_FOUND' + | 'ELEMENT_NOT_IN_VIEWPORT'; + +/** + * `scrollToElement` could not bring the element into view. `code` tells + * whether the element was never in the UI tree (`ELEMENT_NOT_FOUND`) or was + * found but stayed outside the viewport (`ELEMENT_NOT_IN_VIEWPORT`, with its + * last `frame`). + */ +export class ScrollToElementError extends Error { + readonly code: ScrollToElementErrorCode; + + readonly swipes: number; + + readonly frame: UIElement['frame'] | undefined; + + constructor( + code: ScrollToElementErrorCode, + message: string, + details: { swipes: number; frame?: UIElement['frame'] }, + ) { + super(message); + this.name = 'ScrollToElementError'; + this.code = code; + this.swipes = details.swipes; + this.frame = details.frame; + } +} + +/** + * Whether enough of the element is inside the viewport, after removing + * {@link VIEWPORT_INSET} at the top and bottom, to see and tap it: half of + * its height, or {@link MIN_VISIBLE} points for a tall element. + * + * @param element - The element. + * @param windowSize - The viewport size, in the same units as the frame. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + * @returns True when the element is in view. + */ +export function isInViewport( + element: UIElement, + { width, height }: WindowSize, +): boolean { + const { x, y, width: w, height: h } = element.frame; + const top = Math.max(y, height * VIEWPORT_INSET); + const bottom = Math.min(y + h, height * (1 - VIEWPORT_INSET)); + const visibleWidth = Math.min(x + w, width) - Math.max(x, 0); + return visibleWidth > 0 && bottom - top >= Math.min(h / 2, MIN_VISIBLE); +} + +/** + * Whether any part of the element is inside the viewport (the rule a tap + * uses). + * + * @param element - The element. + * @param windowSize - The viewport size. + * @param windowSize.width - Viewport width. + * @param windowSize.height - Viewport height. + * @returns True when the frame intersects the viewport. + */ +function intersectsViewport( + element: UIElement, + { width, height }: WindowSize, +): boolean { + const { x, y, width: w, height: h } = element.frame; + return ( + Math.min(x + w, width) > Math.max(x, 0) && + Math.min(y + h, height) > Math.max(y, 0) + ); +} + +function sameFrame(a: UIElement, b: UIElement): boolean { + return ( + a.frame.x === b.frame.x && + a.frame.y === b.frame.y && + a.frame.width === b.frame.width && + a.frame.height === b.frame.height + ); +} + +/** + * The finger direction that brings an off-screen element toward the middle + * of the viewport: up for an element below it (reveals content below), down + * for one above it. + * + * @param element - The element. + * @param windowSize - The viewport size. + * @returns The swipe direction. + */ +export function swipeToward( + element: UIElement, + windowSize: WindowSize, +): 'up' | 'down' { + const centerY = element.frame.y + element.frame.height / 2; + return centerY > windowSize.height / 2 ? 'up' : 'down'; +} + +export type ScrollDeps = { + snapshot: () => Promise; + windowSize: WindowSize; + /** One scroll gesture; `direction` is the finger direction. */ + swipe: (direction: 'up' | 'down') => Promise; +}; + +/** + * Swipe until the element matching `query` is in the viewport. + * + * Finding the element in the UI tree is not enough: React Native keeps + * off-screen rows in the accessibility tree, so the element must also pass + * {@link isInViewport}. An element that a swipe did not move while the screen + * changed is fixed (for example a footer button) and is accepted when it + * intersects the viewport. After a swipe, the screen is read again until it + * stops changing (scroll momentum) before the result is returned. + * + * `direction` is the finger direction. With `auto`, the direction comes from + * the element's position; while the element is not in the tree, the search + * goes toward content below (finger up) and turns around once when the + * screen stops changing. + * + * @param deps - Backend operations. + * @param query - The element to bring into view. + * @param direction - Finger direction, or `auto`. + * @param maxSwipes - Maximum number of swipes. + * @returns The element as last read, the number of swipes, `inViewport: true`. + * @throws {ScrollToElementError} When the element is not in view after + * `maxSwipes`, or the screen stops changing first. + */ +export async function scrollUntilInViewport( + deps: ScrollDeps, + query: ElementQuery, + direction: ScrollDirection, + maxSwipes: number, +): Promise { + const { windowSize } = deps; + let search: 'up' | 'down' = direction === 'down' ? 'down' : 'up'; + let turned = direction !== 'auto'; + let swipes = 0; + let snap = await deps.snapshot(); + let previousRaw: string | undefined; + let last: UIElement | null = null; + + for (;;) { + const element = findElement(snap.hierarchy, query); + const before = last; + if (element) { + last = element; + } + const fixed = + element !== null && + before !== null && + swipes > 0 && + snap.raw !== previousRaw && + sameFrame(element, before) && + intersectsViewport(element, windowSize); + if (element && fixed) { + // Does not scroll with the content, so momentum cannot move it. + return { element, swipes, inViewport: true }; + } + if (element && isInViewport(element, windowSize)) { + const settled = swipes === 0 ? null : await settle(deps, snap, query); + if (!settled) { + return { element, swipes, inViewport: true }; + } + if (settled.element && isInViewport(settled.element, windowSize)) { + return { element: settled.element, swipes, inViewport: true }; + } + // Momentum carried it out of view: judge again from the settled screen. + snap = settled.snap; + continue; + } + if (snap.raw === previousRaw) { + // The last swipe moved nothing: the end of the list in this direction. + if (direction === 'auto' && !element && !turned) { + turned = true; + search = search === 'up' ? 'down' : 'up'; + } else { + break; + } + } + if (swipes >= maxSwipes) { + break; + } + const next = + direction === 'auto' && element + ? swipeToward(element, windowSize) + : search; + previousRaw = snap.raw; + await deps.swipe(next); + swipes += 1; + snap = await deps.snapshot(); + } + + if (!last) { + throw new ScrollToElementError( + 'ELEMENT_NOT_FOUND', + `Element not found after ${swipes} swipe(s): ${JSON.stringify(query)}\n` + + 'Use device_snapshot to inspect the current UI hierarchy.', + { swipes }, + ); + } + const { frame } = last; + throw new ScrollToElementError( + 'ELEMENT_NOT_IN_VIEWPORT', + `Element is outside the ${windowSize.width}x${windowSize.height} viewport ` + + `after ${swipes} swipe(s): ${describeElement(last)} at ` + + `(${frame.x}, ${frame.y} ${frame.width}x${frame.height}).`, + { swipes, frame }, + ); +} + +/** + * Read the screen until two reads are equal, so a scroll that is still moving + * is not judged mid-flight. + * + * @param deps - Backend operations. + * @param first - The read that found the element in view. + * @param query - The element. + * @returns The settled read and the element in it, or null when the first + * read was already settled. + */ +async function settle( + deps: ScrollDeps, + first: SnapshotResult, + query: ElementQuery, +): Promise<{ snap: SnapshotResult; element: UIElement | null } | null> { + let previous = first; + for (let i = 0; i < MAX_SETTLE_READS; i++) { + const snap = await deps.snapshot(); + if (snap.raw === previous.raw) { + return i === 0 + ? null + : { snap, element: findElement(snap.hierarchy, query) }; + } + previous = snap; + } + return { snap: previous, element: findElement(previous.hierarchy, query) }; +} From 8920003cf113a12f814d127eac3de84289d7af1a Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Fri, 30 Oct 2026 00:07:52 +0000 Subject: [PATCH 2/4] fix: round the frame in the scroll error message --- src/utils/scroll.test.ts | 1 + src/utils/scroll.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils/scroll.test.ts b/src/utils/scroll.test.ts index 20c5b59..a3b8782 100644 --- a/src/utils/scroll.test.ts +++ b/src/utils/scroll.test.ts @@ -212,6 +212,7 @@ describe('scrollUntilInViewport', () => { frame: { y: 1500 }, }); expect((error as Error).message).toContain('after 3 swipe(s)'); + expect((error as Error).message).toContain('at (16, 1500 370x24)'); expect(swipes).toStrictEqual(['up', 'up', 'up']); }); diff --git a/src/utils/scroll.ts b/src/utils/scroll.ts index aa59de8..e79cd0a 100644 --- a/src/utils/scroll.ts +++ b/src/utils/scroll.ts @@ -224,11 +224,14 @@ export async function scrollUntilInViewport( ); } const { frame } = last; + const [x, y, w, h] = [frame.x, frame.y, frame.width, frame.height].map( + Math.round, + ); throw new ScrollToElementError( 'ELEMENT_NOT_IN_VIEWPORT', `Element is outside the ${windowSize.width}x${windowSize.height} viewport ` + `after ${swipes} swipe(s): ${describeElement(last)} at ` + - `(${frame.x}, ${frame.y} ${frame.width}x${frame.height}).`, + `(${x}, ${y} ${w}x${h}).`, { swipes, frame }, ); } From 91f29221d802ebb512c1ba6d3709892f23e22b2b Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Fri, 30 Oct 2026 00:11:06 +0000 Subject: [PATCH 3/4] fix: a zero-height frame is not in the viewport (Android clipped bounds) --- src/utils/scroll.test.ts | 4 ++++ src/utils/scroll.ts | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/utils/scroll.test.ts b/src/utils/scroll.test.ts index a3b8782..91753ea 100644 --- a/src/utils/scroll.test.ts +++ b/src/utils/scroll.test.ts @@ -83,6 +83,10 @@ describe('isInViewport', () => { expect(isInViewport(node('Card', 800, 600), windowSize)).toBe(false); }); + it('rejects a zero-height frame (Android clips hidden rows to the list edge)', () => { + expect(isInViewport(node('Currency', 210, 0), windowSize)).toBe(false); + }); + it('rejects an element outside the viewport horizontally', () => { const element = { ...node('Row', 400), diff --git a/src/utils/scroll.ts b/src/utils/scroll.ts index e79cd0a..a17e402 100644 --- a/src/utils/scroll.ts +++ b/src/utils/scroll.ts @@ -54,7 +54,8 @@ export class ScrollToElementError extends Error { /** * Whether enough of the element is inside the viewport, after removing * {@link VIEWPORT_INSET} at the top and bottom, to see and tap it: half of - * its height, or {@link MIN_VISIBLE} points for a tall element. + * its height, or {@link MIN_VISIBLE} points for a tall element. A zero-size + * frame is never in view. * * @param element - The element. * @param windowSize - The viewport size, in the same units as the frame. @@ -70,7 +71,14 @@ export function isInViewport( const top = Math.max(y, height * VIEWPORT_INSET); const bottom = Math.min(y + h, height * (1 - VIEWPORT_INSET)); const visibleWidth = Math.min(x + w, width) - Math.max(x, 0); - return visibleWidth > 0 && bottom - top >= Math.min(h / 2, MIN_VISIBLE); + const visibleHeight = bottom - top; + // Android clips an off-screen node's bounds to the edge of its scroll + // container, so a hidden row can be reported with zero height. + return ( + visibleWidth > 0 && + visibleHeight > 0 && + visibleHeight >= Math.min(h / 2, MIN_VISIBLE) + ); } /** From df00ae80906211529d56c4f2eb74c8e590ac09d6 Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Fri, 30 Oct 2026 00:28:33 +0000 Subject: [PATCH 4/4] test: complete the SnapshotResult fixture in the scroll tests --- src/utils/scroll.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils/scroll.test.ts b/src/utils/scroll.test.ts index 91753ea..b886b0c 100644 --- a/src/utils/scroll.test.ts +++ b/src/utils/scroll.test.ts @@ -18,8 +18,10 @@ const node = (label: string, y: number, height = 24): UIElement => ({ }); const snap = (elements: UIElement[]): SnapshotResult => ({ + platform: 'ios', hierarchy: elements, raw: JSON.stringify(elements), + timestamp: 0, }); /**