diff --git a/CHANGELOG.md b/CHANGELOG.md index 04be89e..3463cdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `TargetObscuredError` (`code: 'TARGET_OBSCURED'`, with `obscuredBy`, `keyboard` and the tap point) ([#51](https://github.com/MetaMask/device-mcp/pull/51)) - `ScrollToElementError` (`ELEMENT_NOT_FOUND` / `ELEMENT_NOT_IN_VIEWPORT`, with `swipes` and the last `frame`) and the `ScrollDirection` / `ScrollToElementResult` types ([#50](https://github.com/MetaMask/device-mcp/pull/50)) - `tapElement(query, { element })` taps an element the caller already resolved (for example from `waitForElement`) without taking a second snapshot ### Changed +- fix: taps fail instead of landing on another element ([#51](https://github.com/MetaMask/device-mcp/pull/51)) + - iOS `tapElement` / `longPress` hit-test the tap point first (`idb ui describe-point --api axbridge`, about 0.25 s). When another element is there (the keyboard or its AutoFill bar, a sheet, an overlay), they throw `TargetObscuredError` and do not tap. When the hit-test finds nothing, or idb cannot select the accessibility API, the tap goes ahead. + - Android `tapElement` / `longPress` tap the visible part of the frame and refuse an element outside the screen (including rows clipped to zero height), like iOS. The screen size is read once per backend. + - The "outside the viewport" tap error (iOS and Android) has `code: 'ELEMENT_NOT_IN_VIEWPORT'`. - **BREAKING:** fix: `scrollToElement` succeeds only when the element is in the viewport ([#50](https://github.com/MetaMask/device-mcp/pull/50)) - It returns `{ element, swipes, inViewport: true }` instead of the element. Before, it returned as soon as the element was in the UI tree; React Native keeps off-screen rows in the tree, so it often returned after zero swipes with the element below the fold. - After a swipe it reads the screen until it stops moving. It fails with `ScrollToElementError` after `maxAttempts` swipes, or when the screen stops changing. diff --git a/src/backends/adb-backend.test.ts b/src/backends/adb-backend.test.ts index f47a51f..71749f6 100644 --- a/src/backends/adb-backend.test.ts +++ b/src/backends/adb-backend.test.ts @@ -630,9 +630,14 @@ describe('AdbBackend.tapElement', () => { enabled: true, }; + const tapCalls = (): unknown[][] => + mockExecStrict.mock.calls.filter(([, args]) => args?.includes('tap')); + beforeEach(() => { vi.clearAllMocks(); - mockExecStrict.mockResolvedValue(''); + mockExecStrict.mockImplementation(async (_cmd, args) => + args?.includes('wm') ? 'Physical size: 1080x2400' : '', + ); backend = new AdbBackend('emulator-5554'); }); @@ -643,7 +648,10 @@ describe('AdbBackend.tapElement', () => { ); expect(result).toMatchObject({ success: true, x: 300, y: 230 }); - expect(mockExecStrict).toHaveBeenCalledTimes(1); + // One `wm size` (cached per backend) and the tap; no uiautomator dump. + expect( + mockExecStrict.mock.calls.map(([, args]) => args?.[3]), + ).toStrictEqual(['wm', 'input']); expect(mockExecStrict).toHaveBeenCalledWith('adb', [ '-s', 'emulator-5554', @@ -655,6 +663,67 @@ describe('AdbBackend.tapElement', () => { ]); }); + it('reads the screen size once per backend', async () => { + await backend.tapElement({ identifier: 'submit' }, { element }); + await backend.tapElement({ identifier: 'submit' }, { element }); + + expect( + mockExecStrict.mock.calls.filter(([, args]) => args?.includes('wm')), + ).toHaveLength(1); + }); + + it('taps the visible part of a partly off-screen element', async () => { + const partly = { + ...element, + frame: { x: 100, y: 2300, width: 400, height: 300 }, + }; + + const result = await backend.tapElement( + { identifier: 'submit' }, + { element: partly }, + ); + + // Visible part: y 2300..2400 -> center 2350. + expect(result).toMatchObject({ x: 300, y: 2350 }); + }); + + it.each([ + ['below the screen', { x: 42, y: 2513, width: 996, height: 63 }], + [ + 'clipped to zero height at the list edge', + { x: 42, y: 2513, width: 996, height: 0 }, + ], + ])('refuses to tap an element %s', async (_name, frame) => { + await expect( + backend.tapElement( + { identifier: 'submit' }, + { element: { ...element, frame } }, + ), + ).rejects.toMatchObject({ + code: 'ELEMENT_NOT_IN_VIEWPORT', + message: expect.stringContaining('outside the 1080x2400 viewport'), + }); + expect(tapCalls()).toHaveLength(0); + }); + + it('refuses to long-press an element outside the screen', async () => { + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.includes('wm')) { + return 'Physical size: 1080x2400'; + } + return args?.includes('cat') || args?.includes('dump') + ? `` + : ''; + }); + + await expect(backend.longPress({ identifier: 'far' })).rejects.toThrow( + 'outside the 1080x2400 viewport', + ); + expect( + mockExecStrict.mock.calls.filter(([, args]) => args?.includes('swipe')), + ).toHaveLength(0); + }); + it('snapshots to find the element when none is passed', async () => { mockExecStrict.mockImplementation(async (_cmd, args) => args?.includes('cat') || args?.includes('dump') diff --git a/src/backends/adb-backend.ts b/src/backends/adb-backend.ts index b0c7225..fb079e8 100644 --- a/src/backends/adb-backend.ts +++ b/src/backends/adb-backend.ts @@ -59,6 +59,7 @@ import { findElement, describeElement, computeSwipeEnd, + visibleCenter, } from '../utils/element.js'; import { exec, execStrict, isCommandAvailable } from '../utils/exec.js'; import { @@ -219,6 +220,8 @@ export class AdbBackend implements DeviceBackend { readonly #recordingRemotePath = '/sdcard/device-mcp-recording.mp4'; + #windowSize: Promise | undefined; + // Installation is cached; trust is still checked before every helper run. #helperInstalled = false; @@ -313,8 +316,7 @@ export class AdbBackend implements DeviceBackend { ); } - const x = Math.round(element.frame.x + element.frame.width / 2); - const y = Math.round(element.frame.y + element.frame.height / 2); + const { x, y } = visibleCenter(element, await this.#cachedWindowSize()); await this.tapCoordinates(x, y); return { @@ -880,8 +882,10 @@ export class AdbBackend implements DeviceBackend { ); } - const cx = Math.round(element.frame.x + element.frame.width / 2); - const cy = Math.round(element.frame.y + element.frame.height / 2); + const { x: cx, y: cy } = visibleCenter( + element, + await this.#cachedWindowSize(), + ); await this.#adb([ 'shell', 'input', @@ -952,6 +956,20 @@ export class AdbBackend implements DeviceBackend { return texts.join('\n'); } + /** + * The screen size, read once per backend (one `adb shell wm size`) for the + * tap viewport check. A failed read is not cached. + * + * @returns The screen size in pixels. + */ + async #cachedWindowSize(): Promise { + this.#windowSize ??= this.getWindowSize().catch((error: unknown) => { + this.#windowSize = undefined; + throw error; + }); + return this.#windowSize; + } + async getWindowSize(): Promise { const raw = await this.#adb(['shell', 'wm', 'size']); const match = raw.match(/(\d+)x(\d+)/u); diff --git a/src/backends/idb-backend.test.ts b/src/backends/idb-backend.test.ts index d3a56e3..d61d893 100644 --- a/src/backends/idb-backend.test.ts +++ b/src/backends/idb-backend.test.ts @@ -10,6 +10,7 @@ import { IdbUnavailableError } from './idb-errors.js'; import type { UIElement } from './types.js'; import { findElement, generateLocators } from '../utils/element.js'; import * as execModule from '../utils/exec.js'; +import { TargetObscuredError } from '../utils/hit-test.js'; import * as platformModule from '../utils/platform.js'; const connectionMocks = vi.hoisted(() => ({ @@ -867,6 +868,240 @@ describe('IdbBackend.tapElement and longPress viewport handling', () => { }); }); +describe('IdbBackend tap hit-test', () => { + const udid = 'AAAA1111-BBBB-CCCC-DDDD-EEEE2222FFFF'; + let backend: IdbBackend; + + const loginButton: UIElement = { + type: 'Button', + label: 'Unlock', + identifier: 'log-in-button', + frame: { x: 16.3, y: 532.3, width: 369.3, height: 48 }, + enabled: true, + }; + + /** + * @param hit - What `describe-point` returns: a node, an error, or raw text. + */ + const mockDevice = (hit: Record | Error | string): void => { + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.[0] === 'describe' && args.includes('--json')) { + return JSON.stringify({ + os_version: '26.5', + screen_dimensions: { width_points: 402, height_points: 874 }, + }); + } + if (args?.[1] === 'describe-point') { + if (hit instanceof Error) { + throw hit; + } + return typeof hit === 'string' ? hit : JSON.stringify(hit); + } + return ''; + }); + }; + + const tapCalls = (): unknown[][] => + mockExecStrict.mock.calls.filter(([, args]) => args?.[1] === 'tap'); + + beforeEach(() => { + vi.clearAllMocks(); + connectionState.alive = true; + mockSupportsApi.mockResolvedValue(true); + mockSupportsNested.mockResolvedValue(true); + mockExec.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); + backend = new IdbBackend(udid); + }); + + it('hit-tests the tap point with describe-point before tapping', async () => { + mockDevice({ + type: 'Button', + AXLabel: 'Unlock', + AXUniqueId: 'log-in-button', + frame: loginButton.frame, + }); + + await backend.tapElement( + { identifier: 'log-in-button' }, + { element: loginButton }, + ); + + const calls = mockExecStrict.mock.calls + .map(([, args]) => args) + .filter((args) => args[0] === 'ui'); + expect(calls).toStrictEqual([ + [ + 'ui', + 'describe-point', + '201', + '556', + '--api', + 'axbridge', + '--json', + '--udid', + udid, + ], + ['ui', 'tap', '201', '556', '--udid', udid], + ]); + }); + + it('accepts a child of the target at the point (a label inside the button)', async () => { + mockDevice({ + type: 'StaticText', + AXLabel: 'Unlock', + frame: { x: 170, y: 545, width: 60, height: 22 }, + }); + + await backend.tapElement( + { identifier: 'log-in-button' }, + { element: loginButton }, + ); + + expect(tapCalls()).toHaveLength(1); + }); + + it('refuses the tap when the keyboard covers the target (B2)', async () => { + // Keyboard up on the login screen: the AutoFill bar sits over Unlock. + mockDevice({ + type: 'Button', + AXLabel: 'Passwords', + AXUniqueId: null, + frame: { x: 30, y: 546, width: 342, height: 44 }, + }); + + const error = await backend + .tapElement({ identifier: 'log-in-button' }, { element: loginButton }) + .catch((e: unknown) => e); + + expect(error).toBeInstanceOf(TargetObscuredError); + expect(error).toMatchObject({ + code: 'TARGET_OBSCURED', + keyboard: true, + x: 201, + y: 556, + obscuredBy: { type: 'Button', label: 'Passwords' }, + }); + expect((error as Error).message).toContain('by the keyboard'); + expect(tapCalls()).toHaveLength(0); + }); + + it('refuses the tap when a key is at the point', async () => { + mockDevice({ + type: 'Key', + AXLabel: 'g', + frame: { x: 182, y: 651, width: 39, height: 54 }, + }); + const lowButton = { + ...loginButton, + frame: { x: 16, y: 660, width: 370, height: 48 }, + }; + + await expect( + backend.tapElement( + { identifier: 'log-in-button' }, + { element: lowButton }, + ), + ).rejects.toMatchObject({ code: 'TARGET_OBSCURED', keyboard: true }); + expect(tapCalls()).toHaveLength(0); + }); + + it('refuses the tap when an overlay covers the target', async () => { + mockDevice({ + type: 'Other', + AXLabel: 'Bottom sheet', + AXUniqueId: 'sheet', + frame: { x: 0, y: 400, width: 402, height: 474 }, + }); + + await expect( + backend.tapElement( + { identifier: 'log-in-button' }, + { element: loginButton }, + ), + ).rejects.toMatchObject({ + code: 'TARGET_OBSCURED', + keyboard: false, + message: expect.stringContaining('Close the overlay'), + }); + expect(tapCalls()).toHaveLength(0); + }); + + it('taps when the hit-test finds no element at the point', async () => { + mockDevice( + new Error('The axbridge backend found no element at (201.0, 556.0)'), + ); + + await backend.tapElement( + { identifier: 'log-in-button' }, + { element: loginButton }, + ); + + expect(tapCalls()).toHaveLength(1); + }); + + it('taps when the hit-test output is not JSON', async () => { + mockDevice('not json'); + + await backend.tapElement( + { identifier: 'log-in-button' }, + { element: loginButton }, + ); + + expect(tapCalls()).toHaveLength(1); + }); + + it('skips the hit-test when idb cannot select the accessibility API', async () => { + mockSupportsApi.mockResolvedValue(false); + mockDevice(new Error('must not be called')); + + await backend.tapElement( + { identifier: 'log-in-button' }, + { element: loginButton }, + ); + + expect( + mockExecStrict.mock.calls.filter( + ([, args]) => args?.[1] === 'describe-point', + ), + ).toHaveLength(0); + expect(tapCalls()).toHaveLength(1); + }); + + it('hit-tests long presses too', async () => { + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.[0] === 'describe' && args.includes('--json')) { + return JSON.stringify({ + os_version: '26.5', + screen_dimensions: { width_points: 402, height_points: 874 }, + }); + } + if (args?.[1] === 'describe-all') { + return JSON.stringify([ + { + type: 'Button', + AXLabel: 'Unlock', + AXUniqueId: 'log-in-button', + frame: loginButton.frame, + }, + ]); + } + if (args?.[1] === 'describe-point') { + return JSON.stringify({ + type: 'Key', + AXLabel: 'space', + frame: { x: 100, y: 540, width: 200, height: 40 }, + }); + } + return ''; + }); + + await expect( + backend.longPress({ identifier: 'log-in-button' }), + ).rejects.toMatchObject({ code: 'TARGET_OBSCURED' }); + expect(tapCalls()).toHaveLength(0); + }); +}); + describe('IdbBackend device description cache', () => { const udid = 'AAAA1111-BBBB-CCCC-DDDD-EEEE2222FFFF'; let backend: IdbBackend; diff --git a/src/backends/idb-backend.ts b/src/backends/idb-backend.ts index 4b728d9..befed2b 100644 --- a/src/backends/idb-backend.ts +++ b/src/backends/idb-backend.ts @@ -27,9 +27,15 @@ import { findElement, describeElement, computeSwipeEnd, + visibleCenter, } from '../utils/element.js'; import { execStrict, exec } from '../utils/exec.js'; import type { ExecResult } from '../utils/exec.js'; +import { + hitsTarget, + isKeyboardElement, + TargetObscuredError, +} from '../utils/hit-test.js'; import { hardenArtifactFile, resolveArtifactPath, @@ -492,6 +498,7 @@ export class IdbBackend implements DeviceBackend { } const { x, y } = await this.#visibleTapPoint(element); + await this.#assertHittable(element, { x, y }); await this.tapCoordinates(x, y); return { @@ -502,37 +509,68 @@ export class IdbBackend implements DeviceBackend { }; } + async #visibleTapPoint( + element: UIElement, + ): Promise<{ x: number; y: number }> { + return visibleCenter(element, await this.getWindowSize()); + } + /** - * Returns the center of the part of the element's frame that lies inside - * the device viewport, in logical points. Tapping there is safer than - * tapping the raw frame center, which can fall outside the screen for - * off-screen or partially scrolled elements and either hit nothing or an - * unrelated on-screen control. + * Check that a tap at the point reaches the element, with one + * `idb ui describe-point` (about 0.25 s). Throws when another element (an + * overlay, a sheet, the keyboard) is on top. When the hit-test finds no + * element, or the idb version cannot run it, the tap goes ahead: the check + * only blocks taps it knows are wrong. * * @param element - The element to tap. - * @returns The tap point in logical points. - * @throws When the element's frame does not intersect the viewport at all. + * @param point - The tap point. + * @param point.x - X in points. + * @param point.y - Y in points. + * @throws {TargetObscuredError} When another element is at the point. */ - async #visibleTapPoint( + async #assertHittable( element: UIElement, - ): Promise<{ x: number; y: number }> { - const { width, height } = await this.getWindowSize(); - const x0 = Math.max(element.frame.x, 0); - const y0 = Math.max(element.frame.y, 0); - const x1 = Math.min(element.frame.x + element.frame.width, width); - const y1 = Math.min(element.frame.y + element.frame.height, height); - if (x1 <= x0 || y1 <= y0) { - throw new Error( - `Element is outside the ${width}x${height} viewport: ` + - `${describeElement(element)} at (${element.frame.x}, ${element.frame.y} ` + - `${element.frame.width}x${element.frame.height}). ` + - 'Scroll it into view first (device_swipe or scroll_to_element).', + point: { x: number; y: number }, + ): Promise { + if (!this.#idbSupportsApi) { + return; + } + const args = [ + 'ui', + 'describe-point', + String(point.x), + String(point.y), + '--api', + 'axbridge', + '--json', + '--udid', + this.#udid, + ]; + let raw: string; + try { + raw = await this.#execStrict(this.#idbPath, args); + } catch (error) { + if (error instanceof IdbUnavailableError) { + throw error; + } + // "found no element at (x, y)" and other hit-test failures: unknown, + // not obscured. + return; + } + let hit: UIElement; + try { + hit = mapIdbElement(JSON.parse(raw) as Record); + } catch { + return; + } + if (!hitsTarget(element, hit)) { + throw new TargetObscuredError( + element, + hit, + point, + isKeyboardElement(hit), ); } - return { - x: Math.round((x0 + x1) / 2), - y: Math.round((y0 + y1) / 2), - }; } async getElementText(query: ElementQuery): Promise { @@ -891,6 +929,7 @@ export class IdbBackend implements DeviceBackend { } const { x: cx, y: cy } = await this.#visibleTapPoint(element); + await this.#assertHittable(element, { x: cx, y: cy }); await this.#execStrict(this.#idbPath, [ 'ui', 'tap', diff --git a/src/index.ts b/src/index.ts index 21ea0c3..fc49fa0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,7 @@ export { createMcpServer } from './server.js'; export { detectAllDevices, MultipleDevicesError } from './utils/platform.js'; export { IdbUnavailableError } from './backends/idb-errors.js'; export { ScrollToElementError } from './utils/scroll.js'; +export { TargetObscuredError } from './utils/hit-test.js'; export type { ScrollToElementErrorCode } from './utils/scroll.js'; export type { IdbErrorCode } from './backends/idb-errors.js'; export type { DetectedDevice } from './utils/platform.js'; diff --git a/src/utils/element.ts b/src/utils/element.ts index 8c5c571..683d40d 100644 --- a/src/utils/element.ts +++ b/src/utils/element.ts @@ -253,3 +253,43 @@ export function formatHierarchy(elements: UIElement[], depth: number): string { }) .join('\n'); } + +/** + * The center of the part of the element's frame that lies inside the + * viewport. Tapping there is safer than tapping the raw frame center, which + * can fall outside the screen for off-screen or partly scrolled elements and + * hit nothing or an unrelated control. + * + * @param element - The element to tap. + * @param viewport - The viewport size, in the frame's units. + * @param viewport.width - Viewport width. + * @param viewport.height - Viewport height. + * @returns The tap point. + * @throws An error with `code: 'ELEMENT_NOT_IN_VIEWPORT'` when the frame + * does not intersect the viewport (or has no area). + */ +export function visibleCenter( + element: UIElement, + { width, height }: { width: number; height: number }, +): { x: number; y: number } { + const x0 = Math.max(element.frame.x, 0); + const y0 = Math.max(element.frame.y, 0); + const x1 = Math.min(element.frame.x + element.frame.width, width); + const y1 = Math.min(element.frame.y + element.frame.height, height); + if (x1 <= x0 || y1 <= y0) { + throw Object.assign( + new Error( + `Element is outside the ${width}x${height} viewport: ` + + `${describeElement(element)} at (${Math.round(element.frame.x)}, ` + + `${Math.round(element.frame.y)} ${Math.round(element.frame.width)}x` + + `${Math.round(element.frame.height)}). ` + + 'Scroll it into view first (device_swipe or scroll_to_element).', + ), + { code: 'ELEMENT_NOT_IN_VIEWPORT' }, + ); + } + return { + x: Math.round((x0 + x1) / 2), + y: Math.round((y0 + y1) / 2), + }; +} diff --git a/src/utils/hit-test.test.ts b/src/utils/hit-test.test.ts new file mode 100644 index 0000000..955242e --- /dev/null +++ b/src/utils/hit-test.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest'; + +import { hitsTarget, isKeyboardElement } from './hit-test.js'; +import type { UIElement } from '../backends/types.js'; + +const el = ( + type: string, + frame: UIElement['frame'], + extra: Partial = {}, +): UIElement => ({ type, frame, enabled: true, ...extra }); + +const target = el( + 'Button', + { x: 16, y: 532, width: 370, height: 48 }, + { identifier: 'log-in-button', label: 'Unlock' }, +); + +describe('hitsTarget', () => { + it('accepts the target itself', () => { + expect(hitsTarget(target, { ...target })).toBe(true); + }); + + it('accepts an element with the same identifier, whatever its frame', () => { + expect( + hitsTarget( + target, + el( + 'Other', + { x: 0, y: 0, width: 1, height: 1 }, + { identifier: 'log-in-button' }, + ), + ), + ).toBe(true); + }); + + it('accepts a child inside the target frame (1 pt tolerance)', () => { + expect( + hitsTarget( + target, + el('StaticText', { x: 15.5, y: 540, width: 100, height: 20 }), + ), + ).toBe(true); + }); + + it('rejects a larger element around the target (a sheet or container)', () => { + expect( + hitsTarget( + target, + el('Other', { x: 0, y: 400, width: 402, height: 474 }), + ), + ).toBe(false); + }); + + it('rejects an overlapping sibling that sticks out of the target', () => { + expect( + hitsTarget( + target, + el( + 'Button', + { x: 30, y: 560, width: 342, height: 44 }, + { label: 'Close' }, + ), + ), + ).toBe(false); + }); + + it('rejects keyboard elements even inside the target frame', () => { + expect( + hitsTarget( + target, + el( + 'Button', + { x: 30, y: 546, width: 342, height: 44 }, + { label: 'Passwords' }, + ), + ), + ).toBe(false); + expect( + hitsTarget(target, el('Key', { x: 100, y: 540, width: 30, height: 30 })), + ).toBe(false); + }); +}); + +describe('isKeyboardElement', () => { + it.each([ + [el('Keyboard', { x: 0, y: 590, width: 402, height: 226 }), true], + [el('Key', { x: 0, y: 0, width: 1, height: 1 }, { label: 'g' }), true], + [ + el('Button', { x: 0, y: 0, width: 1, height: 1 }, { label: 'Passwords' }), + true, + ], + [ + el('Button', { x: 0, y: 0, width: 1, height: 1 }, { label: 'Unlock' }), + false, + ], + [ + el('KeyboardControllerView', { x: 0, y: 0, width: 402, height: 874 }), + false, + ], + ])('%#: %j -> %s', (element, expected) => { + expect(isKeyboardElement(element)).toBe(expected); + }); +}); diff --git a/src/utils/hit-test.ts b/src/utils/hit-test.ts new file mode 100644 index 0000000..cff414d --- /dev/null +++ b/src/utils/hit-test.ts @@ -0,0 +1,92 @@ +import { describeElement } from './element.js'; +import type { UIElement } from '../backends/types.js'; + +/** Element types of the iOS software keyboard and its accessory bar. */ +const KEYBOARD_TYPES = new Set(['Keyboard', 'Key']); + +/** Labels of the keyboard's input-assistant bar (AutoFill / Passwords). */ +const KEYBOARD_ASSISTANT_LABELS = new Set(['Passwords']); + +/** + * A tap would land on another element than the target: an overlay, a sheet, + * or the software keyboard. Nothing was tapped. + */ +export class TargetObscuredError extends Error { + readonly code = 'TARGET_OBSCURED' as const; + + /** The element at the tap point. */ + readonly obscuredBy: UIElement; + + /** True when the obscuring element belongs to the software keyboard. */ + readonly keyboard: boolean; + + readonly x: number; + + readonly y: number; + + constructor( + target: UIElement, + obscuredBy: UIElement, + point: { x: number; y: number }, + keyboard: boolean, + ) { + const advice = keyboard + ? 'Dismiss the keyboard (device_dismiss_keyboard) and retry.' + : 'Close the overlay, or scroll the target into a free area.'; + super( + `TARGET_OBSCURED: ${describeElement(target)} is covered at ` + + `(${point.x}, ${point.y}) by ${keyboard ? 'the keyboard: ' : ''}` + + `${describeElement(obscuredBy)}. Nothing was tapped. ${advice}`, + ); + this.name = 'TargetObscuredError'; + this.obscuredBy = obscuredBy; + this.keyboard = keyboard; + this.x = point.x; + this.y = point.y; + } +} + +/** + * Whether `hit` (the element at the tap point) is the target or part of it. + * + * The hit-test returns the deepest accessible element, which can be a child + * of the target (a label inside a button), so "part of" means: same + * identifier, or a frame inside the target's frame (1 pt tolerance) that is + * not the keyboard. A larger element than the target is never accepted. + * + * @param target - The element the caller wants to tap. + * @param hit - The element at the tap point. + * @returns True when the tap reaches the target. + */ +export function hitsTarget(target: UIElement, hit: UIElement): boolean { + if (isKeyboardElement(hit)) { + return false; + } + if (target.identifier && hit.identifier === target.identifier) { + return true; + } + const tol = 1; + const t = target.frame; + const h = hit.frame; + return ( + h.x >= t.x - tol && + h.y >= t.y - tol && + h.x + h.width <= t.x + t.width + tol && + h.y + h.height <= t.y + t.height + tol + ); +} + +/** + * Whether the element is part of the software keyboard: a key, the keyboard + * itself, or its AutoFill bar ("Passwords"). + * + * @param element - The element. + * @returns True for keyboard elements. + */ +export function isKeyboardElement(element: UIElement): boolean { + return ( + KEYBOARD_TYPES.has(element.type) || + (element.type === 'Button' && + KEYBOARD_ASSISTANT_LABELS.has(element.label ?? '')) + ); +}