From 0e65fe34ff997e6718f67ff25ba21a305b00857f Mon Sep 17 00:00:00 2001 From: cryptotavares Date: Thu, 29 Oct 2026 19:20:54 +0000 Subject: [PATCH] perf: one snapshot per tap; cache idb describe per connection - `tapElement(query, { element })`: tap an element the caller already resolved (core resolves it with `waitForElement`), so a click takes one snapshot instead of two. The viewport check still runs on iOS. - iOS: cache `idb describe --json` (OS version for the describe-all API order, window size) per companion connection. It was read before every snapshot, tap and swipe (~0.25 s each). Cleared on (re)connect; a failed read is not cached; `getDeviceInfo()` still reads fresh. --- CHANGELOG.md | 8 ++ src/backends/adb-backend.test.ts | 47 ++++++++ src/backends/adb-backend.ts | 22 ++-- src/backends/appium-backend.test.ts | 35 ++++++ src/backends/appium-backend.ts | 22 ++-- src/backends/idb-backend.test.ts | 181 +++++++++++++++++++++++++++- src/backends/idb-backend.ts | 122 ++++++++++++++----- src/backends/index.ts | 18 +-- src/backends/types.ts | 18 ++- src/index.ts | 1 + 10 files changed, 416 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15f8256..70048f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `tapElement(query, { element })` taps an element the caller already resolved (for example from `waitForElement`) without taking a second snapshot + +### Changed + +- iOS (idb): read `idb describe --json` once per companion connection for the OS version and window size, instead of before every snapshot, tap and swipe. `getDeviceInfo()` still reads it fresh + ## [0.7.0] ### Added diff --git a/src/backends/adb-backend.test.ts b/src/backends/adb-backend.test.ts index dd5ea7c..f47a51f 100644 --- a/src/backends/adb-backend.test.ts +++ b/src/backends/adb-backend.test.ts @@ -621,6 +621,53 @@ describe('AdbBackend.getElementText', () => { }); }); +describe('AdbBackend.tapElement', () => { + let backend: AdbBackend; + const element: UIElement = { + type: 'android.widget.Button', + identifier: 'submit', + frame: { x: 100, y: 200, width: 400, height: 60 }, + enabled: true, + }; + + beforeEach(() => { + vi.clearAllMocks(); + mockExecStrict.mockResolvedValue(''); + backend = new AdbBackend('emulator-5554'); + }); + + it('taps a caller-resolved element without a new snapshot', async () => { + const result = await backend.tapElement( + { identifier: 'submit' }, + { element }, + ); + + expect(result).toMatchObject({ success: true, x: 300, y: 230 }); + expect(mockExecStrict).toHaveBeenCalledTimes(1); + expect(mockExecStrict).toHaveBeenCalledWith('adb', [ + '-s', + 'emulator-5554', + 'shell', + 'input', + 'tap', + '300', + '230', + ]); + }); + + it('snapshots to find the element when none is passed', async () => { + mockExecStrict.mockImplementation(async (_cmd, args) => + args?.includes('cat') || args?.includes('dump') + ? SAMPLE_UIAUTOMATOR_XML + : '', + ); + + await expect( + backend.tapElement({ identifier: 'not-on-screen' }), + ).rejects.toThrow('Element not found'); + }); +}); + describe('AdbBackend.screenshot', () => { let backend: AdbBackend; diff --git a/src/backends/adb-backend.ts b/src/backends/adb-backend.ts index 3c8f53a..dc176a6 100644 --- a/src/backends/adb-backend.ts +++ b/src/backends/adb-backend.ts @@ -34,21 +34,22 @@ import { reassembleInstrumentationXml, } from './android-instrumentation/snapshot.js'; import type { + AppStateResult, DeviceBackend, DeviceButton, DeviceInfo, - SnapshotResult, - ScreenshotResult, + ElementQuery, + LogsResult, ScreenshotFileResult, ScreenshotOptions, - LogsResult, + ScreenshotResult, + SnapshotResult, + TapElementOptions, TapResult, - AppStateResult, - ElementQuery, UIElement, - WindowSize, WebViewCdpInput, WebViewCdpOutcome, + WindowSize, } from './types.js'; import { runWebViewCdp } from './webview-cdp.js'; import { ACCEPT_LABELS, DENY_LABELS } from '../utils/alert-labels.js'; @@ -296,9 +297,12 @@ export class AdbBackend implements DeviceBackend { }; } - async tapElement(query: ElementQuery): Promise { - const snapshot = await this.snapshot(); - const element = findElement(snapshot.hierarchy, query); + async tapElement( + query: ElementQuery, + options?: TapElementOptions, + ): Promise { + const element = + options?.element ?? findElement((await this.snapshot()).hierarchy, query); if (!element) { throw new Error( `Element not found: ${JSON.stringify(query)}\n` + diff --git a/src/backends/appium-backend.test.ts b/src/backends/appium-backend.test.ts index 41a384d..abe2b13 100644 --- a/src/backends/appium-backend.test.ts +++ b/src/backends/appium-backend.test.ts @@ -13,6 +13,8 @@ const mockGetStatus = vi.fn(); const mockExecute = vi.fn(); const mockGetPageSource = vi.fn(); const mockUpdateSettings = vi.fn(); +const mockPerformActions = vi.fn(); +const mockReleaseActions = vi.fn(); vi.mock('./webdriver-client.js', () => ({ WebDriverClient: vi.fn().mockImplementation(function mockWebDriverClient() { @@ -21,6 +23,8 @@ vi.mock('./webdriver-client.js', () => ({ execute: mockExecute, getPageSource: mockGetPageSource, updateSettings: mockUpdateSettings, + performActions: mockPerformActions, + releaseActions: mockReleaseActions, }; }), createSession: vi.fn(), @@ -233,6 +237,37 @@ describe('AppiumBackend.screenshot', () => { }); }); +describe('AppiumBackend.tapElement', () => { + it('taps a caller-resolved element without reading the page source', async () => { + vi.clearAllMocks(); + mockGetStatus.mockResolvedValue({}); + mockPerformActions.mockResolvedValue(undefined); + const backend = new AppiumBackend({ + mode: 'attach', + appiumUrl: 'http://localhost:4723', + sessionId: 'session-1', + platform: 'ios', + }); + await backend.ensureConnected(); + + const result = await backend.tapElement( + { identifier: 'submit-btn' }, + { + element: { + type: 'Button', + identifier: 'submit-btn', + frame: { x: 20, y: 700, width: 350, height: 44 }, + enabled: true, + }, + }, + ); + + expect(result).toMatchObject({ success: true, x: 195, y: 722 }); + expect(mockGetPageSource).not.toHaveBeenCalled(); + expect(mockPerformActions).toHaveBeenCalledOnce(); + }); +}); + describe('AppiumBackend.snapshot (Android idle bypass)', () => { const config: AttachSessionConfig = { mode: 'attach', diff --git a/src/backends/appium-backend.ts b/src/backends/appium-backend.ts index b93f464..eec2700 100644 --- a/src/backends/appium-backend.ts +++ b/src/backends/appium-backend.ts @@ -1,18 +1,19 @@ import type { SessionConfig } from './session-file.js'; import type { + AppStateResult, DeviceBackend, DeviceButton, DeviceInfo, - SnapshotResult, - ScreenshotResult, + ElementQuery, + LogsResult, + Platform, ScreenshotFileResult, ScreenshotOptions, - LogsResult, + ScreenshotResult, + SnapshotResult, + TapElementOptions, TapResult, - AppStateResult, - ElementQuery, UIElement, - Platform, WindowSize, } from './types.js'; import { WebDriverClient, createSession } from './webdriver-client.js'; @@ -96,9 +97,12 @@ export class AppiumBackend implements DeviceBackend { }; } - async tapElement(query: ElementQuery): Promise { - const snapshot = await this.snapshot(); - const element = findElement(snapshot.hierarchy, query); + async tapElement( + query: ElementQuery, + options?: TapElementOptions, + ): Promise { + const element = + options?.element ?? findElement((await this.snapshot()).hierarchy, query); if (!element) { throw new Error( `Element not found: ${JSON.stringify(query)}\n` + diff --git a/src/backends/idb-backend.test.ts b/src/backends/idb-backend.test.ts index 3c08c5a..a1f6d7d 100644 --- a/src/backends/idb-backend.test.ts +++ b/src/backends/idb-backend.test.ts @@ -3,7 +3,7 @@ import { EventEmitter } from 'node:events'; import { readFileSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { parseIdbHierarchy, mapIdbElement, IdbBackend } from './idb-backend.js'; import { IdbUnavailableError } from './idb-errors.js'; @@ -824,6 +824,185 @@ describe('IdbBackend.tapElement and longPress viewport handling', () => { backend.longPress({ identifier: 'hidden-btn' }), ).rejects.toThrow('outside the 402x874 viewport'); }); + + it('taps a caller-resolved element without a new snapshot', async () => { + mockScreen([]); + + const result = await backend.tapElement( + { identifier: 'submit-btn' }, + { + element: { + type: 'Button', + identifier: 'submit-btn', + frame: { x: 10, y: 20, width: 100, height: 44 }, + enabled: true, + }, + }, + ); + + expect(result).toMatchObject({ success: true, x: 60, y: 42 }); + expect(mockExecStrict).not.toHaveBeenCalledWith( + '/usr/local/bin/idb', + expect.arrayContaining(['describe-all']), + connectionOptions, + ); + }); + + it('keeps the viewport check for a caller-resolved element', async () => { + mockScreen([]); + + await expect( + backend.tapElement( + { identifier: 'hidden-btn' }, + { + element: { + type: 'Button', + identifier: 'hidden-btn', + frame: { x: 0, y: 1000, width: 402, height: 100 }, + enabled: true, + }, + }, + ), + ).rejects.toThrow('outside the 402x874 viewport'); + }); +}); + +describe('IdbBackend device description cache', () => { + const udid = 'AAAA1111-BBBB-CCCC-DDDD-EEEE2222FFFF'; + let backend: IdbBackend; + + const screenInfo = { + name: 'iPhone 17 Pro', + os_version: '26.5', + state: 'Booted', + screen_dimensions: { width_points: 402, height_points: 874 }, + }; + const button = { + type: 'Button', + AXLabel: 'Submit', + AXUniqueId: 'submit-btn', + frame: { x: 10, y: 20, width: 100, height: 44 }, + enabled: true, + }; + + const describeJsonCalls = (): number => + mockExecStrict.mock.calls.filter( + ([, args]) => args?.includes('describe') && args?.includes('--json'), + ).length; + const describeAllCalls = (): number => + mockExecStrict.mock.calls.filter(([, args]) => + args?.includes('describe-all'), + ).length; + + beforeEach(() => { + vi.clearAllMocks(); + mockSupportsApi.mockResolvedValue(true); + mockChooseAxApi.mockReturnValue('axbridge'); + mockExec.mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' }); + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.includes('describe') && args?.includes('--json')) { + return JSON.stringify(screenInfo); + } + if (args?.includes('describe-all')) { + return JSON.stringify([button]); + } + return ''; + }); + backend = new IdbBackend(udid); + }); + + afterEach(() => { + mockSupportsApi.mockResolvedValue(false); + mockChooseAxApi.mockReturnValue(null); + }); + + it('reads idb describe once per connection for snapshots and taps', async () => { + const element = await backend.waitForElement({ identifier: 'submit-btn' }); + await backend.tapElement({ identifier: 'submit-btn' }, { element }); + await backend.snapshot(); + await backend.tapElement({ identifier: 'submit-btn' }); + await backend.swipe('up'); + + expect(describeJsonCalls()).toBe(1); + // waitForElement, snapshot, and the second (unresolved) tap. + expect(describeAllCalls()).toBe(3); + }); + + it('shares one describe read between concurrent first calls', async () => { + await Promise.all([backend.snapshot(), backend.getWindowSize()]); + + expect(describeJsonCalls()).toBe(1); + }); + + it('reads the description again after a reconnect', async () => { + await backend.snapshot(); + await backend.reconnect(); + await backend.snapshot(); + + expect(describeJsonCalls()).toBe(2); + }); + + it('does not cache a failed read', async () => { + let failures = 1; + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.includes('describe') && args?.includes('--json')) { + if (failures > 0) { + failures -= 1; + throw new Error('companion busy'); + } + return JSON.stringify(screenInfo); + } + return JSON.stringify([button]); + }); + + await expect(backend.getWindowSize()).rejects.toThrow('companion busy'); + expect(await backend.getWindowSize()).toStrictEqual({ + width: 402, + height: 874, + }); + expect(describeJsonCalls()).toBe(2); + }); + + it('getDeviceInfo reads fresh and refreshes the cache', async () => { + await backend.getWindowSize(); + const info = await backend.getDeviceInfo(); + await backend.getWindowSize(); + + expect(info).toStrictEqual({ + platform: 'ios', + deviceId: udid, + name: 'iPhone 17 Pro', + osVersion: '26.5', + state: 'Booted', + }); + expect(describeJsonCalls()).toBe(2); + }); + + it('falls back to the snapshot root when describe is not JSON', async () => { + // Without --api support the snapshot does not need the OS version. + mockSupportsApi.mockResolvedValue(false); + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.includes('describe') && args?.includes('--json')) { + return 'not json'; + } + return JSON.stringify([ + { type: 'Application', frame: { x: 0, y: 0, width: 390, height: 844 } }, + ]); + }); + + expect(await backend.getWindowSize()).toStrictEqual({ + width: 390, + height: 844, + }); + }); + + it('getDeviceInfo rejects a non-JSON description', async () => { + mockExecStrict.mockResolvedValue('not json'); + + await expect(backend.getDeviceInfo()).rejects.toThrow( + `idb describe returned invalid JSON for device ${udid}: not json`, + ); + }); }); describe('IdbBackend.ensureConnected', () => { diff --git a/src/backends/idb-backend.ts b/src/backends/idb-backend.ts index 3cbb9e6..46c1ce0 100644 --- a/src/backends/idb-backend.ts +++ b/src/backends/idb-backend.ts @@ -5,17 +5,18 @@ import { readFile } from 'node:fs/promises'; import { IdbConnection } from './idb-connection.js'; import { classifyIdbFailure, IdbUnavailableError } from './idb-errors.js'; import type { + AppStateResult, DeviceBackend, DeviceButton, DeviceInfo, - SnapshotResult, - ScreenshotResult, + ElementQuery, + LogsResult, ScreenshotFileResult, ScreenshotOptions, - LogsResult, + ScreenshotResult, + SnapshotResult, + TapElementOptions, TapResult, - AppStateResult, - ElementQuery, UIElement, WindowSize, } from './types.js'; @@ -39,6 +40,16 @@ import { } from '../utils/platform.js'; import { stopChildProcess } from '../utils/stop-child-process.js'; +/** `idb describe --json` printed something that is not JSON. */ +class InvalidDescribeJsonError extends Error { + constructor(udid: string, raw: string) { + super( + `idb describe returned invalid JSON for device ${udid}: ${raw.slice(0, 200)}`, + ); + this.name = 'InvalidDescribeJsonError'; + } +} + export class IdbBackend implements DeviceBackend { readonly kind = 'idb' as const; @@ -75,6 +86,15 @@ export class IdbBackend implements DeviceBackend { #stoppingRecording: Promise | undefined; + /** + * `idb describe --json` of the current connection: OS version (for the + * describe-all API order) and screen size. Read once per connection, because + * every read is an idb process (~0.25 s) and both values are fixed while the + * simulator stays booted. Cleared on every (re)connect. A rotation while + * connected is not detected; reconnect after rotating. + */ + #describeCache: Promise> | undefined; + constructor(udid: string) { this.#udid = udid; this.#connection = new IdbConnection(udid); @@ -143,6 +163,7 @@ export class IdbBackend implements DeviceBackend { } await this.#connection.connect(this.#idbPath); await this.#enableSimulatorAccessibility(); + this.#describeCache = undefined; this.#generation += 1; this.#connected = true; } catch (error) { @@ -322,22 +343,56 @@ export class IdbBackend implements DeviceBackend { } } + /** + * Runs `idb describe --json` and stores the result as the cached + * description of the current connection (see `#describeCache`). Concurrent + * callers of `#cachedDescribe()` share one read; a failed read is dropped + * from the cache so the next call retries. + * + * @returns The parsed device description. + */ + async #describe(): Promise> { + const pending = (async (): Promise> => { + const raw = await this.#execStrict(this.#idbPath, [ + 'describe', + '--udid', + this.#udid, + '--json', + ]); + try { + return JSON.parse(raw) as Record; + } catch { + throw new InvalidDescribeJsonError(this.#udid, raw); + } + })(); + this.#describeCache = pending; + pending.catch(() => { + if (this.#describeCache === pending) { + this.#describeCache = undefined; + } + }); + return pending; + } + + /** + * The cached `idb describe --json` of the current connection; reads it on + * first use. + * + * @returns The parsed device description. + */ + async #cachedDescribe(): Promise> { + return this.#describeCache ?? this.#describe(); + } + + /** + * Reads the device description fresh (and refreshes the cache that + * snapshots and taps use). + * + * @returns The device info. + */ async getDeviceInfo(): Promise { await this.ensureConnected(); - const raw = await this.#execStrict(this.#idbPath, [ - 'describe', - '--udid', - this.#udid, - '--json', - ]); - let info: Record; - try { - info = JSON.parse(raw) as Record; - } catch { - throw new Error( - `idb describe returned invalid JSON for device ${this.#udid}: ${raw.slice(0, 200)}`, - ); - } + const info = await this.#describe(); return { platform: 'ios', deviceId: this.#udid, @@ -403,16 +458,20 @@ export class IdbBackend implements DeviceBackend { if (!this.#idbSupportsApi) { return [null]; } - const { osVersion } = await this.getDeviceInfo(); + const info = await this.#cachedDescribe(); + const osVersion = (info.os_version as string) ?? 'Unknown'; const preferred = chooseAxApi(osVersion, true); const complement = preferred === 'axbridge' ? 'ax' : 'axbridge'; return [preferred, complement]; } - async tapElement(query: ElementQuery): Promise { + async tapElement( + query: ElementQuery, + options?: TapElementOptions, + ): Promise { await this.ensureConnected(); - const snapshot = await this.snapshot(); - const element = findElement(snapshot.hierarchy, query); + const element = + options?.element ?? findElement((await this.snapshot()).hierarchy, query); if (!element) { throw new Error( `Element not found: ${JSON.stringify(query)}\n` + @@ -856,15 +915,16 @@ export class IdbBackend implements DeviceBackend { async getWindowSize(): Promise { await this.ensureConnected(); - const raw = await this.#execStrict(this.#idbPath, [ - 'describe', - '--udid', - this.#udid, - '--json', - ]); + // A transport failure still throws, as before; only parse problems fall + // back to the snapshot. + const info = await this.#cachedDescribe().catch((error: unknown) => { + if (error instanceof InvalidDescribeJsonError) { + return undefined; + } + throw error; + }); try { - const info = JSON.parse(raw) as Record; - const dims = info.screen_dimensions as + const dims = info?.screen_dimensions as | { width?: number; height?: number; diff --git a/src/backends/index.ts b/src/backends/index.ts index bdeb6d5..8d4e98a 100644 --- a/src/backends/index.ts +++ b/src/backends/index.ts @@ -4,23 +4,24 @@ import { AppiumBackend } from './appium-backend.js'; import { IdbBackend } from './idb-backend.js'; import { readSessionFile } from './session-file.js'; import type { + AppStateResult, BackendKind, DeviceBackend, DeviceButton, DeviceInfo, + ElementQuery, LogsResult, - ScreenshotResult, + Platform, ScreenshotFileResult, ScreenshotOptions, + ScreenshotResult, SnapshotResult, + TapElementOptions, TapResult, - AppStateResult, - ElementQuery, UIElement, - Platform, - WindowSize, WebViewCdpInput, WebViewCdpOutcome, + WindowSize, } from './types.js'; import { detectPlatform, @@ -229,8 +230,11 @@ export function createLazyBackend( return (await resolve()).snapshot(); }, - async tapElement(query: ElementQuery): Promise { - return (await resolve()).tapElement(query); + async tapElement( + query: ElementQuery, + options?: TapElementOptions, + ): Promise { + return (await resolve()).tapElement(query, options); }, async tapCoordinates(x: number, y: number): Promise { diff --git a/src/backends/types.ts b/src/backends/types.ts index b27403d..4fb3452 100644 --- a/src/backends/types.ts +++ b/src/backends/types.ts @@ -46,6 +46,14 @@ export type TapResult = { targetDescription: string; }; +export type TapElementOptions = { + /** + * The element to tap, already resolved by the caller. Its frame is used as + * is; the backend does not check that it still matches `query`. + */ + element?: UIElement; +}; + export type AppStateResult = { bundleId: string; state: string; @@ -127,7 +135,15 @@ export type DeviceBackend = { snapshot(): Promise; - tapElement(query: ElementQuery): Promise; + /** + * Taps the element that matches `query`. Pass `options.element` (an element + * the caller already resolved, for example from `waitForElement`) to tap it + * without taking a new snapshot. + */ + tapElement( + query: ElementQuery, + options?: TapElementOptions, + ): Promise; tapCoordinates(x: number, y: number): Promise; diff --git a/src/index.ts b/src/index.ts index a2a4aba..b3f4a90 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,7 @@ export type { ScreenshotFileResult, ScreenshotOptions, LogsResult, + TapElementOptions, TapResult, AppStateResult, Platform,