diff --git a/README.md b/README.md index c3a5e9d..ec2c15a 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ The `.device-session` file is typically written by the test runner when it creat ### Element Identification -Elements are identified by accessibility attributes — not internal refs. Matching is **fuzzy**: partial text and case-insensitive matches work. For example, querying `{ label: "Confirm" }` matches an element with label `"Confirm Transaction"`. +Elements are identified by accessibility attributes — not internal refs. Matching is **fuzzy**: partial text and case-insensitive matches work. For example, querying `{ label: "Confirm" }` matches an element with label `"Confirm Transaction"`. An exact match anywhere on screen wins over a partial one: `{ identifier: "account-picker" }` matches `account-picker`, not an earlier `account-picker-avatar`. - **iOS**: accessibility label, accessibility identifier - **Android**: content-description, resource-id, text diff --git a/SKILL.md b/SKILL.md index c6e56d1..7d57d19 100644 --- a/SKILL.md +++ b/SKILL.md @@ -98,7 +98,7 @@ Find an element by query and tap its center. Specify at least one of: `label`, ` 3. `text` — visible text content (matches both `value` and `label`) 4. `type` — element class (least specific, combine with other fields) -**Matching is fuzzy:** partial text and case-insensitive matches work. `{ label: "Confirm" }` matches `"Confirm Transaction"`. +**Matching is fuzzy:** partial text and case-insensitive matches work. `{ label: "Confirm" }` matches `"Confirm Transaction"`. An exact match anywhere on screen wins over a partial one. **Examples:** diff --git a/src/backends/idb-backend.test.ts b/src/backends/idb-backend.test.ts index 3c08c5a..e7165e6 100644 --- a/src/backends/idb-backend.test.ts +++ b/src/backends/idb-backend.test.ts @@ -1177,6 +1177,117 @@ describe('IdbBackend.snapshot accessibility backend selection', () => { expect(hierarchy[0].children?.[0].children).toBeUndefined(); }); + describe('ax supplement for axbridge', () => { + const node = ( + id: string | null, + frame: { x: number; y: number; width: number; height: number }, + children: unknown[] = [], + label: string | null = null, + ): Record => ({ + type: id ? 'Button' : 'RCTViewComponentView', + AXUniqueId: id, + AXLabel: label, + frame, + children, + }); + const axbridgeTree = [ + node(null, { x: 0, y: 0, width: 402, height: 874 }, [ + node('wallet-screen', { x: 0, y: 0, width: 402, height: 874 }, [ + // The unlabelled container that hides the action row. + node(null, { x: 0, y: 134, width: 402, height: 439 }), + node('account-picker', { x: 60, y: 78, width: 163, height: 24 }), + ]), + ]), + ]; + const axTree = [ + node(null, { x: 0, y: 0, width: 402, height: 874 }, [ + node('account-picker', { x: 60, y: 78, width: 163, height: 24 }), + node( + 'wallet-send-button', + { x: 205, y: 242, width: 87, height: 73 }, + [], + 'Send', + ), + node('offscreen-card', { x: 966, y: 1075, width: 180, height: 180 }), + node(null, { x: 1, y: 1, width: 1, height: 1 }, [], 'no id'), + ]), + ]; + + const mockApis = (ax: () => Promise): void => { + mockSupportsApi.mockResolvedValue(true); + mockSupportsNested.mockResolvedValue(true); + mockChooseAxApi.mockReturnValue('axbridge'); + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.includes('describe') && args?.includes('--json')) { + return JSON.stringify({ os_version: '27.0', name: 'iPhone' }); + } + if (args?.includes('axbridge')) { + return JSON.stringify(axbridgeTree); + } + return ax(); + }); + }; + + it('adds identified elements that only ax reports, inside the smallest containing element', async () => { + mockApis(async () => JSON.stringify(axTree)); + + const { hierarchy, raw } = await backend.snapshot(); + + expect(raw).toBe(JSON.stringify(axbridgeTree)); + const send = findElement(hierarchy, { identifier: 'wallet-send-button' }); + expect(send).toMatchObject({ + label: 'Send', + frame: { x: 205, y: 242, width: 87, height: 73 }, + }); + const container = hierarchy[0].children?.[0].children?.[0]; + expect(container?.children?.map((el) => el.identifier)).toStrictEqual([ + 'wallet-send-button', + ]); + // Not contained by any axbridge frame: added at the top level. + expect(hierarchy.at(-1)?.identifier).toBe('offscreen-card'); + // account-picker is not duplicated; unidentified ax nodes are skipped. + const ids: (string | undefined)[] = []; + const walk = (els: typeof hierarchy): void => + els.forEach((el) => { + ids.push(el.identifier ?? el.label); + walk(el.children ?? []); + }); + walk(hierarchy); + expect(ids.filter((id) => id === 'account-picker')).toHaveLength(1); + expect(ids).not.toContain('no id'); + }); + + it('returns the axbridge tree unchanged when the ax read fails', async () => { + mockApis(async () => { + throw new Error('ax failed'); + }); + + const { hierarchy } = await backend.snapshot(); + + expect(hierarchy).toStrictEqual( + parseIdbHierarchy(JSON.stringify(axbridgeTree)), + ); + }); + + it('does not read a supplement when ax is the preferred backend', async () => { + mockSupportsApi.mockResolvedValue(true); + mockChooseAxApi.mockReturnValue('ax'); + mockExecStrict.mockImplementation(async (_cmd, args) => { + if (args?.includes('describe') && args?.includes('--json')) { + return JSON.stringify({ os_version: '17.4', name: 'iPhone' }); + } + return JSON.stringify(axTree); + }); + + await backend.snapshot(); + + const describeCalls = mockExecStrict.mock.calls.filter( + ([, args]) => args?.[1] === 'describe-all', + ); + expect(describeCalls).toHaveLength(1); + }); + }); + it('rethrows a non-Error rejection as a descriptive Error', async () => { mockSupportsApi.mockResolvedValue(false); mockExecStrict.mockRejectedValue('boom'); @@ -1287,7 +1398,10 @@ describe('IdbBackend companion recovery', () => { const apis = mockExecStrict.mock.calls .filter(([, args]) => args?.[1] === 'describe-all') .map(([, args]) => args?.[args.indexOf('--api') + 1]); - expect(apis).toStrictEqual(['axbridge', 'axbridge']); + // axbridge is read and retried once; the one ax read is the concurrent + // best-effort supplement, not a fallback. + expect(apis.filter((api) => api === 'axbridge')).toHaveLength(2); + expect(apis.filter((api) => api === 'ax')).toHaveLength(1); }); it('does not resend a mutating command when delivery is uncertain', async () => { diff --git a/src/backends/idb-backend.ts b/src/backends/idb-backend.ts index 3cbb9e6..b37c6d8 100644 --- a/src/backends/idb-backend.ts +++ b/src/backends/idb-backend.ts @@ -347,10 +347,31 @@ export class IdbBackend implements DeviceBackend { }; } + /** + * Capture the UI hierarchy. When `axbridge` is read (iOS 18+), `ax` is read + * at the same time and its elements whose identifier is missing from the + * `axbridge` tree are added to it (see `addMissingIdentifiedElements`). + * `axbridge` can report a React Native container as one unlabelled view + * without children: on the MetaMask wallet home, the balance and the + * Buy / Swap / Send / Receive buttons, so `wallet-send-button` was not found. + * `raw` stays the `axbridge` output. + * + * @returns The snapshot. + */ async snapshot(): Promise { await this.ensureConnected(); - const raw = await this.#describeAll(); - const hierarchy = parseIdbHierarchy(raw); + const apis = await this.#describeAllApiOrder(); + const supplement = + apis[0] === 'axbridge' ? this.#describeSupplement('ax') : undefined; + const { raw, api } = await this.#describeAll(apis); + let hierarchy = parseIdbHierarchy(raw); + const extra = await supplement; + if (api === 'axbridge' && extra !== undefined) { + hierarchy = addMissingIdentifiedElements( + hierarchy, + parseIdbHierarchy(extra), + ); + } return { platform: 'ios', hierarchy, @@ -359,27 +380,65 @@ export class IdbBackend implements DeviceBackend { }; } + /** + * Arguments of `idb ui describe-all` for one accessibility backend. + * + * @param api - The `--api` value, or null to omit it. + * @returns The idb arguments. + */ + #describeAllArgs(api: 'ax' | 'axbridge' | null): string[] { + const args = ['ui', 'describe-all', '--udid', this.#udid]; + if (api !== null) { + args.push('--api', api); + } + if (this.#idbSupportsNested) { + args.push('--nested'); + } + return args; + } + + /** + * Read a supplementary hierarchy. Best effort: no companion recovery (the + * main read does it), and a failure returns undefined. + * + * @param api - The `--api` value. + * @returns The raw JSON hierarchy, or undefined when the read failed. + */ + async #describeSupplement( + api: 'ax' | 'axbridge', + ): Promise { + try { + return await execStrict(this.#idbPath, this.#describeAllArgs(api), { + env: this.#connection.env, + }); + } catch { + return undefined; + } + } + /** * Run `idb ui describe-all`, trying the version-appropriate accessibility * backend first and falling back to the others if it errors. This keeps * snapshots working across iOS 17-26 regardless of the user's idb client. * - * @returns The raw JSON hierarchy from the first backend that succeeds. + * @param apis - The `--api` values to try, in order. + * @returns The raw JSON hierarchy from the first backend that succeeds, and + * that backend. * @throws When every attempted backend fails. */ - async #describeAll(): Promise { - const apis = await this.#describeAllApiOrder(); + async #describeAll( + apis: ('ax' | 'axbridge' | null)[], + ): Promise<{ raw: string; api: 'ax' | 'axbridge' | null }> { let lastError: unknown; for (const api of apis) { - const args = ['ui', 'describe-all', '--udid', this.#udid]; - if (api !== null) { - args.push('--api', api); - } - if (this.#idbSupportsNested) { - args.push('--nested'); - } try { - return await this.#execStrict(this.#idbPath, args); + return { + raw: await this.#execStrict( + this.#idbPath, + this.#describeAllArgs(api), + ), + api, + }; } catch (error) { if (error instanceof IdbUnavailableError) { // Already reconnected and retried; another --api will not help. @@ -1107,6 +1166,80 @@ export function parseIdbHierarchy(raw: string): UIElement[] { } } +/** + * Adds to `primary` the elements of `secondary` whose identifier is not in + * `primary`. Each one is added, without its children, as the last child of + * the smallest `primary` element whose frame contains its frame, or at the + * top level when none does. Elements with no identifier are not added. The + * inputs are not modified. + * + * @param primary - The main hierarchy (idb `--api axbridge`). + * @param secondary - The other hierarchy (idb `--api ax`). + * @returns `primary` with the missing elements added. + */ +export function addMissingIdentifiedElements( + primary: UIElement[], + secondary: UIElement[], +): UIElement[] { + const result = structuredClone(primary); + const all: UIElement[] = []; + const collect = (elements: UIElement[]): void => { + for (const el of elements) { + all.push(el); + if (el.children) { + collect(el.children); + } + } + }; + collect(result); + const known = new Set(all.map((el) => el.identifier).filter(Boolean)); + + const missing: UIElement[] = []; + const collectMissing = (elements: UIElement[]): void => { + for (const el of elements) { + if (el.identifier && !known.has(el.identifier)) { + missing.push(el); + } + if (el.children) { + collectMissing(el.children); + } + } + }; + collectMissing(secondary); + + for (const el of missing) { + const { children: _children, ...element } = el; + const added: UIElement = structuredClone(element); + const host = all + .filter((candidate) => containsFrame(candidate.frame, el.frame)) + .reduce< + UIElement | undefined + >((best, candidate) => (best === undefined || area(candidate.frame) < area(best.frame) ? candidate : best), undefined); + if (host) { + host.children = [...(host.children ?? []), added]; + } else { + result.push(added); + } + } + return result; +} + +type Frame = UIElement['frame']; + +function area(frame: Frame): number { + return frame.width * frame.height; +} + +function containsFrame(outer: Frame, inner: Frame): boolean { + return ( + area(outer) > 0 && + inner.x >= outer.x && + inner.y >= outer.y && + inner.x + inner.width <= outer.x + outer.width && + inner.y + inner.height <= outer.y + outer.height + ); +} + /** * Normalize idb fields, treating empty child arrays like flat-format leaves. * diff --git a/src/utils/element.test.ts b/src/utils/element.test.ts index 3abd1e6..4512699 100644 --- a/src/utils/element.test.ts +++ b/src/utils/element.test.ts @@ -65,6 +65,25 @@ describe('matchesQuery', () => { expect(matchesQuery(el, { identifier: 'submit-btn' })).toBe(true); }); + it('matches only equal strings when exact', () => { + const el = makeElement({ + identifier: 'account-picker-avatar', + label: 'Agent Account', + value: 'v', + }); + expect(matchesQuery(el, { identifier: 'account-picker' }, true)).toBe( + false, + ); + expect( + matchesQuery(el, { identifier: 'account-picker-avatar' }, true), + ).toBe(true); + expect(matchesQuery(el, { label: 'agent account' }, true)).toBe(false); + expect(matchesQuery(el, { text: 'Agent Account' }, true)).toBe(true); + expect(matchesQuery(el, { text: 'v' }, true)).toBe(true); + expect(matchesQuery(el, { type: 'Butt' }, true)).toBe(false); + expect(matchesQuery(el, {}, true)).toBe(false); + }); + it('does not match when actual is undefined', () => { const el = makeElement({}); expect(matchesQuery(el, { label: 'anything' })).toBe(false); @@ -92,6 +111,52 @@ describe('findElement', () => { expect(found?.identifier).toBe('child'); }); + it('prefers an exact identifier match over an earlier partial match', () => { + const elements = [ + makeElement({ + identifier: 'profile-button', + children: [makeElement({ identifier: 'account-picker-avatar' })], + }), + makeElement({ identifier: 'account-picker' }), + ]; + expect( + findElement(elements, { identifier: 'account-picker' })?.identifier, + ).toBe('account-picker'); + }); + + it('prefers an exact label or text match over an earlier partial match', () => { + const elements = [ + makeElement({ label: 'Send feedback' }), + makeElement({ label: 'Send', value: 'Send' }), + ]; + expect(findElement(elements, { label: 'Send' })?.value).toBe('Send'); + expect(findElement(elements, { text: 'Send' })?.value).toBe('Send'); + }); + + it('falls back to the first partial match when nothing matches exactly', () => { + const elements = [ + makeElement({ identifier: 'io.metamask:id/submit-btn' }), + makeElement({ identifier: 'submit-btn-2' }), + ]; + expect( + findElement(elements, { identifier: 'submit-btn' })?.identifier, + ).toBe('io.metamask:id/submit-btn'); + expect( + findElement(elements, { identifier: 'SUBMIT-BTN-2' })?.identifier, + ).toBe('submit-btn-2'); + }); + + it('requires every field to match exactly for the exact pass', () => { + const elements = [ + makeElement({ identifier: 'btn-long', label: 'Go' }), + makeElement({ identifier: 'btn', label: 'Go now' }), + ]; + // No element matches both fields exactly, so the first fuzzy match wins. + expect( + findElement(elements, { identifier: 'btn', label: 'Go' })?.identifier, + ).toBe('btn-long'); + }); + it('returns null when not found', () => { expect(findElement([], { identifier: 'x' })).toBeNull(); expect( diff --git a/src/utils/element.ts b/src/utils/element.ts index 8c5c571..276c913 100644 --- a/src/utils/element.ts +++ b/src/utils/element.ts @@ -1,15 +1,40 @@ import type { UIElement, ElementQuery } from '../backends/types.js'; +/** + * Finds the element that matches the query. An exact match (see + * `matchesQuery`) anywhere in the tree wins; only when there is none is the + * first fuzzy match (case-insensitive, contains) returned, in tree order. + * Without this, the query `account-picker` returned an earlier + * `account-picker-avatar`. + * + * @param elements - The hierarchy to search. + * @param query - The query. + * @returns The matching element, or null. + */ export function findElement( elements: UIElement[], query: ElementQuery, +): UIElement | null { + return findFirst(elements, query, true) ?? findFirst(elements, query, false); +} + +/** + * @param elements - The hierarchy to search. + * @param query - The query. + * @param exact - Whether every query field must equal the element's. + * @returns The first matching element in tree order, or null. + */ +function findFirst( + elements: UIElement[], + query: ElementQuery, + exact: boolean, ): UIElement | null { for (const el of elements) { - if (matchesQuery(el, query)) { + if (matchesQuery(el, query, exact)) { return el; } if (el.children) { - const found = findElement(el.children, query); + const found = findFirst(el.children, query, exact); if (found) { return found; } @@ -18,21 +43,33 @@ export function findElement( return null; } -export function matchesQuery(element: UIElement, query: ElementQuery): boolean { - if (query.identifier && !fuzzyMatch(element.identifier, query.identifier)) { +/** + * @param element - The element. + * @param query - The query. Every given field must match. + * @param exact - Require equal strings instead of a fuzzy match + * (case-insensitive, contains). Default false. + * @returns True when the element matches. + */ +export function matchesQuery( + element: UIElement, + query: ElementQuery, + exact = false, +): boolean { + const match = exact ? exactMatch : fuzzyMatch; + if (query.identifier && !match(element.identifier, query.identifier)) { return false; } - if (query.label && !fuzzyMatch(element.label, query.label)) { + if (query.label && !match(element.label, query.label)) { return false; } if ( query.text && - !fuzzyMatch(element.value, query.text) && - !fuzzyMatch(element.label, query.text) + !match(element.value, query.text) && + !match(element.label, query.text) ) { return false; } - if (query.type && !fuzzyMatch(element.type, query.type)) { + if (query.type && !match(element.type, query.type)) { return false; } @@ -41,6 +78,10 @@ export function matchesQuery(element: UIElement, query: ElementQuery): boolean { return Boolean(hasAtLeastOneFilter); } +function exactMatch(actual: string | undefined, expected: string): boolean { + return actual === expected; +} + function fuzzyMatch(actual: string | undefined, expected: string): boolean { if (!actual) { return false;