fix: exact element match first; add elements idb axbridge drops on iOS 18+

README.md

@@ -205,7 +205,7 @@ The `.device-session` file is typically written by the test runner when it creat
 
 ### Element Identification
 
-Elements are identified by accessibility attributes — not internal refs. Matching is **fuzzy**: partial text and case-insensitive matches work. For example, querying `{ label: "Confirm" }` matches an element with label `"Confirm Transaction"`.
+Elements are identified by accessibility attributes — not internal refs. Matching is **fuzzy**: partial text and case-insensitive matches work. For example, querying `{ label: "Confirm" }` matches an element with label `"Confirm Transaction"`. An exact match anywhere on screen wins over a partial one: `{ identifier: "account-picker" }` matches `account-picker`, not an earlier `account-picker-avatar`.
 
 - **iOS**: accessibility label, accessibility identifier
 - **Android**: content-description, resource-id, text

SKILL.md

@@ -98,7 +98,7 @@ Find an element by query and tap its center. Specify at least one of: `label`, `
 3. `text` — visible text content (matches both `value` and `label`)
 4. `type` — element class (least specific, combine with other fields)
 
-**Matching is fuzzy:** partial text and case-insensitive matches work. `{ label: "Confirm" }` matches `"Confirm Transaction"`.
+**Matching is fuzzy:** partial text and case-insensitive matches work. `{ label: "Confirm" }` matches `"Confirm Transaction"`. An exact match anywhere on screen wins over a partial one.
 
 **Examples:**
 

src/backends/idb-backend.test.ts

@@ -1177,6 +1177,117 @@ describe('IdbBackend.snapshot accessibility backend selection', () => {
     expect(hierarchy[0].children?.[0].children).toBeUndefined();
   });
 
+  describe('ax supplement for axbridge', () => {
+    const node = (
+      id: string | null,
+      frame: { x: number; y: number; width: number; height: number },
+      children: unknown[] = [],
+      label: string | null = null,
+    ): Record<string, unknown> => ({
+      type: id ? 'Button' : 'RCTViewComponentView',
+      AXUniqueId: id,
+      AXLabel: label,
+      frame,
+      children,
+    });
+    const axbridgeTree = [
+      node(null, { x: 0, y: 0, width: 402, height: 874 }, [
+        node('wallet-screen', { x: 0, y: 0, width: 402, height: 874 }, [
+          // The unlabelled container that hides the action row.
+          node(null, { x: 0, y: 134, width: 402, height: 439 }),
+          node('account-picker', { x: 60, y: 78, width: 163, height: 24 }),
+        ]),
+      ]),
+    ];
+    const axTree = [
+      node(null, { x: 0, y: 0, width: 402, height: 874 }, [
+        node('account-picker', { x: 60, y: 78, width: 163, height: 24 }),
+        node(
+          'wallet-send-button',
+          { x: 205, y: 242, width: 87, height: 73 },
+          [],
+          'Send',
+        ),
+        node('offscreen-card', { x: 966, y: 1075, width: 180, height: 180 }),
+        node(null, { x: 1, y: 1, width: 1, height: 1 }, [], 'no id'),
+      ]),
+    ];
+
+    const mockApis = (ax: () => Promise<string>): void => {
+      mockSupportsApi.mockResolvedValue(true);
+      mockSupportsNested.mockResolvedValue(true);
+      mockChooseAxApi.mockReturnValue('axbridge');
+      mockExecStrict.mockImplementation(async (_cmd, args) => {
+        if (args?.includes('describe') && args?.includes('--json')) {
+          return JSON.stringify({ os_version: '27.0', name: 'iPhone' });
+        }
+        if (args?.includes('axbridge')) {
+          return JSON.stringify(axbridgeTree);
+        }
+        return ax();
+      });
+    };
+
+    it('adds identified elements that only ax reports, inside the smallest containing element', async () => {
+      mockApis(async () => JSON.stringify(axTree));
+
+      const { hierarchy, raw } = await backend.snapshot();
+
+      expect(raw).toBe(JSON.stringify(axbridgeTree));
+      const send = findElement(hierarchy, { identifier: 'wallet-send-button' });
+      expect(send).toMatchObject({
+        label: 'Send',
+        frame: { x: 205, y: 242, width: 87, height: 73 },
+      });
+      const container = hierarchy[0].children?.[0].children?.[0];
+      expect(container?.children?.map((el) => el.identifier)).toStrictEqual([
+        'wallet-send-button',
+      ]);
+      // Not contained by any axbridge frame: added at the top level.
+      expect(hierarchy.at(-1)?.identifier).toBe('offscreen-card');
+      // account-picker is not duplicated; unidentified ax nodes are skipped.
+      const ids: (string | undefined)[] = [];
+      const walk = (els: typeof hierarchy): void =>
+        els.forEach((el) => {
+          ids.push(el.identifier ?? el.label);
+          walk(el.children ?? []);
+        });
+      walk(hierarchy);
+      expect(ids.filter((id) => id === 'account-picker')).toHaveLength(1);
+      expect(ids).not.toContain('no id');
+    });
+
+    it('returns the axbridge tree unchanged when the ax read fails', async () => {
+      mockApis(async () => {
+        throw new Error('ax failed');
+      });
+
+      const { hierarchy } = await backend.snapshot();
+
+      expect(hierarchy).toStrictEqual(
+        parseIdbHierarchy(JSON.stringify(axbridgeTree)),
+      );
+    });
+
+    it('does not read a supplement when ax is the preferred backend', async () => {
+      mockSupportsApi.mockResolvedValue(true);
+      mockChooseAxApi.mockReturnValue('ax');
+      mockExecStrict.mockImplementation(async (_cmd, args) => {
+        if (args?.includes('describe') && args?.includes('--json')) {
+          return JSON.stringify({ os_version: '17.4', name: 'iPhone' });
+        }
+        return JSON.stringify(axTree);
+      });
+
+      await backend.snapshot();
+
+      const describeCalls = mockExecStrict.mock.calls.filter(
+        ([, args]) => args?.[1] === 'describe-all',
+      );
+      expect(describeCalls).toHaveLength(1);
+    });
+  });
+
   it('rethrows a non-Error rejection as a descriptive Error', async () => {
     mockSupportsApi.mockResolvedValue(false);
     mockExecStrict.mockRejectedValue('boom');
@@ -1287,7 +1398,10 @@ describe('IdbBackend companion recovery', () => {
     const apis = mockExecStrict.mock.calls
       .filter(([, args]) => args?.[1] === 'describe-all')
       .map(([, args]) => args?.[args.indexOf('--api') + 1]);
-    expect(apis).toStrictEqual(['axbridge', 'axbridge']);
+    // axbridge is read and retried once; the one ax read is the concurrent
+    // best-effort supplement, not a fallback.
+    expect(apis.filter((api) => api === 'axbridge')).toHaveLength(2);
+    expect(apis.filter((api) => api === 'ax')).toHaveLength(1);
   });
 
   it('does not resend a mutating command when delivery is uncertain', async () => {

src/backends/idb-backend.ts

@@ -347,10 +347,31 @@ export class IdbBackend implements DeviceBackend {
     };
   }
 
+  /**
+   * Capture the UI hierarchy. When `axbridge` is read (iOS 18+), `ax` is read
+   * at the same time and its elements whose identifier is missing from the
+   * `axbridge` tree are added to it (see `addMissingIdentifiedElements`).
+   * `axbridge` can report a React Native container as one unlabelled view
+   * without children: on the MetaMask wallet home, the balance and the
+   * Buy / Swap / Send / Receive buttons, so `wallet-send-button` was not found.
+   * `raw` stays the `axbridge` output.
+   *
+   * @returns The snapshot.
+   */
   async snapshot(): Promise<SnapshotResult> {
     await this.ensureConnected();
-    const raw = await this.#describeAll();
-    const hierarchy = parseIdbHierarchy(raw);
+    const apis = await this.#describeAllApiOrder();
+    const supplement =
+      apis[0] === 'axbridge' ? this.#describeSupplement('ax') : undefined;
+    const { raw, api } = await this.#describeAll(apis);
+    let hierarchy = parseIdbHierarchy(raw);
+    const extra = await supplement;
+    if (api === 'axbridge' && extra !== undefined) {
+      hierarchy = addMissingIdentifiedElements(
+        hierarchy,
+        parseIdbHierarchy(extra),
+      );
+    }
     return {
       platform: 'ios',
       hierarchy,
@@ -359,27 +380,65 @@ export class IdbBackend implements DeviceBackend {
     };
   }
 
+  /**
+   * Arguments of `idb ui describe-all` for one accessibility backend.
+   *
+   * @param api - The `--api` value, or null to omit it.
+   * @returns The idb arguments.
+   */
+  #describeAllArgs(api: 'ax' | 'axbridge' | null): string[] {
+    const args = ['ui', 'describe-all', '--udid', this.#udid];
+    if (api !== null) {
+      args.push('--api', api);
+    }
+    if (this.#idbSupportsNested) {
+      args.push('--nested');
+    }
+    return args;
+  }
+
+  /**
+   * Read a supplementary hierarchy. Best effort: no companion recovery (the
+   * main read does it), and a failure returns undefined.
+   *
+   * @param api - The `--api` value.
+   * @returns The raw JSON hierarchy, or undefined when the read failed.
+   */
+  async #describeSupplement(
+    api: 'ax' | 'axbridge',
+  ): Promise<string | undefined> {
+    try {
+      return await execStrict(this.#idbPath, this.#describeAllArgs(api), {
+        env: this.#connection.env,
+      });
+    } catch {
+      return undefined;
+    }
+  }
+
   /**
    * Run `idb ui describe-all`, trying the version-appropriate accessibility
    * backend first and falling back to the others if it errors. This keeps
    * snapshots working across iOS 17-26 regardless of the user's idb client.
    *
-   * @returns The raw JSON hierarchy from the first backend that succeeds.
+   * @param apis - The `--api` values to try, in order.
+   * @returns The raw JSON hierarchy from the first backend that succeeds, and
+   * that backend.
    * @throws When every attempted backend fails.
    */
-  async #describeAll(): Promise<string> {
-    const apis = await this.#describeAllApiOrder();
+  async #describeAll(
+    apis: ('ax' | 'axbridge' | null)[],
+  ): Promise<{ raw: string; api: 'ax' | 'axbridge' | null }> {
     let lastError: unknown;
     for (const api of apis) {
-      const args = ['ui', 'describe-all', '--udid', this.#udid];
-      if (api !== null) {
-        args.push('--api', api);
-      }
-      if (this.#idbSupportsNested) {
-        args.push('--nested');
-      }
       try {
-        return await this.#execStrict(this.#idbPath, args);
+        return {
+          raw: await this.#execStrict(
+            this.#idbPath,
+            this.#describeAllArgs(api),
+          ),
+          api,
+        };
       } catch (error) {
         if (error instanceof IdbUnavailableError) {
           // Already reconnected and retried; another --api will not help.
@@ -1107,6 +1166,80 @@ export function parseIdbHierarchy(raw: string): UIElement[] {
   }
 }
 
+/**
+ * Adds to `primary` the elements of `secondary` whose identifier is not in
+ * `primary`. Each one is added, without its children, as the last child of
+ * the smallest `primary` element whose frame contains its frame, or at the
+ * top level when none does. Elements with no identifier are not added. The
+ * inputs are not modified.
+ *
+ * @param primary - The main hierarchy (idb `--api axbridge`).
+ * @param secondary - The other hierarchy (idb `--api ax`).
+ * @returns `primary` with the missing elements added.
+ */
+export function addMissingIdentifiedElements(
+  primary: UIElement[],
+  secondary: UIElement[],
+): UIElement[] {
+  const result = structuredClone(primary);
+  const all: UIElement[] = [];
+  const collect = (elements: UIElement[]): void => {
+    for (const el of elements) {
+      all.push(el);
+      if (el.children) {
+        collect(el.children);
+      }
+    }
+  };
+  collect(result);
+  const known = new Set(all.map((el) => el.identifier).filter(Boolean));
+
+  const missing: UIElement[] = [];
+  const collectMissing = (elements: UIElement[]): void => {
+    for (const el of elements) {
+      if (el.identifier && !known.has(el.identifier)) {
+        missing.push(el);
+      }
+      if (el.children) {
+        collectMissing(el.children);
+      }
+    }
+  };
+  collectMissing(secondary);
+
+  for (const el of missing) {
+    const { children: _children, ...element } = el;
+    const added: UIElement = structuredClone(element);
+    const host = all
+      .filter((candidate) => containsFrame(candidate.frame, el.frame))
+      .reduce<
+        UIElement | undefined
+      >((best, candidate) => (best === undefined || area(candidate.frame) < area(best.frame) ? candidate : best), undefined);
+    if (host) {
+      host.children = [...(host.children ?? []), added];
+    } else {
+      result.push(added);
+    }
+  }
+  return result;
+}
+
+type Frame = UIElement['frame'];
+
+function area(frame: Frame): number {
+  return frame.width * frame.height;
+}
+
+function containsFrame(outer: Frame, inner: Frame): boolean {
+  return (
+    area(outer) > 0 &&
+    inner.x >= outer.x &&
+    inner.y >= outer.y &&
+    inner.x + inner.width <= outer.x + outer.width &&
+    inner.y + inner.height <= outer.y + outer.height
+  );
+}
+
 /**
  * Normalize idb fields, treating empty child arrays like flat-format leaves.
  *