From 1fc51ba440b4e7ea30bf8e680cce5d469fc543a8 Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 16:04:31 +0200 Subject: [PATCH 01/11] Capture on Android, and put both platforms in one map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit screenmap has been iOS-only. This adds Android as a peer platform: a run captures on iOS, on Android, or on both, and both land in a single .scrmap behind a platform switcher in the viewer. The device layer splits in two. lib/sim.mjs keeps the simctl primitives, lib/android.mjs is its adb counterpart, and lib/device.mjs holds what is genuinely shared — Metro, the dev-client connect loop, the capture helpers — so the platforms cannot quietly drift apart. argent needed no changes at all: its device tools take an Android serial wherever they take an iOS UDID, so committed-flow replay works on both for free. Three things are not one-to-one with simctl. Android has no "Open in …?" scheme prompt, so there is nothing to pre-approve. The emulator's localhost is not the host's, so the driver opens an adb reverse tunnel before pointing the dev client at Metro — which also covers USB devices, unlike the 10.0.2.2 alias. And the status bar is frozen through SystemUI demo mode rather than a dedicated override. Bundles stay backward compatible by construction. A single-platform run emits exactly the v2 layout it always did. Only a multi-platform run becomes v3: screens move under screens//, each node gains a `captures` map, and `capture` keeps mirroring the first platform so a v2 reader renders the map rather than showing every screen as missing. The diff bundle gains the same axis under its existing base/head split. Capture status is per platform, which is the point: a screen that renders on iOS and crashes on Android is one node with two verdicts. In CI the platforms are separate jobs, because iOS needs macOS and Android is only worth running on ubuntu, which bills at a tenth of the rate. `screenmap-ci merge` folds their bundles into one map so the split stays an implementation detail of the pipeline rather than something a reviewer has to hold in their head. OCR gets a tesseract backend behind the existing interface. Apple Vision is macOS-only, and without a fallback the landing checks, deep-link verification and system-alert dismissal all go dark on a Linux runner. tesseract reads fewer words than Vision, so the run summary and the PR comment now name the backend whenever it is not Vision — a drift warning from a Linux run deserves to be read with more suspicion than one from macOS. Verified against sample-app and its real captures: its untouched pre-existing config still produces identical v2 output, incremental reuse carries all six real captures forward, the multi-platform bundle and diff round-trip, and the refactored iOS driver boots a real simulator and captures from it. The adb driver itself has not met a live emulator — this machine has no Android SDK — and TODOS.md ranks what to check on the first real run. That testing caught one bug worth naming: a single-platform bundle used to answer for any platform asked of it, so a repo turning Android on "reused" its iOS screenshots as Android captures — labelled Android, and indistinguishable from a real run. Bundles now declare which platforms they hold, and merge refuses an input whose platform does not match its mapping. Co-Authored-By: Claude Opus 5 --- README.md | 24 +- TODOS.md | 81 +++++ action.yml | 102 +++++- action/cli/lib/agent.mjs | 32 +- action/cli/lib/android.mjs | 273 +++++++++++++++ action/cli/lib/bundle.mjs | 141 +++++++- action/cli/lib/device.mjs | 152 +++++++++ action/cli/lib/eas.mjs | 64 ++-- action/cli/lib/ocr.mjs | 132 +++++++- action/cli/lib/replay.mjs | 22 +- action/cli/lib/sim.mjs | 154 ++------- action/cli/lib/util.mjs | 45 ++- action/cli/screenmap-ci.mjs | 312 +++++++++++++----- action/templates/config.json | 11 +- action/templates/screenmap-baseline.yml | 3 + action/templates/screenmap-pr.yml | 42 +++ apps/visualiser/src/App.jsx | 3 + apps/visualiser/src/components/Graph.jsx | 5 +- apps/visualiser/src/components/ScreenNode.jsx | 15 +- apps/visualiser/src/components/TopBar.jsx | 38 ++- apps/visualiser/src/index.css | 7 +- apps/visualiser/src/lib/loadBundle.js | 57 +++- apps/visualiser/src/lib/useBundles.js | 25 +- docs/diff-scrmap-format.md | 31 +- docs/scrmap-format.md | 45 ++- plugins/screenmap/skills/screenmap/SKILL.md | 70 +++- .../skills/screenmap/scripts/diff-map.mjs | 92 ++++-- .../skills/screenmap/scripts/pack-map.mjs | 71 ++-- 28 files changed, 1674 insertions(+), 375 deletions(-) create mode 100644 action/cli/lib/android.mjs create mode 100644 action/cli/lib/device.mjs diff --git a/README.md b/README.md index 28640be..bfe1461 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,9 @@ PR opened ──▶ restore baseline (screenmaps branch) ──▶ static parse You need all four of these: - An Expo or React Native app using expo-router or a react-navigation route map, with `expo-dev-client` installed and deep-linkable routes. -- An EAS build profile that produces a simulator build, or a simulator build of your own. +- An EAS build profile that produces a simulator build (iOS) or an APK (Android — set `"android": { "buildType": "apk" }`, since an `.aab` cannot be installed on an emulator), or a build of your own. - A GitHub repo you can add secrets to. -- macOS runner minutes. screenmap runs on macOS only, which bills at ten times the Linux rate. A JavaScript-only pull request takes about 12 minutes end to end. +- Runner minutes. iOS needs macOS, which bills at ten times the Linux rate; Android runs on `ubuntu-latest`. A JavaScript-only pull request takes about 12 minutes end to end. ### Install @@ -121,10 +121,15 @@ You need all four of these: | --- | --- | --- | | `effort` | `balanced` | The preset from step 3. The `effort` input and `SCREENMAP_EFFORT` set the same thing | | `scheme` | from the parsed app config | URL scheme the deep links use | - | `device` | `iPhone 16 Pro` | Simulator to boot. In the Action, the `simulator` input boots the device | + | `platforms` | `["ios"]` | Which platforms a run captures — `["ios"]`, `["android"]`, or both. In the Action each platform is its own job (the `platform` input), folded together afterwards by `screenmap-ci merge` | + | `ios.device` | `iPhone 16 Pro` | Simulator to boot. In the Action, the `simulator` input boots the device | + | `ios.appPath` | discovered under `ios/build` | A prebuilt simulator `.app`. The `app_path` input wins over it | + | `ios.appId` | read from the `.app` | Bundle id. Override when discovery picks the wrong one | + | `android.device` | any attached device, else the first AVD | AVD name, or the model of an attached device. In the Action, the `avd` input | + | `android.appPath` | discovered under `android/app/build/outputs/apk` | A prebuilt `.apk`. The `app_path` input wins over it | + | `android.appId` | read from the APK with `aapt2` | Package name. Also accepted as `android.packageName` | | `appName` | the project directory name | Name recorded in the bundle | - | `bundleId` | read from the `.app` | Override when discovery picks the wrong one | - | `appPath` | discovered under `ios/build` | A prebuilt simulator `.app`. The `app_path` input wins over it | + | `device`, `bundleId`, `appPath` | — | Pre-multi-platform spellings of the `ios.*` keys above; still honoured | | `metroPort` | `8081` | Port Metro starts on | | `params` | `{}` | Real values for route parameters, see below | | `suspects.depth` | from `effort` | Import hops followed out from a changed file | @@ -283,7 +288,7 @@ What you get: - **Runtime states as first-class screens.** Bottom-sheet snap points, modals and drawers are captured as variants of the screen they belong to. - **The path to every screen, saved.** Each screen carries the exact tap sequence that reaches it, ready to replay headlessly. Commit those flows and CI replays them instead of paying an agent to rediscover them. -Output lands in `/.screenmap/out/`, so add that to your `.gitignore`. You need a macOS host with the iOS simulator. +Output lands in `/.screenmap/out/`, so add that to your `.gitignore`. You need a macOS host with the iOS simulator, or an Android emulator (`--platform android`); `--platform both` captures each screen on both and puts them in one map behind a platform switcher. ## The map viewer @@ -322,7 +327,7 @@ Drop a `.scrmap` bundle on the landing page. The demo bundle ships in `public/de ``` 1. **Static parse** (no dependencies). Reads expo-router file conventions and react-navigation route maps (the kind Bluesky keeps in `src/routes.ts`), so the screen list is complete rather than whatever a crawler happened to find. It produces the route list, navigation edges from `Link` and `navigate()` calls, and state hints saying which screens use a bottom-sheet or dialog system. -2. **Agent exploration** in the iOS simulator. A deep-link sweep captures every screen and classifies each capture (real, empty state, not found, error boundary, auth wall). For the screens a deep link cannot reach, an agent drives the app and records the tap path as an [argent](https://argent.swmansion.com) flow in YAML, replayable later with `argent flow run`. Runtime states get captured too: open drawers, bottom-sheet snap points, dialogs. If a sticky error boundary blocks the app, the agent recovers and carries on. +2. **Agent exploration** in the iOS simulator or Android emulator. A deep-link sweep captures every screen and classifies each capture (real, empty state, not found, error boundary, auth wall). For the screens a deep link cannot reach, an agent drives the app and records the tap path as an [argent](https://argent.swmansion.com) flow in YAML, replayable later with `argent flow run`. Runtime states get captured too: open drawers, bottom-sheet snap points, dialogs. If a sticky error boundary blocks the app, the agent recovers and carries on. 3. **Pack.** Everything merges into a producer-agnostic `.scrmap` zip. The format contract is in [docs/scrmap-format.md](docs/scrmap-format.md), which is what you need if you want to write your own producer. 4. **Visualise.** The viewer draws a top-down graph with the root screen at the top-center and phone-framed screenshots. Load a second bundle, a `.diff.scrmap`, and it overlays what a pull request changed. @@ -341,7 +346,10 @@ The expensive part is step 2, and you only pay it once. Recorded flows get commi ## Known limits -- **iOS only.** Android is not supported yet. The interactive phases need a macOS host with the iOS simulator, and there is a web fallback for capture but not for tap recording. +- **One platform per CI job.** iOS needs a macOS runner and Android is only worth running on Linux, so capturing both means two jobs and a `screenmap-ci merge` step; the workflow templates show the shape. A local run does both in one pass. +- **Android's dev-menu muting is best-effort.** iOS writes the preference through `simctl spawn defaults`; Android has to reach the app's SharedPreferences through `run-as`, which only works for a debuggable build. When it fails, the dev-menu floating button stays in the captures — cosmetic, and the run continues. +- **OCR on Linux is tesseract, not Vision.** The landing checks, deep-link verification and system-alert dismissal all read the screen, and tesseract recovers noticeably fewer words than Apple Vision. Screen-to-screen comparisons hold up (same-text scores are unchanged; different-screen scores only move further apart), but a landmark check is likelier to miss, so a drift warning from a Linux run is less certain than one from macOS. The run summary and the PR comment name the backend when it is not Vision. +- **Flows are per platform.** Coordinates are normalized, but layouts and system chrome are not, so a flow recorded on iOS is not guaranteed to replay on Android. Record and commit them per platform. - **Your app needs a router screenmap can read.** expo-router file conventions or a react-navigation route map. Screens registered without URLs are invisible to the static parse, and only show up through agent exploration. - **Edge extraction is regex-based**, so dynamic hrefs resolve to their route pattern. - **It reports, it does not gate.** There is no pass/fail check, by design. A reviewer decides what the screenshots mean. diff --git a/TODOS.md b/TODOS.md index 5aa3d0c..30f6407 100644 --- a/TODOS.md +++ b/TODOS.md @@ -6,6 +6,87 @@ the full write-up of what that turned up is in `site/docs/setup-instruction-fixe --- +## Android: what has and has not been run against a real device + +Android support landed on 2026-09-01. Everything that does not need an Android +device was exercised end to end, most of it against `sample-app` and its real +captures: config resolution, the multi-platform bundle (v3) and diff (v2), +incremental reuse, `screenmap-ci merge`, the viewer's platform switcher, the +tesseract OCR backend, and the refactored iOS driver against a live simulator. +**The `adb` driver in `action/cli/lib/android.mjs` has not been run against a +live emulator**, because the machine it was written on has no Android SDK. Treat +the first real run as the test, and check these in order — they are ranked by how +likely they are to be wrong: + +1. **`muteDevMenu()` is a guess.** It writes + `expo.modules.devmenu.sharedpreferences.xml` through `run-as`. The file name and + the three boolean keys were inferred from the iOS `EXDevMenu*` defaults, not read + off a device. If the dev-menu overlay shows up in captures, dump the real prefs + (`adb shell run-as ls /data/data//shared_prefs`) and fix the names. + Deliberately non-fatal: a wrong guess costs a floating button in the corner of + every screenshot, not a failed run. +2. **`appIdOf()` depends on `aapt2` being findable.** It searches PATH then every + `build-tools/*` under the SDK roots. A runner that ships the SDK without + build-tools falls through to a clear error telling the user to set + `android.packageName`, but that path has never actually fired. +3. **The connect loop's cold-start nudge.** `openUrl` on a force-stopped app is + assumed to launch it straight into the deep link, the way `simctl openurl` does. + If the first bundle never arrives, that assumption is the first thing to check — + diagnostics land in `.screenmap/out/ci/diag/android/` (screenshot, logcat tail, + package list, Metro log). +4. **SystemUI demo mode** is broadcast-based and silently does nothing on a build + where `sysui_demo_allowed` is not settable. A base/head pair with a drifting + clock in the diff means it did not take. + +Also unproven: the KVM setup in `action.yml` on a real ubuntu runner, and whether +`emulator -no-window` plus `-gpu swiftshader_indirect` boots fast enough there to +beat the macOS rate it exists to avoid. + +### The bug the sample-app run caught + +Worth recording because it was invisible and would have shipped a plausible-looking +wrong map. `baselineSide()` originally let a single-platform bundle answer for any +platform asked of it. A repo whose previous baseline was iOS-only, turning Android +on, therefore "reused" all six iOS screenshots as Android captures — labelled +Android in the viewer, and indistinguishable from a real run. Bundles now declare +which platforms they hold (`platformsIn()`), a side that is not there reports +`exists: false` so the new platform captures in full, and `merge` refuses an +input whose platform does not match the one it is being merged as rather than +silently mislabelling it. + +## OCR recall on the Linux lane + +Measured on 2026-09-01 against six real captures (downscaled 368x800) plus a +full-resolution simulator capture: + +- **The coordinate flip is correct**, which was the one thing that had to be. The + tesseract adapter reports pixels from the top-left and Vision reports normalized + from the bottom-left; across four real captures, 22 of 23 strings both backends + read agree on the resulting tap-y to within 0.006. (The one outlier is a screen + with two "About" labels, where the backends matched different instances.) A wrong + flip would have sent every system-alert dismissal to the mirror image of the + button. +- tesseract recovers ~61% of the words Vision does on app screens, and ~47% on a + sparse springboard capture, where it also missed the frozen "9:41" clock that + Vision read. Chrome-heavy, low-text screens are its worst case. Tuning did not + move it: + `--psm 6/11/12/3/4`, `--oem 1`, and a confidence floor all landed within a + point of each other. +- The decisions that gate a capture transfer intact. Landmark containment — the + strong signal — passed on the right screen and scored 0.00 on the wrong one + under both backends. Same-screen jaccard is 1.00 on both; different-screen + jaccard moves *down* (0.88 -> 0.75, 0.59 -> 0.48), which makes the + `bogus-param` probe more conservative rather than less. +- The weak `deeplink-text` fallback (`j >= 0.35`) is equally blunt on both: this + app's screens share enough chrome that different screens score 0.48-0.88. That + is a pre-existing property, not a tesseract regression, but it means a route + with no landmarks is barely verified on either platform. Committed landmarks + remain the only strong signal — say so in the docs rather than tuning 0.35. + +The run summary and the PR comment now name the OCR backend whenever it is not +Vision, so a drift warning from a Linux run can be read with the right amount of +suspicion. + ## Drifted flows have no repair path without an agent `effort=deterministic` (now the automatic choice when no agent key is set) makes diff --git a/action.yml b/action.yml index ed48bb5..2e6b6e9 100644 --- a/action.yml +++ b/action.yml @@ -4,8 +4,8 @@ name: screenmap description: >- Map every screen of an Expo / React Native app and review what a pull request changes on-screen. PR runs diff the head against a cached baseline map, capture only the affected screens on an iOS - simulator, and post a sticky comment linking a preloaded map viewer. Baseline runs refresh the map - of main incrementally and open a flows PR for screens the agent had to explore. + simulator or Android emulator, and post a sticky comment linking a preloaded map viewer. Baseline + runs refresh the map of main incrementally and open a flows PR for screens the agent had to explore. author: aleqsio branding: icon: map @@ -18,6 +18,13 @@ inputs: project: description: Path to the Expo project (repo-relative) default: "." + platform: + description: >- + ios (default, needs a macOS runner) or android (runs on ubuntu, which bills at a tenth of the + macOS rate). One platform per job: to get both in one map, run this action twice — iOS on + macos-latest, Android on ubuntu-latest — and fold the two bundles together with + `screenmap-ci merge --inputs ios=.scrmap,android=.scrmap`. See the workflow templates. + default: ios agent_provider: description: "Agent CLI for screens with no committed flow: claude (default) | codex | gemini | opencode. Or set agent.command in .screenmap/config.json to run any CLI." default: "" @@ -60,17 +67,31 @@ inputs: description: Hosted map viewer origin used in comment links default: https://app.screenmap.dev simulator: - description: Simulator device name to boot (falls back to any available iPhone) + description: (ios) Simulator device name to boot (falls back to any available iPhone) default: iPhone 17 Pro + avd: + description: >- + (android) Name of the AVD to boot. When it does not exist the action creates it from + android_system_image. Ignored if a device or emulator is already attached to adb. + default: screenmap + android_system_image: + description: (android) System image for the AVD the action creates, in sdkmanager form + default: system-images;android-34;google_apis;x86_64 app_path: - description: Path to a prebuilt simulator dev client (.app). Bring your own build — from a previous job's artifact, your own pipeline, or `eas build:run`-style downloads. When set, EAS is not touched. + description: >- + Path to a prebuilt dev client — an .app bundle for ios, an .apk for android. Bring your own + build — from a previous job's artifact, your own pipeline, or `eas build:run`-style downloads. + When set, EAS is not touched. default: "" expo_token: description: EXPO_TOKEN for the built-in EAS lane — the dev client is fetched from EAS (newest finished build whose fingerprint matches the checkout) or built there when none matches. Required unless app_path is set. default: "" eas_profile: - description: eas.json build profile for the simulator dev client (needs developmentClient + ios.simulator) - default: development-simulator + description: >- + eas.json build profile for the dev client. Empty picks per platform: development-simulator for + ios (needs developmentClient + ios.simulator), development-emulator for android (needs + developmentClient + android.buildType "apk" — an .aab cannot be installed on an emulator). + default: "" flows_pr: description: "(baseline) Open a PR against the default branch with flows the agent recorded" default: "true" @@ -95,11 +116,31 @@ runs: - name: Check runner shell: bash run: | - if [ "$(uname)" != "Darwin" ]; then echo "::error::screenmap needs a macOS runner (iOS simulator)"; exit 1; fi + case "${{ inputs.platform }}" in + ios|android) ;; + *) echo "::error::platform must be ios or android (got '${{ inputs.platform }}')"; exit 1 ;; + esac + if [ "${{ inputs.platform }}" = "ios" ] && [ "$(uname)" != "Darwin" ]; then + echo "::error::platform=ios needs a macOS runner (iOS simulator)"; exit 1 + fi if [ -z "${{ inputs.app_path }}${{ inputs.expo_token }}" ]; then echo "::error::provide expo_token (EAS builds the dev client) or app_path (bring your own build)"; exit 1 fi - xcrun simctl list devices available | head -20 + if [ "${{ inputs.platform }}" = "ios" ]; then + xcrun simctl list devices available | head -20 + else + [ -n "$ANDROID_HOME$ANDROID_SDK_ROOT" ] || { echo "::error::no Android SDK on this runner (ANDROID_HOME unset)"; exit 1; } + echo "Android SDK: ${ANDROID_HOME:-$ANDROID_SDK_ROOT}" + fi + + # Screen text drives the landing checks, deep-link verification and system-alert + # dismissal. macOS uses Apple Vision (compiled on demand by lib/ocr.mjs); Linux + # has no Vision, so without tesseract every one of those checks silently goes + # dark and a drifted flow stops being detectable. + - name: Install OCR backend (Linux) + if: runner.os == 'Linux' + shell: bash + run: sudo apt-get update -qq && sudo apt-get install -y -qq tesseract-ocr imagemagick - name: Install screenmap-ci shell: bash @@ -130,7 +171,8 @@ runs: elif [ -f yarn.lock ]; then yarn install --frozen-lockfile else npm ci; fi - - name: Boot simulator + - name: Boot simulator (ios) + if: inputs.platform == 'ios' shell: bash run: | udid=$(xcrun simctl list devices available -j | python3 -c 'import json,sys; d=json.load(sys.stdin)["devices"]; ds=[x for v in d.values() for x in v]; m=[x for x in ds if x["name"]=="${{ inputs.simulator }}"] or [x for x in ds if "iPhone" in x["name"]]; print(m[0]["udid"])') @@ -138,6 +180,38 @@ runs: xcrun simctl bootstatus "$udid" -b echo "SIM_UDID=$udid" >> "$GITHUB_ENV" + # KVM turns a ~40-minute software-rendered emulator boot into a ~2-minute one. + # It is the difference between Android being cheaper than macOS and not. + - name: Enable KVM (android on Linux) + if: inputs.platform == 'android' && runner.os == 'Linux' + shell: bash + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null + sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm + kvm-ok 2>/dev/null || echo "note: kvm-ok unavailable; continuing" + + - name: Boot emulator (android) + if: inputs.platform == 'android' + shell: bash + run: | + set -e + sdk="${ANDROID_HOME:-$ANDROID_SDK_ROOT}" + export PATH="$sdk/platform-tools:$sdk/emulator:$sdk/cmdline-tools/latest/bin:$PATH" + echo "$sdk/platform-tools" >> "$GITHUB_PATH" + echo "$sdk/emulator" >> "$GITHUB_PATH" + # a device already attached (self-hosted runner, or a previous step) wins + if adb devices | awk 'NR>1 && $2=="device"' | grep -q .; then + echo "using the attached device: $(adb devices | awk 'NR>1 && $2=="device" {print $1}' | head -1)" + else + if ! avdmanager list avd -c | grep -qx "${{ inputs.avd }}"; then + yes | sdkmanager --install "${{ inputs.android_system_image }}" >/dev/null + echo no | avdmanager create avd -n "${{ inputs.avd }}" -k "${{ inputs.android_system_image }}" --force + fi + # the driver (lib/android.mjs) boots and waits for sys.boot_completed; + # starting it here would just duplicate that logic + echo "AVD ${{ inputs.avd }} ready — the run boots it" + fi + - name: Dev client (EAS or provided) id: app shell: bash @@ -147,11 +221,16 @@ runs: set -e if [ -n "${{ inputs.app_path }}" ]; then app="$(cd "$(dirname "${{ inputs.app_path }}")" && pwd)/$(basename "${{ inputs.app_path }}")" - [ -d "$app" ] || { echo "::error::app_path not found: $app"; exit 1; } + # an iOS dev client is a .app directory; an Android one is an .apk file + [ -e "$app" ] || { echo "::error::app_path not found: $app"; exit 1; } echo "using provided dev client: $app" else command -v eas >/dev/null || npm install -g eas-cli --silent - screenmap-ci resolve-app --project "${{ inputs.project }}" --profile "${{ inputs.eas_profile }}" | tee /tmp/screenmap-app.json + profile="${{ inputs.eas_profile }}" + if [ -z "$profile" ]; then + if [ "${{ inputs.platform }}" = "android" ]; then profile=development-emulator; else profile=development-simulator; fi + fi + screenmap-ci resolve-app --project "${{ inputs.project }}" --platform "${{ inputs.platform }}" --profile "$profile" | tee /tmp/screenmap-app.json app=$(python3 -c 'import json,sys;print(json.load(open("/tmp/screenmap-app.json"))["appPath"])') fi echo "app=$app" >> "$GITHUB_OUTPUT" @@ -188,6 +267,7 @@ runs: AGENT_MAX_SCREENS: ${{ inputs.agent_max_screens }} AGENT_API_KEY: ${{ inputs.agent_api_key }} SCREENMAP_APP_PATH: ${{ steps.app.outputs.app }} + SCREENMAP_PLATFORMS: ${{ inputs.platform }} GH_TOKEN: ${{ inputs.github_token }} run: | set -e diff --git a/action/cli/lib/agent.mjs b/action/cli/lib/agent.mjs index c83063e..3334982 100644 --- a/action/cli/lib/agent.mjs +++ b/action/cli/lib/agent.mjs @@ -86,7 +86,25 @@ export function agentInfo(config) { return { provider: p.custom ? 'custom command' : p.name, keyEnv: p.keyEnv ?? null, hasKey: p.keyEnv ? !!env[p.keyEnv] : null } } -export function runAgent({ projectDir, config, screens, scheme, udid, bundleId, outScreensDir, outFlowsDir, notesPath, summaryPath, mode, prContext }) { +// What the agent needs to know that differs per platform: which CLI drives the +// device, what the app id is called, and what the device is. argent is the same +// on both — its device tools take an iOS UDID or an Android serial alike. +const PLATFORM_BRIEF = { + ios: { + cli: '`xcrun simctl` for deep links/screenshots', + deepLink: (id, url) => `xcrun simctl openurl ${id} "${url}"`, + shot: (id, out) => `xcrun simctl io ${id} screenshot ${out}`, + device: 'simulator', appId: 'bundle', + }, + android: { + cli: '`adb` for deep links/screenshots', + deepLink: (id, url) => `adb -s ${id} shell am start -a android.intent.action.VIEW -d '${url}'`, + shot: (id, out) => `adb -s ${id} exec-out screencap -p > ${out}`, + device: 'device', appId: 'package', + }, +} + +export function runAgent({ projectDir, config, screens, scheme, udid, bundleId, platform = 'ios', deviceName, outScreensDir, outFlowsDir, notesPath, summaryPath, mode, prContext }) { const info = agentInfo(config) if (!screens.length) return { ran: false, reason: 'nothing to explore', ...info } if (!config.agent.enabled) return { ran: false, reason: config.effort === 'deterministic' ? 'effort=deterministic — flows replay, nothing is re-checked' : 'agent disabled in .screenmap/config.json', ...info } @@ -101,7 +119,13 @@ export function runAgent({ projectDir, config, screens, scheme, udid, bundleId, const repoSkill = path.join(projectDir, config.skillFile) const repoSkillText = fs.existsSync(repoSkill) ? fs.readFileSync(repoSkill, 'utf8') : null - const prompt = `You are running the screenmap skill's capture phases headlessly in CI (no simulator MCP — use \`xcrun simctl\` for deep links/screenshots and the \`argent\` CLI for taps/swipes: \`argent run …\` (\`argent tools\` lists them; if \`argent\` is not on PATH, run \`npx -y @swmansion/argent@0.21.0\` from a directory OUTSIDE the project, e.g. /tmp, because this repo's devEngines pin breaks npx inside it)). The app is already running on simulator ${udid} (bundle ${bundleId}, scheme ${scheme}://), Metro is up. Do not rebuild, reinstall, or checkout anything. + const brief = PLATFORM_BRIEF[platform] ?? PLATFORM_BRIEF.ios + const device = deviceName ?? config.device ?? brief.device + + const prompt = `You are running the screenmap skill's capture phases headlessly in CI on ${platform.toUpperCase()} (no simulator MCP — use ${brief.cli} and the \`argent\` CLI for taps/swipes: \`argent run …\` (\`argent tools\` lists them; its device tools take this ${brief.device}'s id directly; if \`argent\` is not on PATH, run \`npx -y @swmansion/argent@0.21.0\` from a directory OUTSIDE the project, e.g. /tmp, because this repo's devEngines pin breaks npx inside it)). The app is already running on ${brief.device} ${udid} (${brief.appId} ${bundleId}, scheme ${scheme}://), Metro is up. Do not rebuild, reinstall, or checkout anything. + +Deep link: ${brief.deepLink(udid, `${scheme}://some/path`)} +Screenshot: ${brief.shot(udid, `${outScreensDir}/.png`)} Read the skill at ${SKILL_DIR}/SKILL.md for conventions (capture naming, flow recording format, safety rules: never tap destructive/purchase/sign-out controls, never record credentials). The project lives at ${projectDir}. All output paths below are absolute — write to them exactly. ${repoSkillText ? `\nProject-specific guidance (.screenmap/SKILL.md) — follow it:\n---\n${repoSkillText}\n---\n` : ''} @@ -110,13 +134,13 @@ ${budgeted.map((s) => `- ${s.id} urlPath=${s.urlPath} slug=${s.slug} deepLink Rules: 1. Screenshots go to ${outScreensDir}/.png (state variants: --.png). Use exactly these slugs. -2. Flows go to ${outFlowsDir}/ as argent YAML + .meta.json sidecars (formatVersion 2) — nav- for the tap path from app launch (\`${scheme}://\`), visit- for the bare deep link, plus one flow per state variant you capture. Coordinates normalized 0–1 for a ${config.device}. Every sidecar MUST include \`"landmarks": [2–5 words visible on the arrival screen that identify it — titles/section headers/fixed labels, never live content]\`; CI verifies replays by OCR-ing for them. +2. Flows go to ${outFlowsDir}/ as argent YAML + .meta.json sidecars (formatVersion 2) — nav- for the tap path from app launch (\`${scheme}://\`), visit- for the bare deep link, plus one flow per state variant you capture. Coordinates normalized 0–1 for a ${device}. Every sidecar MUST include \`"landmarks": [2–5 words visible on the arrival screen that identify it — titles/section headers/fixed labels, never live content]\`; CI verifies replays by OCR-ing for them. 3. Prefer the deep link first; if it shows an error/not-found, find real params (public API, other screens) and note what you used. 4. Write ${notesPath}: JSON { "": "one sentence describing what this screen shows${mode === 'pr' ? ' / what visibly changed in this PR' : ''}" } for each screen you handled${mode === 'pr' ? ', or { "note": "...", "verdict": "unaffected" } if the PR diff shows no visible change there' : ''}. 5. Write ${summaryPath}: JSON { "captured": [routeIds], "skipped": [{ "id", "why" }], "flows": [flow names] } when done. 6. Budget: these ${budgeted.length} screens only. Be economical — no broad exploration.${prContext ? `\n\nPR context: ${prContext}` : ''}` - log(`agent (${provider.name}): exploring ${budgeted.length} screen(s)${skipped.length ? `, ${skipped.length} over budget` : ''}`) + log(`agent (${provider.name}, ${platform}): exploring ${budgeted.length} screen(s)${skipped.length ? `, ${skipped.length} over budget` : ''}`) let r if (provider.custom) { // custom command template: {promptFile} is substituted; the prompt is also diff --git a/action/cli/lib/android.mjs b/action/cli/lib/android.mjs new file mode 100644 index 0000000..fb0f091 --- /dev/null +++ b/action/cli/lib/android.mjs @@ -0,0 +1,273 @@ +// Android emulator/device driver — the `adb` half of the device layer, mirroring +// lib/sim.mjs's iOS driver. Everything here is `adb` and the SDK's `emulator` +// binary: no MCP, no LLM, and it runs on a Linux runner, which is the whole +// point (macOS minutes bill at ten times the Linux rate). +// +// Most calls map one-to-one onto their simctl counterpart. Three do not: +// - Android has no "Open in …?" scheme prompt, so there is nothing to +// pre-approve and no prompt to tap through. +// - the emulator cannot reach the host's Metro on `localhost`, so the driver +// opens an `adb reverse` tunnel instead of rewriting the URL to 10.0.2.2 — +// the tunnel also covers physical devices over USB, which 10.0.2.2 does not. +// - the status bar is frozen through SystemUI demo mode rather than a +// dedicated override command. +import fs from 'node:fs' +import path from 'node:path' +import { spawn, spawnSync } from 'node:child_process' +import { sh, shOk, sleep, log } from './util.mjs' + +// SDK tools: PATH first, then the standard SDK layout under ANDROID_HOME / +// ANDROID_SDK_ROOT. GitHub's ubuntu runners set ANDROID_HOME but do not always +// put platform-tools on PATH. +const sdkRoots = () => [process.env.ANDROID_HOME, process.env.ANDROID_SDK_ROOT, + process.env.HOME && path.join(process.env.HOME, 'Library', 'Android', 'sdk'), + process.env.HOME && path.join(process.env.HOME, 'Android', 'Sdk')].filter(Boolean) + +const toolCache = new Map() +function sdkTool(name, ...subdirs) { + if (toolCache.has(name)) return toolCache.get(name) + let found = null + if (spawnSync('which', [name], { encoding: 'utf8' }).status === 0) found = name + if (!found) for (const root of sdkRoots()) { + for (const d of subdirs) { + const p = path.join(root, d, name) + if (fs.existsSync(p)) { found = p; break } + } + if (found) break + } + toolCache.set(name, found) + return found +} +export const adbPath = () => sdkTool('adb', 'platform-tools') +export const emulatorPath = () => sdkTool('emulator', 'emulator', 'tools') + +function adb(id, args, opts = {}) { + const bin = adbPath() + if (!bin) throw new Error('adb not found — install Android platform-tools or set ANDROID_HOME') + return sh(bin, [...(id ? ['-s', id] : []), ...args], opts) +} +function adbOk(id, args, opts = {}) { + const bin = adbPath() + if (!bin) return false + return shOk(bin, [...(id ? ['-s', id] : []), ...args], opts) +} +// One shell string rather than argv: adb joins its arguments with spaces and +// hands the result to the device's own shell, so a deep link containing `&` +// (`?a=1&b=2`) would otherwise background the command on the device. Quoting +// here is the only place that can fix it. +const shellQuote = (s) => `'${String(s).replace(/'/g, `'\\''`)}'` + +export const platform = 'android' +export const label = 'Android emulator' + +export function listBooted() { + const bin = adbPath() + if (!bin) return [] + let out = '' + try { out = sh(bin, ['devices', '-l']) } catch { return [] } + return out.split('\n').slice(1).map((l) => l.trim()).filter((l) => /\sdevice(\s|$)/.test(l)) + .map((l) => { + const id = l.split(/\s+/)[0] + const model = l.match(/model:(\S+)/)?.[1]?.replace(/_/g, ' ') + let name = model + try { name = sh(bin, ['-s', id, 'shell', 'getprop', 'ro.product.model']).trim() || model } catch {} + return { id, name: name ?? id } + }) +} + +export function listAvds() { + const bin = emulatorPath() + if (!bin) return [] + try { return sh(bin, ['-list-avds']).split('\n').map((s) => s.trim()).filter(Boolean) } catch { return [] } +} + +// Wait for the device to finish booting. `wait-for-device` only waits for adb +// to see it; the package manager is not up until sys.boot_completed flips, and +// installing before that fails in ways that read as a broken APK. +async function waitBootComplete(id, timeoutMs = 300000) { + const deadline = Date.now() + timeoutMs + adbOk(id, ['wait-for-device']) + while (Date.now() < deadline) { + let done = '' + try { done = adb(id, ['shell', 'getprop', 'sys.boot_completed']).trim() } catch {} + if (done === '1') { + // dismiss the lock screen; a freshly booted AVD comes up locked and every + // capture would otherwise be the lock screen + adbOk(id, ['shell', 'input', 'keyevent', '82']) + adbOk(id, ['shell', 'wm', 'dismiss-keyguard']) + return true + } + await sleep(2000) + } + return false +} + +export async function ensureBooted(config) { + const booted = listBooted() + if (booted.length) { + const pick = config.device ? booted.find((d) => d.name === config.device) ?? booted[0] : booted[0] + log(`android device already available: ${pick.name} (${pick.id})`) + await waitBootComplete(pick.id, 60000) + return pick + } + const bin = emulatorPath() + if (!bin) throw new Error('no Android device connected and no emulator binary found (set ANDROID_HOME)') + const avds = listAvds() + if (!avds.length) throw new Error('no Android device connected and no AVD defined — create one with avdmanager') + const avd = avds.find((a) => a === config.device) ?? avds[0] + log(`booting AVD ${avd}`) + // detached: the emulator runs for the whole session and must outlive this call + const proc = spawn(bin, ['-avd', avd, '-no-snapshot', '-no-boot-anim', '-no-audio', + ...(process.env.SCREENMAP_EMULATOR_WINDOW === '1' ? [] : ['-no-window']), + '-gpu', 'swiftshader_indirect'], { detached: true, stdio: 'ignore' }) + proc.unref() + const deadline = Date.now() + 300000 + while (Date.now() < deadline) { + const now = listBooted() + if (now.length) { await waitBootComplete(now[0].id); return now[0] } + await sleep(3000) + } + throw new Error(`AVD ${avd} did not come up within 5 minutes`) +} + +// SystemUI demo mode is Android's answer to `simctl status_bar override`: +// identical clock/battery/signal on every capture, so base and head +// screenshots only differ where the app differs. +export function freezeStatusBar(id) { + adbOk(id, ['shell', 'settings', 'put', 'global', 'sysui_demo_allowed', '1']) + const demo = (...kv) => adbOk(id, ['shell', 'am', 'broadcast', '-a', 'com.android.systemui.demo', ...kv]) + demo('-e', 'command', 'enter') + demo('-e', 'command', 'clock', '-e', 'hhmm', '0941') + demo('-e', 'command', 'battery', '-e', 'level', '100', '-e', 'plugged', 'false') + demo('-e', 'command', 'network', '-e', 'wifi', 'show', '-e', 'level', '4') + demo('-e', 'command', 'network', '-e', 'mobile', 'show', '-e', 'datatype', 'none', '-e', 'level', '4') + demo('-e', 'command', 'notifications', '-e', 'visible', 'false') +} + +export function findBuiltApp(projectDir) { + const roots = [ + path.join(projectDir, 'android', 'app', 'build', 'outputs', 'apk', 'debug'), + path.join(projectDir, 'android', 'app', 'build', 'outputs', 'apk', 'release'), + ] + for (const d of roots) { + if (!fs.existsSync(d)) continue + const apk = fs.readdirSync(d).find((f) => f.endsWith('.apk')) + if (apk) return path.join(d, apk) + } + return null +} + +function buildToolsDirs() { + const dirs = [] + for (const root of sdkRoots()) { + const bt = path.join(root, 'build-tools') + if (!fs.existsSync(bt)) continue + for (const v of fs.readdirSync(bt).sort().reverse()) dirs.push(path.join('build-tools', v)) + } + return dirs.length ? dirs : ['build-tools'] +} + +// Package name out of the APK. aapt2 is the modern tool and ships in every +// build-tools release; aapt is the fallback for older SDK installs. +export function appIdOf(apkPath) { + const aapt2 = sdkTool('aapt2', ...buildToolsDirs()) + if (aapt2) { + try { return sh(aapt2, ['dump', 'packagename', apkPath]).trim().split('\n')[0] } catch {} + } + const aapt = sdkTool('aapt', ...buildToolsDirs()) + if (aapt) { + try { return sh(aapt, ['dump', 'badging', apkPath]).match(/package: name='([^']+)'/)?.[1] ?? null } catch {} + } + throw new Error('cannot read the package name from the APK (no aapt2/aapt) — set packageName in .screenmap/config.json') +} + +// -r reinstall, -t allow test-only builds (EAS development APKs are marked +// test-only), -g pre-grant every runtime permission the manifest declares — +// which is most of grantPrivacy's job done at install time. +export function installApp(id, apkPath) { + const bin = adbPath() + const run = () => spawnSync(bin, ['-s', id, 'install', '-r', '-t', '-g', apkPath], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }) + let r = run() + let out = (r.stdout || '') + (r.stderr || '') + // a signature clash with a previously installed build is the common case + if ((r.status !== 0 || /Failure/.test(out)) && /INSTALL_FAILED_UPDATE_INCOMPATIBLE|signatures do not match/.test(out)) { + log('signature mismatch — uninstalling the previous build and retrying') + adbOk(id, ['uninstall', appIdOf(apkPath)]) + r = run() + out = (r.stdout || '') + (r.stderr || '') + } + if (r.status !== 0 || /Failure/.test(out)) throw new Error(`adb install failed: ${out.trim().slice(-400)}`) +} + +export function terminate(id, pkg) { adbOk(id, ['shell', 'am', 'force-stop', pkg]) } +export function launch(id, pkg) { + // monkey launches the default LAUNCHER activity without us having to know its + // name, which varies between bare and managed Expo projects + adb(id, ['shell', `monkey -p ${shellQuote(pkg)} -c android.intent.category.LAUNCHER 1`]) +} +export function openUrl(id, url, pkg) { + adb(id, ['shell', `am start -a android.intent.action.VIEW -d ${shellQuote(url)}${pkg ? ` ${shellQuote(pkg)}` : ''}`]) +} + +export function screenshot(id, outPath) { + fs.mkdirSync(path.dirname(outPath), { recursive: true }) + freezeStatusBar(id) // tooling in between (argent) can drop demo mode + const bin = adbPath() + // exec-out keeps the PNG byte-exact; `adb shell screencap` mangles newlines + const r = spawnSync(bin, ['-s', id, 'exec-out', 'screencap', '-p'], { maxBuffer: 128 * 1024 * 1024 }) + if (r.status !== 0 || !r.stdout?.length) throw new Error(`screencap failed: ${(r.stderr || '').toString().slice(-300)}`) + fs.writeFileSync(outPath, r.stdout) + return outPath +} + +// `adb install -g` already granted everything in the manifest; this covers a +// device where that flag was refused, so a mis-tap can never summon a runtime +// permission dialog that then sits over every later capture. +const RUNTIME_PERMS = ['CAMERA', 'RECORD_AUDIO', 'ACCESS_FINE_LOCATION', 'ACCESS_COARSE_LOCATION', + 'READ_CONTACTS', 'WRITE_CONTACTS', 'READ_CALENDAR', 'WRITE_CALENDAR', 'READ_EXTERNAL_STORAGE', + 'WRITE_EXTERNAL_STORAGE', 'READ_MEDIA_IMAGES', 'READ_MEDIA_VIDEO', 'READ_MEDIA_AUDIO', + 'POST_NOTIFICATIONS', 'ACTIVITY_RECOGNITION'] +export function grantPrivacy(id, pkg) { + const granted = [] + for (const p of RUNTIME_PERMS) if (adbOk(id, ['shell', 'pm', 'grant', pkg, `android.permission.${p}`])) granted.push(p) + if (granted.length) log(`pre-granted ${granted.length} runtime permissions`) +} + +// expo-dev-menu's Android preferences live in the app's own SharedPreferences, +// which only `run-as` can reach and only for a debuggable build. Best-effort by +// design: when it fails the dev-menu floating button stays in the captures, +// which is cosmetic, so nothing here is allowed to abort a run. +// +// NOTE: unlike the iOS `defaults write` path, this has NOT been verified +// against a running emulator — see "Known limits" in the README. +export function muteDevMenu(id, pkg) { + const xml = `\n\n\n\n\n` + const target = `/data/data/${pkg}/shared_prefs/expo.modules.devmenu.sharedpreferences.xml` + const inner = `mkdir -p /data/data/${pkg}/shared_prefs && printf %s ${shellQuote(xml)} > ${target}` + const ok = adbOk(id, ['shell', `run-as ${shellQuote(pkg)} sh -c ${shellQuote(inner)}`]) + if (!ok) log('could not mute the dev menu (run-as unavailable) — its overlay may appear in captures') +} + +// Nothing to approve: Android resolves a custom scheme straight to the app. +export function approveScheme() {} +// …and so there is no prompt to tap through either. +export function nudgeOpenPrompt() { return false } + +// The emulator's `localhost` is the emulator, not the host. `adb reverse` maps +// the device's port back to the runner's, which also works over USB — unlike +// the 10.0.2.2 alias, which is emulator-only. +export function connectMetro(id, port) { + if (!adbOk(id, ['reverse', `tcp:${port}`, `tcp:${port}`])) { + log(`adb reverse tcp:${port} failed — the app may not reach Metro`) + return false + } + log(`adb reverse tcp:${port} → host`) + return true +} + +export function diagnostics(id, dir) { + fs.mkdirSync(dir, { recursive: true }) + try { screenshot(id, path.join(dir, 'connect-timeout.png')) } catch {} + try { fs.writeFileSync(path.join(dir, 'packages.txt'), adb(id, ['shell', 'pm', 'list', 'packages'])) } catch {} + try { fs.writeFileSync(path.join(dir, 'logcat.txt'), adb(id, ['logcat', '-d', '-t', '500'])) } catch {} +} diff --git a/action/cli/lib/bundle.mjs b/action/cli/lib/bundle.mjs index 98dcce1..8eadde7 100644 --- a/action/cli/lib/bundle.mjs +++ b/action/cli/lib/bundle.mjs @@ -1,7 +1,7 @@ // .scrmap / .diff.scrmap helpers: read a baseline bundle, turn its map back // into a parse-routes-shaped graph, pack a new baseline (reusing screenshots // for unchanged screens), and call the skill's diff-map.mjs to pack a diff. -import { execFileSync } from 'node:child_process' +import { execFileSync, spawnSync } from 'node:child_process' import fs from 'node:fs' import path from 'node:path' import { readJson, writeJson, ensureDir, log } from './util.mjs' @@ -35,6 +35,44 @@ export function graphFromMap(manifest, map) { } } +// Which platforms a bundle actually carries. A v3 bundle lists them; a v1/v2 +// bundle has exactly one, named by the manifest's platform label (and a bundle +// old enough to name nothing predates Android, so it is iOS). +export function platformsIn(manifest) { + if (Array.isArray(manifest?.app?.platforms) && manifest.app.platforms.length) { + return manifest.app.platforms.map((p) => p.platform) + } + const label = manifest?.app?.platform + if (label === 'android-emulator') return ['android'] + if (label === 'ios-simulator' || !label) return ['ios'] + return [label] +} + +// Read one platform's side of a previous baseline: where its screenshots live, +// how to find a node's capture record, and its capture-status map. +// +// A bundle is only allowed to answer for platforms it actually holds. This is +// the whole point: when a repo turns Android on, its previous baseline is +// iOS-only, and a side that answered anyway would hand every Android screen the +// iOS screenshot — captures that look real, are labelled Android, and are not. +// An absent side reports exists:false, so every route counts as stale and the +// new platform captures in full. +export function baselineSide(prev, platform) { + const has = platformsIn(prev.manifest) + if (!has.includes(platform)) { + return { dir: null, exists: false, capture: () => null, status: {} } + } + const multi = has.length > 1 + const dir = multi ? path.join(prev.dir, 'screens', platform) : prev.screensDir + const rawStatus = readJson(path.join(prev.dir, 'capture-status.json'), {}) + return { + dir, + exists: fs.existsSync(dir), + capture: (node) => (multi ? node?.captures?.[platform] : node?.capture) ?? null, + status: multi ? rawStatus[platform] ?? {} : rawStatus, + } +} + export function parseRoutes(projectDir, outPath) { execFileSync('node', [path.join(SKILL_SCRIPTS, 'parse-routes.mjs'), projectDir, '--out', outPath], { stdio: ['ignore', 'ignore', 'inherit'] }) return readJson(outPath) @@ -50,54 +88,129 @@ export function computeSuspects({ diffDir, baseGraph, headGraph, changedFiles, p return readJson(path.join(diffDir, 'suspects.json')) } -export function packDiff({ diffDir, device, out }) { +// `platforms` is [{ platform, device }] in capture order; diff-map.mjs takes +// them as parallel comma-separated lists. +export function packDiff({ diffDir, device, platforms, out }) { const args = [path.join(SKILL_SCRIPTS, 'diff-map.mjs'), 'pack', diffDir] - if (device) args.push('--device', device) + if (platforms?.length) { + args.push('--platforms', platforms.map((p) => p.platform).join(',')) + args.push('--device', platforms.map((p) => p.device ?? '').join(',')) + } else if (device) args.push('--device', device) if (out) args.push('--out', out) execFileSync('node', args, { stdio: ['ignore', process.stderr, 'inherit'] }) return out ?? fs.readdirSync(diffDir).filter((f) => f.endsWith('.diff.scrmap')).map((f) => path.join(diffDir, f))[0] } +// How a platform is named in the manifest's `app.platform` field. +export const PLATFORM_LABELS = { ios: 'ios-simulator', android: 'android-emulator' } + +// Build one platform's capture record for a route: the bare screenshot plus +// its state variants, resolved against that platform's screens directory. +function captureFor(r, cs, shotFiles, prefix) { + const baseShot = shotFiles.find((f) => f.replace(/\.\w+$/, '') === r.slug) + const states = shotFiles.filter((f) => f.startsWith(r.slug + '--')) + .map((f) => ({ name: f.replace(/\.\w+$/, '').slice(r.slug.length + 2), screenshot: prefix + f })) + .sort((a, b) => a.name.localeCompare(b.name)) + return { + status: cs.status ?? (baseShot ? 'ok' : 'missing'), + note: cs.note ?? null, + needsNavigation: cs.needsNavigation ?? false, + screenshot: baseShot ? prefix + baseShot : null, + states, + } +} + // Pack a baseline .scrmap from loose parts. Mirrors pack-map.mjs but takes // explicit dirs and adds graph.json + commit metadata (producer extensions; // viewers ignore unknown files/fields). -export function packBaseline({ graph, screensDir, flowsDir, captureStatus = {}, appName, device, commit, ref, out }) { - const shotFiles = fs.existsSync(screensDir) ? fs.readdirSync(screensDir).filter((f) => /\.(png|jpe?g|webp)$/i.test(f)) : [] +// +// `platforms` is [{ platform, device, screensDir }] in capture order. With a +// single platform the output is byte-for-byte the v2 layout it always was — +// `screens/.png` and one `capture` per node. With more than one, screens +// move into `screens//` and each node gains a `captures` map; `capture` +// keeps mirroring the FIRST platform so a v2 viewer still renders the map +// instead of showing every screen as missing. +export function packBaseline({ graph, platforms, screensDir, flowsDir, captureStatus = {}, appName, device, commit, ref, out }) { + // legacy single-platform call shape + if (!platforms) platforms = [{ platform: 'ios', device, screensDir }] + const multi = platforms.length > 1 + // captureStatus is per-platform when multi, flat (iOS-only) when not + const statusFor = (p) => (multi ? captureStatus[p] ?? {} : captureStatus) + + const sides = platforms.map((p) => { + const prefix = multi ? `screens/${p.platform}/` : 'screens/' + const files = fs.existsSync(p.screensDir) ? fs.readdirSync(p.screensDir).filter((f) => /\.(png|jpe?g|webp)$/i.test(f)) : [] + return { ...p, prefix, files, status: statusFor(p.platform) } + }) const flowFiles = flowsDir && fs.existsSync(flowsDir) ? fs.readdirSync(flowsDir).filter((f) => f.endsWith('.yaml') || f.endsWith('.meta.json')) : [] + const nodes = graph.routes.map((r) => { - const cs = captureStatus[r.id] ?? {} - const baseShot = shotFiles.find((f) => f.replace(/\.\w+$/, '') === r.slug) - const states = shotFiles.filter((f) => f.startsWith(r.slug + '--')).map((f) => ({ name: f.replace(/\.\w+$/, '').slice(r.slug.length + 2), screenshot: 'screens/' + f })).sort((a, b) => a.name.localeCompare(b.name)) + const per = Object.fromEntries(sides.map((s) => [s.platform, captureFor(r, s.status[r.id] ?? {}, s.files, s.prefix)])) return { id: r.id, urlPath: r.urlPath, file: r.file ?? null, slug: r.slug, group: r.layoutDir ?? '', navigator: r.navigator ?? null, params: r.params ?? [], presentation: r.presentation ?? null, stateHints: r.stateHints ?? [], - capture: { status: cs.status ?? (baseShot ? 'ok' : 'missing'), note: cs.note ?? null, needsNavigation: cs.needsNavigation ?? false, screenshot: baseShot ? 'screens/' + baseShot : null, states }, + capture: per[sides[0].platform], + ...(multi ? { captures: per } : {}), } }) const map = { nodes, edges: graph.edges ?? [], flows: [] } const manifest = { - formatVersion: 2, flowFormat: 'argent', generator: 'screenmap-ci/0.1', - app: { name: appName, scheme: graph.scheme ?? null, platform: 'ios-simulator', device: device ?? null, mode: graph.mode ?? null }, + formatVersion: multi ? 3 : 2, flowFormat: 'argent', generator: 'screenmap-ci/0.1', + app: { + name: appName, scheme: graph.scheme ?? null, + platform: PLATFORM_LABELS[sides[0].platform] ?? sides[0].platform, + device: sides[0].device ?? null, mode: graph.mode ?? null, + ...(multi ? { platforms: sides.map((s) => ({ platform: s.platform, label: PLATFORM_LABELS[s.platform] ?? s.platform, device: s.device ?? null })) } : {}), + }, source: { commit: commit ?? null, ref: ref ?? null }, generatedAt: new Date().toISOString(), } const stage = fs.mkdtempSync(path.join(path.dirname(out), '.pack-')) + let shotCount = 0 try { writeJson(path.join(stage, 'manifest.json'), manifest) writeJson(path.join(stage, 'map.json'), map) writeJson(path.join(stage, 'graph.json'), graph) writeJson(path.join(stage, 'capture-status.json'), captureStatus) ensureDir(path.join(stage, 'screens')); ensureDir(path.join(stage, 'flows')) - for (const f of shotFiles) fs.copyFileSync(path.join(screensDir, f), path.join(stage, 'screens', f)) + for (const s of sides) { + const dest = ensureDir(path.join(stage, 'screens', ...(multi ? [s.platform] : []))) + for (const f of s.files) { fs.copyFileSync(path.join(s.screensDir, f), path.join(dest, f)); shotCount++ } + } for (const f of flowFiles) fs.copyFileSync(path.join(flowsDir, f), path.join(stage, 'flows', f)) fs.rmSync(out, { force: true }) execFileSync('zip', ['-r', '-q', out, 'manifest.json', 'map.json', 'graph.json', 'capture-status.json', 'screens', 'flows'], { cwd: stage }) } finally { fs.rmSync(stage, { recursive: true, force: true }) } - log(`packed ${out}: ${nodes.length} nodes, ${shotFiles.length} shots, ${flowFiles.length / 2 | 0} flows`) + log(`packed ${out}: ${nodes.length} nodes, ${shotCount} shots across ${sides.map((s) => s.platform).join('+')}, ${flowFiles.length / 2 | 0} flows`) return { manifest, map } } +// Downscale captures to 800px on the long edge before packing. `sips` is +// macOS-only and the Android lane runs on Linux, so fall back through the +// resizers a ubuntu runner actually has — without one, bundles would quietly +// ship full-resolution screenshots and grow several-fold. +let resizer +function findResizer() { + if (resizer !== undefined) return resizer + const cands = [ + ['sips', (f) => ['-Z', '800', f]], + ['magick', (f) => [f, '-resize', '800x800>', f]], + ['convert', (f) => [f, '-resize', '800x800>', f]], + ] + for (const [bin, args] of cands) { + if (spawnSync('which', [bin], { encoding: 'utf8' }).status === 0) return (resizer = { bin, args }) + } + log('no image resizer found (sips/magick/convert) — screenshots ship at full size') + return (resizer = null) +} + export function downscaleAll(dir) { if (!fs.existsSync(dir)) return - for (const f of fs.readdirSync(dir)) if (/\.png$/i.test(f)) { try { execFileSync('sips', ['-Z', '800', path.join(dir, f)], { stdio: 'ignore' }) } catch {} } + const r = findResizer() + if (!r) return + for (const f of fs.readdirSync(dir)) { + const p = path.join(dir, f) + if (fs.statSync(p).isDirectory()) { downscaleAll(p); continue } // per-platform subdirs + if (/\.png$/i.test(f)) { try { execFileSync(r.bin, r.args(p), { stdio: 'ignore' }) } catch {} } + } } diff --git a/action/cli/lib/device.mjs b/action/cli/lib/device.mjs new file mode 100644 index 0000000..a937e75 --- /dev/null +++ b/action/cli/lib/device.mjs @@ -0,0 +1,152 @@ +// Platform-agnostic session: Metro, the dev-client connect loop, and the +// capture helpers, driving whichever device driver the run asked for. +// +// The two drivers (lib/sim.mjs, lib/android.mjs) expose the same primitives; +// everything that is genuinely shared — starting Metro, nudging the dev client +// onto it, waiting for the first bundle, the diagnostics dump on failure — +// lives here so neither platform can quietly drift from the other. +import { spawn } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' +import { sh, sleep, log } from './util.mjs' +import { ocrAvailable } from './ocr.mjs' +import * as ios from './sim.mjs' +import * as android from './android.mjs' + +export const DRIVERS = { ios, android } +export const PLATFORMS = Object.keys(DRIVERS) + +export function driverFor(platform) { + const d = DRIVERS[platform] + if (!d) throw new Error(`unknown platform "${platform}" — expected ${PLATFORMS.join(' | ')}`) + return d +} + +// Capture on whichever device a session is bound to. replay.mjs takes one of +// these rather than a bare id, so a flow replays identically on either platform. +export const screenshot = (device, outPath) => device.driver.screenshot(device.id, outPath) + +// Metro in the background. Resolves `ready` when the server listens, and +// exposes `bundled` (first successful bundle) for the caller to await after +// launching the app. +export function startMetro(projectDir, port = 8081) { + const cli = fs.existsSync(path.join(projectDir, 'node_modules', 'expo', 'bin', 'cli')) + ? [path.join(projectDir, 'node_modules', 'expo', 'bin', 'cli'), 'start', '--port', String(port)] + : null + if (!cli) throw new Error('expo not installed in project (node_modules/expo missing)') + const proc = spawn('node', cli, { cwd: projectDir, env: { ...process.env, CI: '1', EXPO_NO_TELEMETRY: '1' }, stdio: ['ignore', 'pipe', 'pipe'] }) + let out = '' + let readyRes, bundledRes + const ready = new Promise((r) => (readyRes = r)) + const bundled = new Promise((r) => (bundledRes = r)) + const onData = (d) => { + const s = d.toString() + out += s + if (/Waiting on http:\/\/localhost:\d+/.test(out)) readyRes(true) + if (/Bundled\s|Bundling complete|\d+% \(\d+\/\d+\)/.test(out) && /Bundled\s|Bundling complete/.test(out)) bundledRes(true) + if (/(^|\n)\s*(error|Error|ERROR)/.test(s)) log('metro:', s.trim().slice(0, 300)) + } + proc.stdout.on('data', onData) + proc.stderr.on('data', onData) + proc.on('exit', (code) => { log(`metro exited (${code})`); readyRes(false); bundledRes(false) }) + const stop = () => { try { proc.kill('SIGTERM') } catch {} } + return { proc, ready, bundled, stop, output: () => out } +} + +export async function waitFor(promise, ms, label) { + const t = await Promise.race([promise, sleep(ms).then(() => 'timeout')]) + if (t === 'timeout') throw new Error(`timed out waiting for ${label} (${ms}ms)`) + return t +} + +// Boot the whole stack: device, app, Metro, first bundle. Returns a session +// with capture helpers; call session.close() at the end. +export async function openSession({ projectDir, config, scheme, platform = 'ios' }) { + const driver = driverFor(platform) + const { id, name } = await driver.ensureBooted(config) + driver.freezeStatusBar(id) + const appPath = config.appPath ?? driver.findBuiltApp(projectDir) + if (!appPath) { + throw new Error(platform === 'android' + ? 'no built dev client found under android/app/build/outputs/apk — build it first (expo run:android --no-bundler) or pass app_path' + : 'no built dev client found under ios/build — build it first (expo run:ios --no-bundler)') + } + const appId = config.appId ?? driver.appIdOf(appPath) + driver.installApp(id, appPath) + driver.grantPrivacy(id, appId) + driver.muteDevMenu(id, appId) + driver.approveScheme(id, scheme, appId) + await sleep(3000) // let the launcher settle (iOS resprings SpringBoard above) + // resolve the OCR backend now — compiling the Vision helper lazily inside the + // connect loop starves a small runner while Metro bundles, and simctl openurl + // then times out + ocrAvailable() + const metro = startMetro(projectDir, config.metroPort) + await waitFor(metro.ready, 120000, 'Metro to start') + // the emulator's localhost is not the host's — open the tunnel before the + // dev client is ever pointed at Metro (no-op on iOS) + driver.connectMetro(id, config.metroPort) + driver.terminate(id, appId) + await sleep(800) + driver.launch(id, appId) + await sleep(3000) + // the dev client opens on its launcher; a deep link routes it to Metro. + // Re-nudge every 15s — a cold device sometimes swallows the first one. + // expo-dev-client's connect URL loads a specific Metro without a tap. + const connectUrl = `${scheme}://expo-development-client/?url=${encodeURIComponent(`http://localhost:${config.metroPort}`)}` + const deadline = Date.now() + 300000 + let bundledOk = false + let nudges = 0 + while (Date.now() < deadline) { + // after a few foreground nudges, cold-start into the link instead: opening + // a URL on a terminated app launches it straight into the deep link, + // skipping any launcher race + if (nudges > 0 && nudges % 3 === 0) { try { driver.terminate(id, appId) } catch {}; await sleep(800) } + // opening a URL can time out (POSIX 60) when the device is under load — a + // missed nudge, not a fatal error + try { driver.openUrl(id, connectUrl, appId) } catch (e) { log('openurl nudge failed:', e.message.split('\n')[0]) } + nudges++ + const r = await Promise.race([metro.bundled, sleep(15000).then(() => 'tick')]) + if (r === true) { bundledOk = true; break } + if (r === false) break + log('waiting for the first JS bundle…') + try { driver.nudgeOpenPrompt(id, projectDir) } catch {} + } + if (!bundledOk) { + log('metro tail:\n' + metro.output().split('\n').slice(-25).join('\n')) + try { + const diagDir = path.join(projectDir, '.screenmap', 'out', 'ci', 'diag', platform) + driver.diagnostics(id, diagDir) + fs.writeFileSync(path.join(diagDir, 'metro.log'), metro.output()) + let status = 'curl failed' + try { status = sh('curl', ['-s', '-m', '5', `http://localhost:${config.metroPort}/status`]) } catch {} + fs.writeFileSync(path.join(diagDir, 'metro-status.txt'), status) + log('connect diagnostics written to', diagDir) + } catch (e) { log('diagnostics failed:', e.message) } + metro.stop() + throw new Error(`timed out waiting for first JS bundle (${platform})`) + } + await sleep(config.waits.boot) + log(`session ready: ${appId} on ${name} (${id}), Metro :${config.metroPort}`) + let firstVisit = true + const session = { + platform, driver, id, udid: id, appId, bundleId: appId, scheme, config, + deviceName: name ?? config.device, + screenshot(outPath) { return driver.screenshot(id, outPath) }, + async visit(url, outPath, waitMs) { + try { driver.openUrl(id, url, appId) } catch { await sleep(2000); driver.openUrl(id, url, appId) } // one retry for transient timeouts + await sleep(waitMs ?? config.waits.transition) + // dev builds often show a one-off toast right after the bundle loads; + // give the very first capture extra time to settle + if (firstVisit) { await sleep(config.waits.settle ?? 6000); firstVisit = false } + driver.screenshot(id, outPath) + return outPath + }, + async relaunch() { + driver.terminate(id, appId); await sleep(800); driver.launch(id, appId); await sleep(4000) + firstVisit = true // dev builds re-show their load-time toast after a relaunch + }, + close() { metro.stop() }, + } + return session +} diff --git a/action/cli/lib/eas.mjs b/action/cli/lib/eas.mjs index 74d04d0..ebff35f 100644 --- a/action/cli/lib/eas.mjs +++ b/action/cli/lib/eas.mjs @@ -1,10 +1,17 @@ -// EAS dev-client resolution: the Action owns no build pipeline. A simulator -// dev client comes from EAS — reuse the newest finished build whose -// fingerprint matches the checkout (JS-only changes never rebuild), otherwise -// trigger `eas build` on Expo's infrastructure and wait. Requires EXPO_TOKEN -// and an EAS-linked project (extra.eas.projectId) with a simulator profile in -// eas.json, e.g.: -// "development-simulator": { "developmentClient": true, "distribution": "internal", "ios": { "simulator": true } } +// EAS dev-client resolution: the Action owns no build pipeline. A dev client +// comes from EAS — reuse the newest finished build whose fingerprint matches +// the checkout (JS-only changes never rebuild), otherwise trigger `eas build` +// on Expo's infrastructure and wait. Requires EXPO_TOKEN and an EAS-linked +// project (extra.eas.projectId) with a profile per platform in eas.json: +// +// "development-simulator": { "developmentClient": true, "distribution": "internal", +// "ios": { "simulator": true } } +// "development-emulator": { "developmentClient": true, "distribution": "internal", +// "android": { "buildType": "apk" } } +// +// `buildType: "apk"` is not optional on Android: EAS defaults to an .aab, which +// an emulator cannot install, and the failure surfaces much later as a +// confusing adb error. import { spawnSync } from 'node:child_process' import fs from 'node:fs' import os from 'node:os' @@ -29,10 +36,12 @@ function parseJson(text, what) { return JSON.parse(text.slice(start, Math.max(text.lastIndexOf(']'), text.lastIndexOf('}')) + 1)) } -export function fingerprintOf(projectDir, eas) { +export const DEFAULT_PROFILES = { ios: 'development-simulator', android: 'development-emulator' } + +export function fingerprintOf(projectDir, eas, platform = 'ios') { // must be eas-cli's own computation — a bare @expo/fingerprint run hashes // differently from what EAS records on builds, so reuse would never match - const r = run(eas[0], [...eas.slice(1), 'fingerprint:generate', '--platform', 'ios', '--non-interactive', '--json'], { cwd: projectDir, env: process.env }) + const r = run(eas[0], [...eas.slice(1), 'fingerprint:generate', '--platform', platform, '--non-interactive', '--json'], { cwd: projectDir, env: process.env }) if (r.status !== 0) { log('fingerprint failed (will build fresh):', (r.stderr || r.stdout || '').slice(-300)); return null } try { return parseJson(r.stdout, 'fingerprint').hash ?? null } catch { return null } } @@ -46,8 +55,16 @@ function download(url, dest) { if (r.status !== 0) throw new Error(`download failed: ${(r.stderr || '').slice(-300)}`) } -function extractApp(archive, destDir) { +// iOS artifacts arrive as a tarball around a .app bundle; Android artifacts are +// the installable file itself, so there is nothing to unpack. +function extractApp(archive, destDir, platform) { ensureDir(destDir) + if (platform === 'android') { + if (/\.aab$/.test(archive)) throw new Error('EAS returned an .aab, which no emulator can install — set "android": { "buildType": "apk" } on the build profile') + const dest = path.join(destDir, path.basename(archive)) + fs.cpSync(archive, dest) + return dest + } if (/\.(tar\.gz|tgz)$/.test(archive)) { const r = run('tar', ['-xzf', archive, '-C', destDir]) if (r.status !== 0) throw new Error(`extract failed: ${(r.stderr || '').slice(-300)}`) @@ -59,40 +76,47 @@ function extractApp(archive, destDir) { return path.join(destDir, app) } +// The artifact URL keeps its own extension; naming the download after it is +// what lets extractApp tell an .apk from an .aab from a tarball. +function archiveName(url, platform) { + const ext = (url.split('?')[0].match(/\.(apk|aab|tar\.gz|tgz|zip)$/i) ?? [])[0] + return `client${ext ?? (platform === 'android' ? '.apk' : '.tar.gz')}` +} + // Returns { appPath, reused, fingerprint, buildId }. Reuse is best-effort: // any step of the fingerprint match failing falls through to a fresh build. -export function resolveApp({ projectDir, profile, workDir }) { +export function resolveApp({ projectDir, profile, workDir, platform = 'ios' }) { if (!process.env.EXPO_TOKEN) throw new Error('EXPO_TOKEN not set — pass expo_token (or provide app_path / a prebuilt client)') const eas = easCommand() const cwdOpts = { cwd: projectDir, env: process.env } const dest = ensureDir(workDir) - const fingerprint = fingerprintOf(projectDir, eas) + const fingerprint = fingerprintOf(projectDir, eas, platform) if (fingerprint) { - const r = run(eas[0], [...eas.slice(1), 'build:list', '--platform', 'ios', '--status', 'finished', + const r = run(eas[0], [...eas.slice(1), 'build:list', '--platform', platform, '--status', 'finished', '--build-profile', profile, '--fingerprint-hash', fingerprint, '--limit', '1', '--json', '--non-interactive'], cwdOpts) if (r.status === 0) { try { const hit = parseJson(r.stdout, 'build:list')[0] const url = artifactUrl(hit) if (url) { - log(`EAS: reusing build ${hit.id} (fingerprint ${fingerprint.slice(0, 12)})`) - const archive = path.join(dest, 'client.tar.gz') + log(`EAS: reusing ${platform} build ${hit.id} (fingerprint ${fingerprint.slice(0, 12)})`) + const archive = path.join(dest, archiveName(url, platform)) download(url, archive) - return { appPath: extractApp(archive, path.join(dest, 'client')), reused: true, fingerprint, buildId: hit.id } + return { appPath: extractApp(archive, path.join(dest, 'client'), platform), reused: true, fingerprint, buildId: hit.id, platform } } } catch (e) { log('EAS reuse lookup failed (will build fresh):', e.message) } } else log('EAS build:list failed (will build fresh):', (r.stderr || r.stdout || '').slice(-300)) } - log(`EAS: no reusable build — building profile "${profile}" (this runs on EAS, not this runner)`) - const b = run(eas[0], [...eas.slice(1), 'build', '--platform', 'ios', '--profile', profile, + log(`EAS: no reusable ${platform} build — building profile "${profile}" (this runs on EAS, not this runner)`) + const b = run(eas[0], [...eas.slice(1), 'build', '--platform', platform, '--profile', profile, '--non-interactive', '--json', '--wait'], cwdOpts) if (b.status !== 0) throw new Error(`eas build failed: ${(b.stderr || b.stdout || '').slice(-800)}`) const build = [].concat(parseJson(b.stdout, 'build'))[0] const url = artifactUrl(build) if (!url) throw new Error(`eas build finished without an artifact URL (status ${build?.status})`) - const archive = path.join(dest, 'client.tar.gz') + const archive = path.join(dest, archiveName(url, platform)) download(url, archive) - return { appPath: extractApp(archive, path.join(dest, 'client')), reused: false, fingerprint, buildId: build.id } + return { appPath: extractApp(archive, path.join(dest, 'client'), platform), reused: false, fingerprint, buildId: build.id, platform } } diff --git a/action/cli/lib/ocr.mjs b/action/cli/lib/ocr.mjs index 38b1c17..77e3143 100644 --- a/action/cli/lib/ocr.mjs +++ b/action/cli/lib/ocr.mjs @@ -1,6 +1,17 @@ -// Screen text via Apple Vision (see native/ocr.swift). Compiled once into a -// cache dir; ~0.5s per capture afterwards. Used to decide whether a replayed -// flow landed on the screen it claims, and to spot system alerts. +// Screen text, behind one interface with two backends. +// +// vision Apple Vision via native/ocr.swift, compiled once into a cache +// dir; ~0.5s per capture afterwards. macOS only. +// tesseract the `tesseract` binary. The Linux lane: Android captures run on +// ubuntu runners, where there is no Vision, and without OCR the +// landing checks, deep-link verification and system-alert +// dismissal all go dark. +// +// Both emit the SAME item shape — `{ text, x, y, w, h }` with normalized +// coordinates and Vision's **bottom-left** origin — because callers convert to +// tap coordinates with `y = 1 - (y + h / 2)`. Tesseract reports pixels from +// the top-left, so the adapter below does the flip; getting this wrong taps +// the mirror image of the button. import { execFileSync, spawnSync } from 'node:child_process' import fs from 'node:fs' import os from 'node:os' @@ -8,26 +19,119 @@ import path from 'node:path' import { log } from './util.mjs' const SRC = new URL('./native/ocr.swift', import.meta.url).pathname -let binPath = null -export function ocrAvailable() { - if (binPath) return true - if (process.platform !== 'darwin') return false + +// SCREENMAP_OCR forces a backend: vision | tesseract | off. Mostly for testing +// the Linux path from a Mac — otherwise the best available backend wins. +const forced = () => (process.env.SCREENMAP_OCR || '').toLowerCase() || null + +let visionBin +function visionAvailable() { + if (visionBin !== undefined) return !!visionBin + if (process.platform !== 'darwin' || !fs.existsSync(SRC)) return !(visionBin = null) const dir = path.join(os.homedir(), '.cache', 'screenmap-ci') fs.mkdirSync(dir, { recursive: true }) const bin = path.join(dir, `ocr-${fs.statSync(SRC).size}`) if (!fs.existsSync(bin)) { log('compiling Vision OCR helper (one-time)…') const r = spawnSync('swiftc', ['-O', '-o', bin, SRC], { encoding: 'utf8' }) - if (r.status !== 0) { log('swiftc failed:', (r.stderr || '').slice(-300)); return false } + if (r.status !== 0) { log('swiftc failed:', (r.stderr || '').slice(-300)); return !(visionBin = null) } } - binPath = bin + visionBin = bin return true } +let tessBin +function tesseractAvailable() { + if (tessBin !== undefined) return !!tessBin + const r = spawnSync('tesseract', ['--version'], { encoding: 'utf8' }) + return !!(tessBin = r.status === 0 ? 'tesseract' : null) +} + +// Which backend a call would use, or null. Resolving this also does the +// one-time compile, so callers can pay that cost at a moment of their choosing +// (openSession does, so it does not land mid-way through the Metro connect +// loop and starve a small runner). +export function ocrBackend() { + const f = forced() + if (f === 'off') return null + if (f === 'vision') return visionAvailable() ? 'vision' : null + if (f === 'tesseract') return tesseractAvailable() ? 'tesseract' : null + if (visionAvailable()) return 'vision' + if (tesseractAvailable()) return 'tesseract' + return null +} +export const ocrAvailable = () => ocrBackend() !== null + +// PNG dimensions from the IHDR chunk — tesseract's TSV reports pixels and says +// nothing about the page size, and this beats shelling out to `sips`/`identify` +// for a value that sits in the first 24 bytes of the file. +function pngSize(file) { + const fd = fs.openSync(file, 'r') + try { + const b = Buffer.alloc(24) + if (fs.readSync(fd, b, 0, 24, 0) < 24) return null + if (b.toString('latin1', 1, 4) !== 'PNG' || b.toString('latin1', 12, 16) !== 'IHDR') return null + return { w: b.readUInt32BE(16), h: b.readUInt32BE(20) } + } catch { return null } finally { fs.closeSync(fd) } +} + +function ocrVision(png) { + return execFileSync(visionBin, [png], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }) + .split('\n').filter(Boolean) + .map((l) => { try { return JSON.parse(l) } catch { return null } }) + .filter(Boolean) +} + +// Tesseract TSV is one row per word; Vision reports whole strings, and the +// callers' word-set comparisons and "tap the button labelled X" lookups both +// assume that granularity. So group words back into their source line +// (block/par/line triple) and union their boxes. +function ocrTesseract(png) { + const size = pngSize(png) + if (!size) return [] + // psm 11 (sparse text) reads scattered UI labels far better than the default + // page-layout mode, which expects paragraphs of prose. + const r = spawnSync('tesseract', [png, 'stdout', '--psm', '11', '-c', 'tessedit_create_tsv=1'], + { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }) + if (r.status !== 0) { log('tesseract failed:', (r.stderr || '').trim().slice(-300)); return [] } + const lines = (r.stdout || '').split('\n') + const head = lines[0]?.split('\t') ?? [] + const col = Object.fromEntries(head.map((h, i) => [h.trim(), i])) + if (col.text === undefined || col.left === undefined) return [] + const groups = new Map() + for (const raw of lines.slice(1)) { + const f = raw.split('\t') + if (f.length < head.length) continue + if (Number(f[col.level]) !== 5) continue // 5 = word + const text = (f[col.text] ?? '').trim() + if (!text) continue + if (Number(f[col.conf]) < 40) continue // low-confidence words are usually icon noise + const key = `${f[col.block_num]}/${f[col.par_num]}/${f[col.line_num]}` + const left = Number(f[col.left]), top = Number(f[col.top]) + const width = Number(f[col.width]), height = Number(f[col.height]) + const g = groups.get(key) + if (!g) groups.set(key, { words: [text], x0: left, y0: top, x1: left + width, y1: top + height }) + else { + g.words.push(text) + g.x0 = Math.min(g.x0, left); g.y0 = Math.min(g.y0, top) + g.x1 = Math.max(g.x1, left + width); g.y1 = Math.max(g.y1, top + height) + } + } + return [...groups.values()].map((g) => ({ + text: g.words.join(' '), + x: g.x0 / size.w, + // top-left pixels → Vision's bottom-left normalized origin + y: 1 - g.y1 / size.h, + w: (g.x1 - g.x0) / size.w, + h: (g.y1 - g.y0) / size.h, + })) +} + export function ocr(pngPath) { - if (!ocrAvailable()) return [] + const backend = ocrBackend() + if (!backend) return [] try { - return execFileSync(binPath, [pngPath], { encoding: 'utf8' }).split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l) } catch { return null } }).filter(Boolean) + return backend === 'vision' ? ocrVision(pngPath) : ocrTesseract(pngPath) } catch { return [] } } @@ -43,12 +147,14 @@ export function words(items) { export const jaccard = (a, b) => { if (!a.size && !b.size) return 1; let i = 0; for (const w of a) if (b.has(w)) i++; return i / (a.size + b.size - i) } export const containment = (needles, hay) => { if (!needles.size) return null; let i = 0; for (const w of needles) if (hay.has(w)) i++; return i / needles.size } -const ALERT_HINTS = [/would like/i, /don['’]t allow/i, /^allow$/i, /allow while using/i, /allow once/i, /^not now$/i, /turn on/i, /^ok$/i] +const ALERT_HINTS = [/would like/i, /don['’]t allow/i, /^allow$/i, /allow while using/i, /allow once/i, /^not now$/i, /turn on/i, /^ok$/i, + // Android's runtime permission dialog wording + /while using the app/i, /only this time/i, /^deny$/i, /don['’]t allow/i] // a system permission/alert is up if several of its tell-tale strings are visible export function alertButtons(items) { const texts = items.map((i) => i.text.trim()) const hits = texts.filter((t) => ALERT_HINTS.some((re) => re.test(t))) if (hits.length < 2) return [] - const order = [/don['’]t allow/i, /^not now$/i, /^ok$/i, /allow once/i, /allow while using/i, /^allow$/i, /limit access/i] + const order = [/don['’]t allow/i, /^deny$/i, /^not now$/i, /^ok$/i, /only this time/i, /allow once/i, /while using the app/i, /allow while using/i, /^allow$/i, /limit access/i] return items.filter((i) => order.some((re) => re.test(i.text.trim()))).sort((a, b) => order.findIndex((re) => re.test(a.text)) - order.findIndex((re) => re.test(b.text))) } diff --git a/action/cli/lib/replay.mjs b/action/cli/lib/replay.mjs index d427dff..e388e96 100644 --- a/action/cli/lib/replay.mjs +++ b/action/cli/lib/replay.mjs @@ -12,7 +12,7 @@ import fs from 'node:fs' import path from 'node:path' import YAML from 'yaml' -import { screenshot } from './sim.mjs' +import { screenshot } from './device.mjs' import { argentFlow, argentRun } from './argent.mjs' import { ocr, words, jaccard, containment, alertButtons } from './ocr.mjs' import { readJson, ensureDir, log, sleep } from './util.mjs' @@ -39,16 +39,16 @@ export function loadFlows(projectDir, dirs) { return byRoute } -function runFragment(steps, name, tmpDir, udid) { +function runFragment(steps, name, tmpDir, device) { const file = path.join(tmpDir, `${name}.yaml`) fs.writeFileSync(file, YAML.stringify({ steps })) - const r = argentFlow(file, udid) + const r = argentFlow(file, device.id) if (!r.ok) log(`argent fragment ${name} failed:`, r.raw.trim().slice(-400)) return r.ok } // Replays one flow and writes its captures into outDir. -export async function replayFlow(rec, { udid, outDir, tmpDir }) { +export async function replayFlow(rec, { device, outDir, tmpDir }) { const doc = YAML.parse(fs.readFileSync(rec.yaml, 'utf8')) const steps = doc?.steps ?? [] const captures = Object.entries(rec.meta.steps ?? {}).filter(([, s]) => s.capture).map(([i, s]) => ({ after: Number(i), file: s.capture })).sort((a, b) => a.after - b.after) @@ -57,26 +57,26 @@ export async function replayFlow(rec, { udid, outDir, tmpDir }) { let cursor = 0, seg = 0 for (const cap of captures) { const frag = steps.slice(cursor, cap.after + 1) - if (frag.length && !runFragment(frag, `${rec.name}-${seg++}`, tmpDir, udid)) return { ok: false, written } + if (frag.length && !runFragment(frag, `${rec.name}-${seg++}`, tmpDir, device)) return { ok: false, written } await sleep(400) - screenshot(udid, path.join(outDir, cap.file)) + screenshot(device, path.join(outDir, cap.file)) written.push(cap.file) cursor = cap.after + 1 } - if (cursor < steps.length && !runFragment(steps.slice(cursor), `${rec.name}-${seg++}`, tmpDir, udid)) return { ok: false, written } + if (cursor < steps.length && !runFragment(steps.slice(cursor), `${rec.name}-${seg++}`, tmpDir, device)) return { ok: false, written } return { ok: true, written } } // If a system alert is on screen, tap its most conservative button (Don't // Allow / Not Now / OK) and return true. -export function dismissAlert(udid, shotPath) { +export function dismissAlert(device, shotPath) { const items = ocr(shotPath) const buttons = alertButtons(items) if (!buttons.length) return false const b = buttons[0] // Vision boxes: normalized, origin bottom-left → argent taps: origin top-left const x = b.x + b.w / 2, y = 1 - (b.y + b.h / 2) - const r = argentRun('gesture-tap', { udid, x: x.toFixed(4), y: y.toFixed(4) }) + const r = argentRun('gesture-tap', { udid: device.id, x: x.toFixed(4), y: y.toFixed(4) }) log(`dismissed system alert via "${b.text}" (${r.ok ? 'ok' : 'tap failed'})`) return r.ok } @@ -127,10 +127,10 @@ export async function verifyDeepLink({ shot, rec, probeBogus }) { return { ok: true, method: 'unverified', score: null } } -export async function verifyLanding({ shot, rec, udid, probe }) { +export async function verifyLanding({ shot, rec, device, probe }) { // probe(): async () => path of a fresh deep-link capture of the same route (only used as fallback) let items = ocr(shot) - if (alertButtons(items).length && dismissAlert(udid, shot)) { await sleep(600); screenshot(udid, shot); items = ocr(shot) } + if (alertButtons(items).length && dismissAlert(device, shot)) { await sleep(600); screenshot(device, shot); items = ocr(shot) } const seen = words(items) const marks = landmarksOf(rec.meta) if (marks.size >= 2) { diff --git a/action/cli/lib/sim.mjs b/action/cli/lib/sim.mjs index c4e48c6..7b49db1 100644 --- a/action/cli/lib/sim.mjs +++ b/action/cli/lib/sim.mjs @@ -1,16 +1,21 @@ -// iOS simulator + Metro driver. Everything here is `xcrun simctl` and a -// background Metro process — no MCP, no LLM, works on a macOS runner. -import { spawn } from 'node:child_process' +// iOS simulator driver — the `xcrun simctl` half of the device layer, matching +// lib/android.mjs's interface. No MCP, no LLM, works on a macOS runner. +// +// The shared session orchestration (Metro, the connect loop, capture helpers) +// lives in lib/device.mjs; this file is only the iOS-specific primitives. import fs from 'node:fs' import path from 'node:path' -import { sh, shOk, sleep, log } from './util.mjs' +import { sh, shOk, log } from './util.mjs' import { argentAvailable, argentRun, grantPermissions } from './argent.mjs' import { ocr, ocrAvailable } from './ocr.mjs' +export const platform = 'ios' +export const label = 'iOS simulator' + // iOS ≥18.3 gates simctl openurl for a custom scheme behind an // "Open in …?" prompt. Pre-approving the scheme in LaunchServices skips it // (the Detox/Maestro technique); harmless on versions without the prompt. -function approveScheme(udid, scheme, bundleId) { +export function approveScheme(udid, scheme, bundleId) { shOk('xcrun', ['simctl', 'spawn', udid, 'defaults', 'write', 'com.apple.launchservices.schemeapproval', `com.apple.CoreSimulator.CoreSimulatorBridge-->${scheme}`, '-string', bundleId]) // SpringBoard caches approvals; respring so the write takes effect now @@ -22,7 +27,7 @@ function approveScheme(udid, scheme, bundleId) { // standard UserDefaults (see DevMenuPreferences.swift), so mark onboarding done // before the first launch — the runtime equivalent of the // EXDevMenuIsOnboardingFinished Info.plist flag, without rebuilding the client. -function muteDevMenu(udid, bundleId) { +export function muteDevMenu(udid, bundleId) { shOk('xcrun', ['simctl', 'spawn', udid, 'defaults', 'write', bundleId, 'EXDevMenuIsOnboardingFinished', '-bool', 'true']) shOk('xcrun', ['simctl', 'spawn', udid, 'defaults', 'write', bundleId, 'EXDevMenuShowsAtLaunch', '-bool', 'false']) // expo-dev-menu 57 added a floating gear that defaults to on and lands in the @@ -31,7 +36,7 @@ function muteDevMenu(udid, bundleId) { } // Belt-and-braces for the same prompt: OCR the screen and tap "Open". -function tapOpenPrompt(udid, projectDir) { +export function nudgeOpenPrompt(udid, projectDir) { if (!ocrAvailable()) return false const shot = path.join(projectDir, '.screenmap', 'out', 'ci', 'open-prompt.png') fs.mkdirSync(path.dirname(shot), { recursive: true }) @@ -49,20 +54,20 @@ function tapOpenPrompt(udid, projectDir) { export function listBooted() { const j = JSON.parse(sh('xcrun', ['simctl', 'list', 'devices', 'booted', '-j'])) - return Object.values(j.devices).flat().filter((d) => d.state === 'Booted') + return Object.values(j.devices).flat().filter((d) => d.state === 'Booted').map((d) => ({ id: d.udid, name: d.name })) } -export async function ensureBooted(deviceName) { +export async function ensureBooted(config) { const booted = listBooted() - if (booted.length) { log(`simulator already booted: ${booted[0].name} (${booted[0].udid})`); return booted[0].udid } + if (booted.length) { log(`simulator already booted: ${booted[0].name} (${booted[0].id})`); return booted[0] } const j = JSON.parse(sh('xcrun', ['simctl', 'list', 'devices', 'available', '-j'])) const all = Object.values(j.devices).flat() - const pick = all.find((d) => d.name === deviceName) ?? all.find((d) => /iPhone/.test(d.name)) - if (!pick) throw new Error(`no available simulator (wanted "${deviceName}")`) + const pick = all.find((d) => d.name === config.device) ?? all.find((d) => /iPhone/.test(d.name)) + if (!pick) throw new Error(`no available simulator (wanted "${config.device}")`) log(`booting ${pick.name} (${pick.udid})`) sh('xcrun', ['simctl', 'boot', pick.udid]) sh('xcrun', ['simctl', 'bootstatus', pick.udid, '-b']) - return pick.udid + return { id: pick.udid, name: pick.name } } // presentation mode: identical clock/battery/signal on every capture, so @@ -85,7 +90,7 @@ export function findBuiltApp(projectDir) { return null } -export function bundleIdOf(appPath) { +export function appIdOf(appPath) { return sh('defaults', ['read', path.join(appPath, 'Info'), 'CFBundleIdentifier']) } @@ -100,6 +105,7 @@ export function screenshot(udid, outPath) { fs.mkdirSync(path.dirname(outPath), { recursive: true }) freezeStatusBar(udid) // tooling in between (argent) can clear the override sh('xcrun', ['simctl', 'io', udid, 'screenshot', outPath]) + return outPath } // pre-grant privacy so a mis-tap can never summon a system permission dialog @@ -109,118 +115,12 @@ export function grantPrivacy(udid, bundleId) { if (argentAvailable()) { const g = grantPermissions(udid, bundleId); if (g.length) log(`pre-granted: ${g.join(', ')}`) } } -// Metro in the background. Resolves `ready` when the server listens, and -// exposes `bundled` (first successful bundle) for the caller to await after -// launching the app. -export function startMetro(projectDir, port = 8081) { - const cli = fs.existsSync(path.join(projectDir, 'node_modules', 'expo', 'bin', 'cli')) - ? [path.join(projectDir, 'node_modules', 'expo', 'bin', 'cli'), 'start', '--port', String(port)] - : null - if (!cli) throw new Error('expo not installed in project (node_modules/expo missing)') - const proc = spawn('node', cli, { cwd: projectDir, env: { ...process.env, CI: '1', EXPO_NO_TELEMETRY: '1' }, stdio: ['ignore', 'pipe', 'pipe'] }) - let out = '' - let readyRes, bundledRes - const ready = new Promise((r) => (readyRes = r)) - const bundled = new Promise((r) => (bundledRes = r)) - const onData = (d) => { - const s = d.toString() - out += s - if (/Waiting on http:\/\/localhost:\d+/.test(out)) readyRes(true) - if (/Bundled\s|Bundling complete|\d+% \(\d+\/\d+\)/.test(out) && /Bundled\s|Bundling complete/.test(out)) bundledRes(true) - if (/(^|\n)\s*(error|Error|ERROR)/.test(s)) log('metro:', s.trim().slice(0, 300)) - } - proc.stdout.on('data', onData) - proc.stderr.on('data', onData) - proc.on('exit', (code) => { log(`metro exited (${code})`); readyRes(false); bundledRes(false) }) - const stop = () => { try { proc.kill('SIGTERM') } catch {} } - return { proc, ready, bundled, stop, output: () => out } -} - -export async function waitFor(promise, ms, label) { - const t = await Promise.race([promise, sleep(ms).then(() => 'timeout')]) - if (t === 'timeout') throw new Error(`timed out waiting for ${label} (${ms}ms)`) - return t -} +// The simulator shares the host's network stack, so Metro on localhost is +// already reachable — nothing to tunnel. +export function connectMetro() { return true } -// Boot the whole stack: simulator, app, Metro, first bundle. Returns a -// session with capture helpers; call session.close() at the end. -export async function openSession({ projectDir, config, scheme }) { - const udid = await ensureBooted(config.device) - freezeStatusBar(udid) - const appPath = config.appPath ?? findBuiltApp(projectDir) - if (!appPath) throw new Error('no built dev client found under ios/build — build it first (expo run:ios --no-bundler)') - const bundleId = config.bundleId ?? bundleIdOf(appPath) - installApp(udid, appPath) - grantPrivacy(udid, bundleId) - muteDevMenu(udid, bundleId) - approveScheme(udid, scheme, bundleId) - await sleep(3000) // let SpringBoard settle after the respring - // compile the OCR helper now — doing it lazily inside the connect loop - // starves a small runner while Metro bundles, and simctl openurl times out - ocrAvailable() - const metro = startMetro(projectDir, config.metroPort) - await waitFor(metro.ready, 120000, 'Metro to start') - terminate(udid, bundleId) - await sleep(800) - launch(udid, bundleId) - await sleep(3000) - // the dev client opens on its launcher; a deep link routes it to Metro. - // Re-nudge every 15s — a cold simulator sometimes swallows the first one. - // expo-dev-client's connect URL loads a specific Metro without a tap. - const connectUrl = `${scheme}://expo-development-client/?url=${encodeURIComponent(`http://localhost:${config.metroPort}`)}` - const deadline = Date.now() + 300000 - let bundledOk = false - let nudges = 0 - while (Date.now() < deadline) { - // after a few foreground nudges, cold-start into the link instead: openurl - // on a terminated app launches it straight into the deep link, skipping - // any launcher race - if (nudges > 0 && nudges % 3 === 0) { try { terminate(udid, bundleId) } catch {}; await sleep(800) } - // openurl can time out (POSIX 60) when the sim is under load — a missed - // nudge, not a fatal error - try { openUrl(udid, connectUrl) } catch (e) { log('openurl nudge failed:', e.message.split('\n')[0]) } - nudges++ - const r = await Promise.race([metro.bundled, sleep(15000).then(() => 'tick')]) - if (r === true) { bundledOk = true; break } - if (r === false) break - log('waiting for the first JS bundle…') - try { tapOpenPrompt(udid, projectDir) } catch {} - } - if (!bundledOk) { - log('metro tail:\n' + metro.output().split('\n').slice(-25).join('\n')) - try { - const diagDir = path.join(projectDir, '.screenmap', 'out', 'ci', 'diag') - fs.mkdirSync(diagDir, { recursive: true }) - sh('xcrun', ['simctl', 'io', udid, 'screenshot', path.join(diagDir, 'connect-timeout.png')]) - fs.writeFileSync(path.join(diagDir, 'metro.log'), metro.output()) - fs.writeFileSync(path.join(diagDir, 'listapps.txt'), sh('xcrun', ['simctl', 'listapps', udid])) - let status = 'curl failed' - try { status = sh('curl', ['-s', '-m', '5', `http://localhost:${config.metroPort}/status`]) } catch {} - fs.writeFileSync(path.join(diagDir, 'metro-status.txt'), status) - log('connect diagnostics written to', diagDir) - } catch (e) { log('diagnostics failed:', e.message) } - metro.stop() - throw new Error('timed out waiting for first JS bundle') - } - await sleep(config.waits.boot) - log(`session ready: ${bundleId} on ${udid}, Metro :${config.metroPort}`) - const deviceName = listBooted().find((d) => d.udid === udid)?.name ?? config.device - let firstVisit = true - return { - udid, bundleId, scheme, config, deviceName, - async visit(url, outPath, waitMs) { - try { openUrl(udid, url) } catch { await sleep(2000); openUrl(udid, url) } // one retry for transient simctl timeouts - await sleep(waitMs ?? config.waits.transition) - // dev builds often show a one-off toast right after the bundle loads; - // give the very first capture extra time to settle - if (firstVisit) { await sleep(config.waits.settle ?? 6000); firstVisit = false } - screenshot(udid, outPath) - return outPath - }, - async relaunch() { - terminate(udid, bundleId); await sleep(800); launch(udid, bundleId); await sleep(4000) - firstVisit = true // dev builds re-show their load-time toast after a relaunch - }, - close() { metro.stop() }, - } +export function diagnostics(udid, dir) { + fs.mkdirSync(dir, { recursive: true }) + try { sh('xcrun', ['simctl', 'io', udid, 'screenshot', path.join(dir, 'connect-timeout.png')]) } catch {} + try { fs.writeFileSync(path.join(dir, 'listapps.txt'), sh('xcrun', ['simctl', 'listapps', udid])) } catch {} } diff --git a/action/cli/lib/util.mjs b/action/cli/lib/util.mjs index 0e51b41..186d7ba 100644 --- a/action/cli/lib/util.mjs +++ b/action/cli/lib/util.mjs @@ -74,9 +74,23 @@ function agentKeyPresent(user) { return !!(process.env[keyEnv] || process.env.AGENT_API_KEY) } +// Every platform a run can capture on. Order matters: it is the order captures +// happen in, and the first entry is the one single-platform consumers (older +// viewers, the PR comment image) see as the map's device. +export const PLATFORMS = ['ios', 'android'] + +// Per-platform device defaults. Android's is null on purpose — there is no +// equivalent of "iPhone 16 Pro is always there", so the driver takes whatever +// device is attached or the first defined AVD, and names it in the summary. +const PLATFORM_DEFAULTS = { + ios: { device: 'iPhone 16 Pro', appId: null, appPath: null }, + android: { device: null, appId: null, appPath: null }, +} + export function loadConfig(projectDir) { const base = { - scheme: null, bundleId: null, appPath: null, device: 'iPhone 16 Pro', metroPort: 8081, + scheme: null, bundleId: null, appPath: null, device: null, metroPort: 8081, + platforms: ['ios'], waits: { transition: 2500, network: 6000, boot: 15000 }, suspects: { broadCap: 8 }, agent: { enabled: true, model: null, provider: null, command: null, keyEnv: null }, @@ -102,10 +116,37 @@ export function loadConfig(projectDir) { agent: { ...defaults.agent, ...(user.agent ?? {}) }, } if (process.env.AGENT_MAX_SCREENS) merged.agent.maxScreens = Number(process.env.AGENT_MAX_SCREENS) || merged.agent.maxScreens - if (process.env.SCREENMAP_APP_PATH) merged.appPath = process.env.SCREENMAP_APP_PATH // a prebuilt client (e.g. from EAS) beats ios/build discovery + + // platforms: env wins, then config.platforms, then the legacy single-platform + // default. An unknown name is a typo worth failing on rather than silently + // capturing nothing. + const wanted = (process.env.SCREENMAP_PLATFORMS || '').trim() + ? process.env.SCREENMAP_PLATFORMS.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean) + : [].concat(user.platforms ?? base.platforms) + for (const p of wanted) if (!PLATFORMS.includes(p)) throw new Error(`unknown platform "${p}" — expected ${PLATFORMS.join(' | ')}`) + merged.platforms = PLATFORMS.filter((p) => wanted.includes(p)) // canonical order, deduped + + // Per-platform blocks, with the pre-multi-platform top-level keys still + // meaning what they always meant: iOS. + for (const p of PLATFORMS) { + const legacy = p === 'ios' ? { device: user.device, appId: user.bundleId ?? user.appId, appPath: user.appPath } : {} + const defined = { ...PLATFORM_DEFAULTS[p], ...Object.fromEntries(Object.entries(legacy).filter(([, v]) => v != null)), ...(user[p] ?? {}) } + // an Android block may name the package as packageName; it is the same field + if (defined.packageName && !defined.appId) defined.appId = defined.packageName + const envPath = process.env[`SCREENMAP_APP_PATH_${p.toUpperCase()}`] || (merged.platforms.length === 1 ? process.env.SCREENMAP_APP_PATH : null) + if (envPath) defined.appPath = envPath // a prebuilt client (e.g. from EAS) beats on-disk discovery + merged[p] = defined + } return merged } +// The flattened view one platform's session needs: shared knobs (waits, params, +// agent, scheme) with that platform's device/app fields resolved on top. +export function platformConfig(config, platform) { + const p = config[platform] ?? {} + return { ...config, platform, device: p.device ?? null, appId: p.appId ?? null, appPath: p.appPath ?? null } +} + // substitute :param placeholders in a urlPath with sample values export function deepLinkFor(scheme, route, params = {}) { let p = route.urlPath ?? '/' diff --git a/action/cli/screenmap-ci.mjs b/action/cli/screenmap-ci.mjs index f8b6305..6546f91 100755 --- a/action/cli/screenmap-ci.mjs +++ b/action/cli/screenmap-ci.mjs @@ -13,18 +13,26 @@ // screenmap-ci flows-pr --repo owner/name --flows [--base main] --title "…" [--body "…"] // screenmap-ci flows-adopt --project [--from .screenmap/out/flows] [--to .screenmap/flows] [--force] // move locally recorded flows into the directory CI replays from -// screenmap-ci resolve-app --project [--profile development-simulator] (EAS: reuse-by-fingerprint or build) +// screenmap-ci resolve-app --project [--platform ios|android] [--profile ] +// (EAS: reuse-by-fingerprint or build) +// screenmap-ci merge --inputs ios=a.scrmap,android=b.scrmap --out combined.scrmap +// fold per-platform baselines into one multi-platform map +// +// baseline and pr capture on every platform in config.platforms (default +// ["ios"]); --platform narrows a run to one of them, which is how the +// Action splits iOS and Android across two runners. // // Runs locally too: the same commands the Action runs, against your own -// simulator. See docs/ci.md. +// simulator or emulator. See docs/ci.md. import fs from 'node:fs' import path from 'node:path' -import { parseArgs, loadConfig, readJson, writeJson, ensureDir, exists, log, sh, deepLinkFor } from './lib/util.mjs' -import { openSession } from './lib/sim.mjs' -import { readBaseline, parseRoutes, computeSuspects, packBaseline, packDiff, downscaleAll } from './lib/bundle.mjs' +import { parseArgs, loadConfig, platformConfig, readJson, writeJson, ensureDir, exists, log, sh, deepLinkFor } from './lib/util.mjs' +import { openSession } from './lib/device.mjs' +import { readBaseline, parseRoutes, computeSuspects, packBaseline, packDiff, downscaleAll, baselineSide, platformsIn } from './lib/bundle.mjs' import { loadFlows, replayFlow, verifyLanding, verifyDeepLink } from './lib/replay.mjs' import { argentAvailable, argentVersion } from './lib/argent.mjs' import { runAgent, agentInfo } from './lib/agent.mjs' +import { ocrBackend } from './lib/ocr.mjs' import { upsertStickyComment, publishToBranch, openFlowsPR, repoSlug } from './lib/github.mjs' const { opts, positional } = parseArgs(process.argv.slice(2)) @@ -56,7 +64,7 @@ async function captureRoutes({ project, config, scheme, session, routes, flows, await session.relaunch() // every replay starts from a clean app let wrote = 0, broke = null for (const rec of recs) { - const { ok, written } = await replayFlow(rec, { udid: session.udid, outDir, tmpDir: path.join(work, 'tmp') }) + const { ok, written } = await replayFlow(rec, { device: session, outDir, tmpDir: path.join(work, 'tmp') }) wrote += written.length if (!ok) { broke = rec.name; log(`replay ${rec.name} failed — falling back`); break } } @@ -64,7 +72,7 @@ async function captureRoutes({ project, config, scheme, session, routes, flows, if (!broke && wrote > 0 && exists(shot)) { const primary = f.nav ?? f.visit ?? recs[0] const v = await verifyLanding({ - shot, rec: primary, udid: session.udid, + shot, rec: primary, device: session, probe: async () => { try { await session.relaunch(); const p = path.join(work, 'tmp', `${r.slug}.deeplink.png`); await session.visit(deepLinkFor(scheme, r, config.params), p, r.params?.length ? config.waits.network : config.waits.transition); return p } catch { return null } }, }) if (v.ok) { result.replay.push(r.id); done = true; log(`replay ${primary.name} ✓ (${v.method}${v.score != null ? ` ${v.score}` : ''})`) } @@ -144,7 +152,8 @@ async function captureRoutes({ project, config, scheme, session, routes, flows, const screens = candidates.map((r) => ({ id: r.id, urlPath: r.urlPath, slug: r.slug, file: r.file, deepLink: deepLinkFor(scheme, r, config.params), reason: r.reason })) if (extra.length) log(`effort=${config.effort} (scan=${scan}): ${result.unflowed.length} flowless + ${extra.length} re-checked`) const a = runAgent({ - projectDir: project, config, screens, scheme, udid: session.udid, bundleId: session.bundleId, + projectDir: project, config, screens, scheme, udid: session.id, bundleId: session.appId, + platform: session.platform, deviceName: session.deviceName, outScreensDir: path.join(agentDir, 'screens'), outFlowsDir: path.join(agentDir, 'flows'), notesPath: path.join(agentDir, 'notes.json'), summaryPath: path.join(agentDir, 'summary.json'), mode: agentMode, prContext, @@ -180,63 +189,99 @@ async function baseline() { const commit = opts.commit ?? git(['rev-parse', 'HEAD'], project) const ref = opts.ref ?? git(['rev-parse', '--abbrev-ref', 'HEAD'], project) const appName = config.appName ?? path.basename(project) - const screensDir = ensureDir(path.join(work, 'screens')) - let captureStatus = {} + const platforms = opts.platform ? [String(opts.platform)] : config.platforms + const multi = platforms.length > 1 - // previous baseline → reuse what didn't change - let routes = graph.routes - let reused = 0 - let prev = null + // Which screens changed is static analysis — the same answer on every + // platform — so the suspect set is computed once and each platform then + // decides reuse against its own side of the previous bundle. + let prev = null, suspect = null, prevCommit = null if (opts.previous && exists(opts.previous) && !opts.full) { prev = readBaseline(opts.previous, path.join(work, 'prev')) - const prevCommit = prev.manifest.source?.commit + prevCommit = prev.manifest.source?.commit const changed = prevCommit ? git(['diff', '--name-only', prevCommit, 'HEAD'], project) : null if (changed === null) { log('previous baseline commit not in history — doing a full capture') + prev = null } else { const suspects = computeSuspects({ diffDir: path.join(work, 'diff'), baseGraph: prev.graph, headGraph: graph, changedFiles: changed.split('\n').filter(Boolean), projectDir: project, depth: config.suspects.depth, broadCap: config.suspects.broadCap }) - const suspect = new Set(suspects.capture.filter((c) => c.status !== 'D').map((c) => c.id)) + suspect = new Set(suspects.capture.filter((c) => c.status !== 'D').map((c) => c.id)) + } + } + + const flowDirs = [config.flowsDir, '.screenmap/out/flows'] + const flows = loadFlows(project, flowDirs) + const flowsDirForPack = flowDirs.map((d) => path.resolve(project, d)).find((d) => exists(d)) ?? path.resolve(project, config.flowsDir) + + const sides = [] + for (const platform of platforms) { + const pc = platformConfig(config, platform) + const screensDir = ensureDir(path.join(work, 'screens', platform)) + const captureStatus = {} + let routes = graph.routes + let reused = 0 + if (prev) { + const side = baselineSide(prev, platform) const prevById = new Map(prev.map.nodes.map((n) => [n.id, n])) - const prevStatus = readJson(path.join(prev.dir, 'capture-status.json'), {}) routes = [] for (const r of graph.routes) { - const p = prevById.get(r.id) - const stale = suspect.has(r.id) || !p?.capture?.screenshot || ['error-boundary', 'loading', 'missing'].includes(p?.capture?.status) + const c = side.capture(prevById.get(r.id)) + // a platform turned on since the last baseline has no side of its own, + // so nothing is reusable and it captures in full + const stale = suspect.has(r.id) || !side.exists || !c?.screenshot || ['error-boundary', 'loading', 'missing'].includes(c?.status) if (stale) routes.push({ ...r, reason: suspect.has(r.id) ? 'changed since baseline' : 'no usable previous capture' }) - else { copyShots(prev.screensDir, screensDir, r.slug); reused++; if (prevStatus[r.id]) captureStatus[r.id] = prevStatus[r.id] } + else { copyShots(side.dir, screensDir, r.slug); reused++; if (side.status[r.id]) captureStatus[r.id] = side.status[r.id] } } - log(`incremental baseline: ${routes.length} to capture, ${reused} reused from ${prevCommit.slice(0, 7)}`) + log(`incremental baseline (${platform}): ${routes.length} to capture, ${reused} reused from ${prevCommit.slice(0, 7)}`) } + if (opts.only) { const only = new Set(String(opts.only).split(',')); routes = routes.filter((r) => only.has(r.id)) } + if (opts.limit) routes = routes.slice(0, Number(opts.limit)) + + let cap = { replay: [], deeplink: [], agent: [], failed: [], unflowed: [] } + let deviceName = pc.device + if (routes.length && !opts['no-sim']) { + const session = await openSession({ projectDir: project, config: pc, scheme, platform }) + deviceName = session.deviceName + try { + cap = await captureRoutes({ project, config: pc, scheme, session, routes, flows, outDir: screensDir, work: path.join(work, platform), agentMode: 'baseline', agentEnabled: !opts['no-agent'] }) + } finally { session.close() } + } + for (const id of cap.failed) captureStatus[id] = { status: 'missing', note: `deep link failed in CI (${platform})` } + downscaleAll(screensDir) + sides.push({ platform, device: deviceName, screensDir, cap, captureStatus, reused }) } - if (opts.only) { const only = new Set(String(opts.only).split(',')); routes = routes.filter((r) => only.has(r.id)) } - if (opts.limit) routes = routes.slice(0, Number(opts.limit)) - const flowDirs = [config.flowsDir, '.screenmap/out/flows'] - const flows = loadFlows(project, flowDirs) - const flowsDirForPack = flowDirs.map((d) => path.resolve(project, d)).find((d) => exists(d)) ?? path.resolve(project, config.flowsDir) - let cap = { replay: [], deeplink: [], agent: [], failed: [], unflowed: [] } - let deviceName = config.device - if (routes.length && !opts['no-sim']) { - const session = await openSession({ projectDir: project, config, scheme }) - deviceName = session.deviceName - try { - cap = await captureRoutes({ project, config, scheme, session, routes, flows, outDir: screensDir, work, agentMode: 'baseline', agentEnabled: !opts['no-agent'] }) - } finally { session.close() } - } - for (const id of cap.failed) captureStatus[id] = { status: 'missing', note: 'deep link failed in CI' } - downscaleAll(screensDir) const out = path.resolve(opts.out ?? path.join(work, `${appName}-${(commit ?? 'local').slice(0, 7)}.scrmap`)) - packBaseline({ graph, screensDir, flowsDir: flowsDirForPack, captureStatus, appName, device: deviceName, commit, ref, out }) + packBaseline({ + graph, flowsDir: flowsDirForPack, appName, commit, ref, out, + platforms: sides.map((s) => ({ platform: s.platform, device: s.device, screensDir: s.screensDir })), + captureStatus: multi ? Object.fromEntries(sides.map((s) => [s.platform, s.captureStatus])) : sides[0].captureStatus, + }) + const sum = (f) => sides.reduce((n, s) => n + f(s), 0) const summary = { - kind: 'baseline', app: appName, commit, ref, bundle: out, total: graph.routes.length, reused, - device: deviceName, argent: argentVersion(), - captured: { replay: cap.replay.length, deeplink: cap.deeplink.length, agent: cap.agent.length, failed: cap.failed }, - unflowed: cap.unflowed.map((r) => r.id), drifted: cap.drifted ?? [], agent: cap.agentRun ?? { ran: false }, recordedFlowsDir: cap.recordedFlowsDir ?? null, + kind: 'baseline', app: appName, commit, ref, bundle: out, total: graph.routes.length, + reused: sum((s) => s.reused), + platforms: sides.map((s) => ({ + platform: s.platform, device: s.device, reused: s.reused, + captured: { replay: s.cap.replay.length, deeplink: s.cap.deeplink.length, agent: s.cap.agent.length, failed: s.cap.failed }, + unflowed: s.cap.unflowed.map((r) => r.id), drifted: s.cap.drifted ?? [], agent: s.cap.agentRun ?? { ran: false }, + })), + // the first platform stays the headline one so existing consumers (the PR + // comment, the shot renderer) keep reading the fields they always read + device: sides.map((s) => s.device).filter(Boolean).join(' · ') || null, + argent: argentVersion(), ocr: ocrBackend(), + captured: { + replay: sum((s) => s.cap.replay.length), deeplink: sum((s) => s.cap.deeplink.length), + agent: sum((s) => s.cap.agent.length), failed: sides.flatMap((s) => s.cap.failed), + }, + unflowed: [...new Set(sides.flatMap((s) => s.cap.unflowed.map((r) => r.id)))], + drifted: sides.flatMap((s) => s.cap.drifted ?? []), + agent: sides[0].cap.agentRun ?? { ran: false }, + recordedFlowsDir: sides.map((s) => s.cap.recordedFlowsDir).find(Boolean) ?? null, } writeJson(path.join(work, 'summary.json'), summary) console.log(JSON.stringify(summary, null, 2)) } - async function pr() { const project = path.resolve(opts.project ?? '.') const config = loadConfig(project) @@ -249,6 +294,8 @@ async function pr() { const baseSha = opts.base ?? base.manifest.source?.commit ?? null const headSha = opts.head ?? git(['rev-parse', 'HEAD'], project) const appName = config.appName ?? base.manifest.app?.name ?? path.basename(project) + const platforms = opts.platform ? [String(opts.platform)] : config.platforms + const multi = platforms.length > 1 let changed = opts['changed-files'] ? fs.readFileSync(opts['changed-files'], 'utf8').split('\n').filter(Boolean) : null if (!changed && baseSha) changed = (git(['diff', '--name-only', `${baseSha}...${headSha}`], project) ?? git(['diff', '--name-only', baseSha, headSha], project) ?? '').split('\n').filter(Boolean) if (!changed) throw new Error('cannot determine changed files: pass --changed-files or make sure the base commit is fetched') @@ -257,76 +304,106 @@ async function pr() { const suspects = computeSuspects({ diffDir, baseGraph: base.graph, headGraph, changedFiles: changed, projectDir: project, depth: config.suspects.depth, broadCap: config.suspects.broadCap }) writeJson(path.join(diffDir, 'pr.json'), { number: opts.pr ? Number(opts.pr) : undefined, title: opts.title, url: opts.url, baseSha, headSha, baseRef: opts['base-ref'] ?? base.manifest.source?.ref ?? null, headRef: opts['head-ref'] ?? null }) - // base side comes from the baseline — nothing is captured twice - const baseStatus = readJson(path.join(base.dir, 'capture-status.json'), {}) - const baseSubset = {} - for (const c of suspects.capture.filter((c) => c.side !== 'head')) { - copyShots(base.screensDir, path.join(diffDir, 'base', 'screens'), c.slug) - if (baseStatus[c.id]) baseSubset[c.id] = baseStatus[c.id] - } - writeJson(path.join(diffDir, 'base', 'capture-status.json'), baseSubset) - - // head side: capture suspects on the PR head const headRoutes = suspects.capture.filter((c) => c.side !== 'base').map((c) => ({ ...headGraph.routes.find((r) => r.id === c.id), reason: `${c.status}: ${c.reason}${c.via?.length ? ' via ' + c.via.join(', ') : ''}` })).filter((r) => r.id) const flows = loadFlows(project, [config.flowsDir, '.screenmap/out/flows']) - let cap = { replay: [], deeplink: [], agent: [], failed: [], unflowed: [] } - let deviceName = config.device - if (headRoutes.length && !opts['no-sim']) { - const session = await openSession({ projectDir: project, config, scheme }) - deviceName = session.deviceName - try { - cap = await captureRoutes({ - project, config, scheme, session, routes: headRoutes, flows, outDir: path.join(diffDir, 'head', 'screens'), work, - agentMode: 'pr', agentEnabled: !opts['no-agent'], - prContext: `${opts.title ?? ''} — changed files: ${changed.slice(0, 40).join(', ')}${changed.length > 40 ? ` (+${changed.length - 40})` : ''}`, - }) - } finally { session.close() } + const prContext = `${opts.title ?? ''} — changed files: ${changed.slice(0, 40).join(', ')}${changed.length > 40 ? ` (+${changed.length - 40})` : ''}` + + const sides = [] + const baseStatusByPlatform = {}, headStatusByPlatform = {} + for (const platform of platforms) { + const pc = platformConfig(config, platform) + const sub = multi ? [platform] : [] + // base side comes from the baseline — nothing is captured twice + const prevSide = baselineSide(base, platform) + const baseSubset = {} + for (const c of suspects.capture.filter((c) => c.side !== 'head')) { + copyShots(prevSide.dir, path.join(diffDir, 'base', 'screens', ...sub), c.slug) + if (prevSide.status[c.id]) baseSubset[c.id] = prevSide.status[c.id] + } + baseStatusByPlatform[platform] = baseSubset + + // head side: capture suspects on the PR head + let cap = { replay: [], deeplink: [], agent: [], failed: [], unflowed: [] } + let deviceName = pc.device + const headScreens = path.join(diffDir, 'head', 'screens', ...sub) + if (headRoutes.length && !opts['no-sim']) { + const session = await openSession({ projectDir: project, config: pc, scheme, platform }) + deviceName = session.deviceName + try { + cap = await captureRoutes({ + project, config: pc, scheme, session, routes: headRoutes, flows, outDir: headScreens, work: path.join(work, platform), + agentMode: 'pr', agentEnabled: !opts['no-agent'], prContext, + }) + } finally { session.close() } + } + // notes are about the change, not the device — the first platform to + // produce them wins rather than each overwriting the last + if (cap.notes && !exists(path.join(diffDir, 'notes.json'))) writeJson(path.join(diffDir, 'notes.json'), cap.notes) + const headStatus = {} + for (const id of cap.failed) headStatus[id] = { status: 'missing', note: `deep link failed in CI (${platform})` } + headStatusByPlatform[platform] = headStatus + downscaleAll(headScreens) + sides.push({ platform, device: deviceName, cap }) } - if (cap.notes) writeJson(path.join(diffDir, 'notes.json'), cap.notes) - const headStatus = {} - for (const id of cap.failed) headStatus[id] = { status: 'missing', note: 'deep link failed in CI' } - writeJson(path.join(diffDir, 'head', 'capture-status.json'), headStatus) - downscaleAll(path.join(diffDir, 'head', 'screens')) + writeJson(path.join(diffDir, 'base', 'capture-status.json'), multi ? baseStatusByPlatform : baseStatusByPlatform[platforms[0]]) + writeJson(path.join(diffDir, 'head', 'capture-status.json'), multi ? headStatusByPlatform : headStatusByPlatform[platforms[0]]) + const out = path.resolve(opts.out ?? path.join(work, `${appName}-${opts.pr ? `pr${opts.pr}` : (headSha ?? 'head').slice(0, 7)}.diff.scrmap`)) - packDiff({ diffDir, device: deviceName, out }) + packDiff({ diffDir, platforms: sides.map((s) => ({ platform: s.platform, device: s.device })), out }) const diff = readJson(path.join(diffDir, 'diff.json')) + const sum = (f) => sides.reduce((n, s) => n + f(s), 0) const summary = { kind: 'pr', app: appName, pr: opts.pr ? Number(opts.pr) : null, title: opts.title ?? null, baseSha, headSha, bundle: out, - baselineGeneratedAt: base.manifest.generatedAt, device: deviceName, argent: argentVersion(), + baselineGeneratedAt: base.manifest.generatedAt, + device: sides.map((s) => s.device).filter(Boolean).join(' · ') || null, + argent: argentVersion(), ocr: ocrBackend(), + platforms: sides.map((s) => ({ + platform: s.platform, device: s.device, + captured: { replay: s.cap.replay.length, deeplink: s.cap.deeplink.length, agent: s.cap.agent.length, failed: s.cap.failed }, + drifted: s.cap.drifted ?? [], unverified: s.cap.unverified ?? [], agent: s.cap.agentRun ?? { ran: false }, + })), suspects: { added: suspects.capture.filter((c) => c.status === 'A').length, modified: suspects.capture.filter((c) => c.status === 'M').length, removed: suspects.capture.filter((c) => c.status === 'D').length, broadFiles: suspects.broadFiles }, - captured: { replay: cap.replay.length, deeplink: cap.deeplink.length, agent: cap.agent.length, failed: cap.failed }, - agent: cap.agentRun ?? { ran: false }, recordedFlowsDir: cap.recordedFlowsDir ?? null, drifted: cap.drifted ?? [], - unverified: cap.unverified ?? [], + captured: { + replay: sum((s) => s.cap.replay.length), deeplink: sum((s) => s.cap.deeplink.length), + agent: sum((s) => s.cap.agent.length), failed: sides.flatMap((s) => s.cap.failed), + }, + agent: sides[0].cap.agentRun ?? { ran: false }, + recordedFlowsDir: sides.map((s) => s.cap.recordedFlowsDir).find(Boolean) ?? null, + drifted: sides.flatMap((s) => s.cap.drifted ?? []), + unverified: sides.flatMap((s) => s.cap.unverified ?? []), diff: { nodes: diff.nodes, dismissed: diff.dismissed ?? [], edges: diff.edges, states: (diff.states ?? []).filter((s) => s.reason !== 'hint') }, // base first so head wins on collision: a removed route only exists on the // base side, and without it the comment prints its bare id ("grind") // where every other row shows a path ("/grind") routes: Object.fromEntries([...base.graph.routes, ...headGraph.routes].map((r) => [r.id, r.urlPath])), - shots: collectShots(diffDir, [...headGraph.routes, ...base.graph.routes], [...diff.nodes.map((d) => d.id), ...(diff.dismissed ?? []).map((d) => d.id)]), + shots: collectShots(diffDir, [...headGraph.routes, ...base.graph.routes], [...diff.nodes.map((d) => d.id), ...(diff.dismissed ?? []).map((d) => d.id)], multi ? platforms[0] : null), } writeJson(path.join(work, 'summary.json'), summary) console.log(JSON.stringify(summary, null, 2)) } -// Bundle-relative paths of every capture the comment might want to show: -// per node, the bare screen on each side plus one entry per named state. Only -// files that actually exist are listed, so the renderer can just check. -function collectShots(diffDir, routes, ids) { +// Paths the PR comment's images are published under. The comment shows ONE +// platform — a side-by-side strip of both would not fit GitHub's table — so a +// multi-platform run passes the platform whose screens the comment should show +// (the first captured), and the bundle still carries every platform. +function collectShots(diffDir, routes, ids, platform = null) { const slugOf = new Map(routes.map((r) => [r.id, r.slug])) + const sub = platform ? [platform] : [] + const rel = (side, f) => [side, 'screens', ...sub, f].join('/') const out = {} for (const id of new Set(ids)) { const slug = slugOf.get(id) if (!slug) continue const entry = { states: {} } for (const side of ['head', 'base']) { - const dir = path.join(diffDir, side, 'screens') + const dir = path.join(diffDir, side, 'screens', ...sub) if (!exists(dir)) continue for (const f of fs.readdirSync(dir)) { const stem = f.replace(/\.\w+$/, '') - if (stem === slug) entry[side] = `${side}/screens/${f}` + if (stem === slug) entry[side] = rel(side, f) else if (stem.startsWith(slug + '--')) { const name = stem.slice(slug.length + 2) - ;(entry.states[name] ??= {})[side] = `${side}/screens/${f}` + ;(entry.states[name] ??= {})[side] = rel(side, f) } } } @@ -487,7 +564,11 @@ function renderComment(s, { mapUrl, changesUrl, artifactUrl, shotUrl, shotsBase, else if (a.keyEnv === null && a.hasKey === null) agentDesc += ' · no LLM key configured' } const foot = [ - `Captured on ${s.device ?? 'simulator'}: ${s.captured.replay} by flow replay${s.argent ? ` (argent ${s.argent})` : ''}, ${s.captured.deeplink} by deep link, ${s.captured.agent} by agent (${agentDesc}).`, + `Captured on ${s.device ?? 'simulator'}${s.platforms?.length > 1 ? ` (${s.platforms.map((p) => p.platform).join(' + ')})` : ''}: ${s.captured.replay} by flow replay${s.argent ? ` (argent ${s.argent})` : ''}, ${s.captured.deeplink} by deep link, ${s.captured.agent} by agent (${agentDesc}).`, + // tesseract reads roughly two thirds of the words Vision does, which makes + // a drift or verification warning likelier to be the OCR's fault than the + // app's. Say which backend read the screens whenever it is not Vision. + s.ocr && s.ocr !== 'vision' ? `Screen text read with ${s.ocr === 'tesseract' ? 'tesseract (lower recall than Vision — verification warnings here are less certain)' : s.ocr}.` : null, s.baselineGeneratedAt ? `Compared against baseline \`${(s.baseSha ?? '').slice(0, 7)}\` from ${new Date(s.baselineGeneratedAt).toISOString().slice(0, 16).replace('T', ' ')} UTC.` : null, `${A.length} added · ${M.length} changed · ${D.length} removed · ${dismissed.length} suspect${dismissed.length === 1 ? '' : 's'} cleared by looking.`, s.suspects.broadFiles?.length ? `${s.suspects.broadFiles.length} broadly-imported changed file${s.suspects.broadFiles.length === 1 ? '' : 's'} excluded from suspect marking.` : null, @@ -630,12 +711,67 @@ async function shot() { } async function resolveAppCmd() { - const { resolveApp } = await import('./lib/eas.mjs') + const { resolveApp, DEFAULT_PROFILES } = await import('./lib/eas.mjs') const project = path.resolve(opts.project ?? '.') - const res = resolveApp({ projectDir: project, profile: opts.profile ?? 'development-simulator', workDir: path.resolve(opts.work ?? path.join(project, '.screenmap', 'out', 'ci', 'eas')) }) + const platform = String(opts.platform ?? 'ios') + const res = resolveApp({ + projectDir: project, platform, + profile: opts.profile ?? DEFAULT_PROFILES[platform] ?? DEFAULT_PROFILES.ios, + workDir: path.resolve(opts.work ?? path.join(project, '.screenmap', 'out', 'ci', 'eas', platform)), + }) console.log(JSON.stringify(res, null, 2)) } -const commands = { baseline, pr, comment, status, publish, 'flows-pr': flowsPr, 'flows-adopt': flowsAdopt, 'resolve-app': resolveAppCmd, shot } -if (!commands[cmd]) { console.error('usage: screenmap-ci [options]'); process.exit(1) } +// Fold single-platform baselines into one multi-platform map. iOS has to run on +// a macOS runner and Android is only worth doing on Linux, so the two platforms +// are captured by separate jobs; this is what makes their output one bundle +// rather than two the reader has to hold side by side. +// +// The graph, edges and flows come from the FIRST input: they are static +// analysis of the same commit, so every input agrees on them, and picking one +// beats reconciling identical copies. +async function merge() { + if (!opts.inputs) throw new Error('--inputs ios=a.scrmap,android=b.scrmap is required') + const parsed = String(opts.inputs).split(',').map((pair) => { + const i = pair.indexOf('=') + if (i < 0) throw new Error(`--inputs entry "${pair}" must be =`) + return { platform: pair.slice(0, i).trim(), file: path.resolve(pair.slice(i + 1).trim()) } + }) + const work = ensureDir(path.resolve(opts.work ?? path.join(process.cwd(), '.screenmap-merge'))) + const sides = [] + let first = null + const captureStatus = {} + for (const { platform, file } of parsed) { + if (!exists(file)) { log(`merge: skipping ${platform} — ${file} not found`); continue } + const b = readBaseline(file, path.join(work, platform)) + first ??= b + const side = baselineSide(b, platform) + // Refuse a bundle that does not carry the platform it is being merged as. + // Two artifact paths swapped in a workflow is an easy mistake to make and an + // impossible one to spot afterwards: the map would show iOS screenshots + // under the Android switch and look entirely plausible. + if (!side.exists) { + throw new Error(`merge: ${file} does not carry ${platform} captures (it holds ${platformsIn(b.manifest).join(', ')}) — check the --inputs mapping`) + } + // a single-platform input has its screens at screens/, a multi-platform one + // at screens// — baselineSide answers for both + sides.push({ platform, device: b.manifest.app?.device ?? null, screensDir: side.dir }) + captureStatus[platform] = side.status + } + if (!sides.length) throw new Error('merge: none of the inputs existed') + if (sides.length === 1) log(`merge: only ${sides[0].platform} was available — writing a single-platform map`) + const out = path.resolve(opts.out ?? path.join(work, 'merged.scrmap')) + packBaseline({ + graph: first.graph, platforms: sides, flowsDir: first.flowsDir, + captureStatus: sides.length > 1 ? captureStatus : captureStatus[sides[0].platform], + appName: opts.app ?? first.manifest.app?.name ?? 'app', + commit: opts.commit ?? first.manifest.source?.commit ?? null, + ref: opts.ref ?? first.manifest.source?.ref ?? null, + out, + }) + console.log(JSON.stringify({ kind: 'merge', bundle: out, platforms: sides.map((s) => ({ platform: s.platform, device: s.device })) }, null, 2)) +} + +const commands = { baseline, pr, comment, status, publish, 'flows-pr': flowsPr, 'flows-adopt': flowsAdopt, 'resolve-app': resolveAppCmd, merge, shot } +if (!commands[cmd]) { console.error('usage: screenmap-ci [options]'); process.exit(1) } commands[cmd]().catch((e) => { console.error('[screenmap-ci] failed:', e.message); process.exit(1) }) diff --git a/action/templates/config.json b/action/templates/config.json index f6f596e..5e67df6 100644 --- a/action/templates/config.json +++ b/action/templates/config.json @@ -3,8 +3,17 @@ "$effort": "fast | balanced (default) | thorough — token cost and speed vs accuracy. fast: agent only sees screens no flow can reach. balanced: also re-checks routes whose deep link guesses a param. thorough: every screen goes through the agent. Anything you set below wins over the preset.", "effort": "balanced", "scheme": "myapp", - "device": "iPhone 17 Pro", "appName": "myapp", + "$platforms": "Which platforms a local run captures: [\"ios\"] (default), [\"android\"], or both — both puts every screen in one map with a platform switcher. In CI each platform is its own job (see the workflow templates), so the Action takes one platform and the jobs are merged afterwards.", + "platforms": ["ios"], + "ios": { + "$comment": "device is the simulator name; bundleId/appPath are discovered when omitted. The pre-multi-platform top-level device/bundleId/appPath keys still mean iOS.", + "device": "iPhone 17 Pro" + }, + "android": { + "$comment": "device is the AVD name or the model of an attached device; null takes whatever is attached, else the first AVD. packageName/appPath are discovered when omitted.", + "device": null + }, "waits": { "transition": 2500, "network": 6000, "boot": 15000 }, "suspects": { "broadCap": 8 }, "agent": { diff --git a/action/templates/screenmap-baseline.yml b/action/templates/screenmap-baseline.yml index 0047a3a..8983014 100644 --- a/action/templates/screenmap-baseline.yml +++ b/action/templates/screenmap-baseline.yml @@ -37,3 +37,6 @@ jobs: # agent_provider: claude # or codex | gemini | opencode — pass the matching key above expo_token: ${{ secrets.EXPO_TOKEN }} # EAS builds/serves the dev client (or pass app_path instead) flows_pr: "true" + # platform: ios # or android on ubuntu-latest; for both, split into two + # jobs and fold them with `screenmap-ci merge` + # (see the note at the bottom of screenmap-pr.yml) diff --git a/action/templates/screenmap-pr.yml b/action/templates/screenmap-pr.yml index 769f52a..d9bf5a0 100644 --- a/action/templates/screenmap-pr.yml +++ b/action/templates/screenmap-pr.yml @@ -27,5 +27,47 @@ jobs: agent_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # optional — omit for deterministic-only # agent_provider: claude # or codex | gemini | opencode — pass the matching key above expo_token: ${{ secrets.EXPO_TOKEN }} # EAS builds/serves the dev client (or pass app_path instead) + # platform: ios # or android (then use runs-on: ubuntu-latest — a tenth of the macOS rate) # project: apps/mobile # publish: "false" # private repos that only want the artifact link + +# ─── Both platforms in one map ──────────────────────────────────────────────── +# iOS needs macOS and Android is only worth running on Linux, so they are two +# jobs whose bundles are folded together afterwards. Replace the job above with: +# +# jobs: +# ios: +# runs-on: macos-26 +# steps: +# - uses: actions/checkout@v4 +# with: { fetch-depth: 0 } +# - uses: actions/setup-node@v4 +# with: { node-version: 22 } +# - uses: aleqsio/screenmap@v1 +# id: map +# with: +# mode: pr +# platform: ios +# publish: "false" # publish once, after the merge +# expo_token: ${{ secrets.EXPO_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: { name: screenmap-ios, path: "${{ steps.map.outputs.bundle }}" } +# +# android: # same, with platform: android on ubuntu-latest +# runs-on: ubuntu-latest +# ... +# +# merge: +# needs: [ios, android] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/download-artifact@v4 +# with: { path: bundles } +# - run: | +# npx --yes github:aleqsio/screenmap#v1 --help >/dev/null 2>&1 || true +# screenmap-ci merge \ +# --inputs ios=bundles/screenmap-ios/*.scrmap,android=bundles/screenmap-android/*.scrmap \ +# --out combined.scrmap +# +# The merged bundle opens in the viewer with an iOS / Android switcher, and every +# screen keeps its own per-platform capture status. diff --git a/apps/visualiser/src/App.jsx b/apps/visualiser/src/App.jsx index 9c31a1a..cb0f27c 100644 --- a/apps/visualiser/src/App.jsx +++ b/apps/visualiser/src/App.jsx @@ -17,6 +17,9 @@ export default function App() { setMode={b.setMode} hasChanges={b.hasChanges} overlaid={b.overlaid} + platforms={b.platforms} + platform={b.platform} + setPlatform={b.setPlatform} onOpenBuffer={b.open} onCloseChanges={b.closeChanges} /> diff --git a/apps/visualiser/src/components/Graph.jsx b/apps/visualiser/src/components/Graph.jsx index 296e3e7..2c087d0 100644 --- a/apps/visualiser/src/components/Graph.jsx +++ b/apps/visualiser/src/components/Graph.jsx @@ -32,7 +32,7 @@ function statusBadge(node) { const HIDDEN_STEPS = ['wait', 'screenshot'] -export default function Graph({ bundle, mode, setMode, hasChanges, overlaid, onOpenBuffer, onCloseChanges }) { +export default function Graph({ bundle, mode, setMode, hasChanges, overlaid, platforms, platform, setPlatform, onOpenBuffer, onCloseChanges }) { const { manifest, map, images, diff } = bundle const diffMode = mode === 'changes' && !!diff const [positions, setPositions] = useState(null) @@ -509,6 +509,9 @@ export default function Graph({ bundle, mode, setMode, hasChanges, overlaid, onO overlaid={overlaid} stats={stats} diffStats={diffStats} + platforms={platforms} + platform={platform} + setPlatform={setPlatform} onOpenBuffer={onOpenBuffer} onCloseChanges={onCloseChanges} />} diff --git a/apps/visualiser/src/components/ScreenNode.jsx b/apps/visualiser/src/components/ScreenNode.jsx index cd9dc5a..101d706 100644 --- a/apps/visualiser/src/components/ScreenNode.jsx +++ b/apps/visualiser/src/components/ScreenNode.jsx @@ -100,6 +100,7 @@ export default function ScreenNode({ data, selected }) { const shownBase = activeState ? baseStates.find((s) => s.name === activeState)?.img ?? null : imgBase const canSwap = diff?.status === 'M' && !!shown && !!shownBase const [hovered, setHovered] = useState(false) + const [aspect, setAspect] = useState(null) const [diffImg, setDiffImg] = useState(null) useEffect(() => { if (!(hovered && canSwap)) { setDiffImg(null); return } @@ -123,11 +124,23 @@ export default function ScreenNode({ data, selected }) {
setHovered(true) : undefined} onMouseLeave={canSwap ? () => setHovered(false) : undefined} > {displayed ? ( - {node.urlPath} + {node.urlPath} { + const { naturalWidth: w, naturalHeight: h } = e.currentTarget + if (w && h) setAspect(`${w} / ${h}`) + }} + /> ) : (
{node.capture.status === 'missing' ? 'no capture' : node.capture.status}
)} diff --git a/apps/visualiser/src/components/TopBar.jsx b/apps/visualiser/src/components/TopBar.jsx index ee5dc54..da8a79b 100644 --- a/apps/visualiser/src/components/TopBar.jsx +++ b/apps/visualiser/src/components/TopBar.jsx @@ -43,11 +43,17 @@ function ThemeMenu() { ) } -export default function TopBar({ manifest, mode, setMode, hasChanges, overlaid, stats, diffStats, onOpenBuffer, onCloseChanges }) { +const PLATFORM_LABEL = { ios: 'iOS', android: 'Android' } + +export default function TopBar({ manifest, mode, setMode, hasChanges, overlaid, stats, diffStats, platforms = [], platform, setPlatform, onOpenBuffer, onCloseChanges }) { const diffMode = mode === 'changes' + // The switcher only earns its space when there is something to switch to; a + // single-platform map keeps the bar it always had. + const multiPlatform = platforms.length > 1 + const activeDevice = platforms.find((p) => p.platform === platform)?.device ?? manifest.app.device const sub = diffMode ? `${manifest.pr ? `PR #${manifest.pr.number} · ` : ''}${manifest.pr?.title ?? `${(manifest.base?.commit ?? 'base').slice(0, 7)} → ${(manifest.head?.commit ?? 'head').slice(0, 7)}`}${overlaid ? '' : ' · no map backdrop'}` - : `${manifest.app.mode ?? 'map'} · ${manifest.app.device ?? 'unknown device'} · ${new Date(manifest.generatedAt).toLocaleDateString()}` + : `${manifest.app.mode ?? 'map'} · ${activeDevice ?? 'unknown device'} · ${new Date(manifest.generatedAt).toLocaleDateString()}` // Two width thresholds, both measured rather than guessed. Below 520px the // identity panel and the tool panel no longer share a line — they used to @@ -93,6 +99,34 @@ export default function TopBar({ manifest, mode, setMode, hasChanges, overlaid, + {multiPlatform && ( + v && setPlatform?.(v)} + aria-label="Platform" + className="shrink-0" + > + {platforms.map((p) => ( + + + + {PLATFORM_LABEL[p.platform] ?? p.platform} + + + {p.device ?? p.label ?? p.platform} + + ))} + + )} +
{diffMode && diffStats ? ( <> diff --git a/apps/visualiser/src/index.css b/apps/visualiser/src/index.css index 98a55fa..f1b29d0 100644 --- a/apps/visualiser/src/index.css +++ b/apps/visualiser/src/index.css @@ -264,16 +264,19 @@ transition-duration: 0.3s; transition-timing-function: cubic-bezier(0.2, 0, 0, 1); } +/* --shot-aspect is set from the capture's own dimensions once it loads (see + ScreenNode); the iPhone ratio is only the shape the frame holds until then, + and for an Android capture it is replaced rather than cropped to. */ .phone img { display: block; width: 100%; - aspect-ratio: 402 / 874; + aspect-ratio: var(--shot-aspect, 402 / 874); object-fit: cover; border-radius: 19px; /* concentric: 24 − 5 */ background: var(--paper-2); } .no-shot { - aspect-ratio: 402 / 874; + aspect-ratio: var(--shot-aspect, 402 / 874); border-radius: 19px; display: grid; place-items: center; diff --git a/apps/visualiser/src/lib/loadBundle.js b/apps/visualiser/src/lib/loadBundle.js index 241dc41..8c4e602 100644 --- a/apps/visualiser/src/lib/loadBundle.js +++ b/apps/visualiser/src/lib/loadBundle.js @@ -17,11 +17,14 @@ export async function loadBundle(buffer) { } const manifest = JSON.parse(text('manifest.json')) if (manifest.kind === 'diff') return loadDiffBundle(files, manifest, text) - if (manifest.formatVersion !== 1 && manifest.formatVersion !== 2) { + // v3 adds the platform axis: screens live under screens// and each + // node carries a `captures` map. `capture` still mirrors the first platform, + // so everything downstream keeps working until withPlatform() swaps it. + if (![1, 2, 3].includes(manifest.formatVersion)) { throw new Error(`unsupported formatVersion ${manifest.formatVersion}`) } const map = JSON.parse(text('map.json')) - if (manifest.formatVersion === 2) { + if (manifest.formatVersion >= 2) { map.flows = Object.keys(files) .filter((n) => /^flows\/.+\.yaml$/.test(n)) .map((n) => { @@ -50,7 +53,7 @@ export async function loadBundle(buffer) { // (removed) nodes and edges are merged in so the map shows what disappeared. // Screenshot paths get side-prefixed keys ("head/screens/…") into `images`. function loadDiffBundle(files, manifest, text) { - if (manifest.formatVersion !== 1) { + if (![1, 2].includes(manifest.formatVersion)) { throw new Error(`unsupported diff formatVersion ${manifest.formatVersion}`) } const diff = JSON.parse(text('diff.json')) @@ -77,12 +80,18 @@ function loadDiffBundle(files, manifest, text) { screenshot: cap.screenshot ? `${side}/${cap.screenshot}` : null, states: (cap.states ?? []).map((s) => ({ ...s, screenshot: `${side}/${s.screenshot}` })), }) + // v2 diffs carry one capture per platform; side-prefix each of them so + // withPlatform() can swap the whole node over in one step + const sideCaptures = (n, side) => + n?.captures ? Object.fromEntries(Object.entries(n.captures).map(([p, c]) => [p, sideCapture(c, side)])) : null const baseById = new Map(baseMap.nodes.map((n) => [n.id, n])) const headIds = new Set(headMap.nodes.map((n) => n.id)) const nodes = headMap.nodes.map((n) => ({ ...n, capture: sideCapture(n.capture, 'head'), captureBase: baseById.has(n.id) ? sideCapture(baseById.get(n.id).capture, 'base') : null, + captures: sideCaptures(n, 'head'), + capturesBase: baseById.has(n.id) ? sideCaptures(baseById.get(n.id), 'base') : null, diff: nodeDiff.get(n.id) ?? null, stateDiff: stateDiff[n.id] ?? null, })) @@ -91,6 +100,8 @@ function loadDiffBundle(files, manifest, text) { ...b, capture: sideCapture(b.capture, 'base'), captureBase: null, + captures: sideCaptures(b, 'base'), + capturesBase: null, diff: nodeDiff.get(b.id) ?? { id: b.id, status: 'D', reason: 'route-removed' }, stateDiff: stateDiff[b.id] ?? null, }) @@ -270,3 +281,43 @@ export function flowResolution(map) { export function isInteractive(flow) { return (flow.steps ?? []).some((s) => ['tap', 'swipe', 'type', 'touch_path'].includes(s.action)) } + + +// The platforms a bundle carries, in capture order. A single-platform bundle +// (every v1/v2 map) reports the one platform its manifest names, so callers +// never have to special-case "before multi-platform". +export function platformsOf(bundle) { + const app = bundle?.manifest?.app + if (!app) return [] + if (Array.isArray(app.platforms) && app.platforms.length) return app.platforms + const label = app.platform ?? null + const platform = label === 'android-emulator' ? 'android' : label === 'ios-simulator' ? 'ios' : (label ?? 'ios') + return [{ platform, label, device: app.device ?? null }] +} + +// Point every node's `capture` at one platform's captures. Applied before the +// map/diff merge, so nothing downstream needs to know platforms exist — it +// keeps reading `capture` exactly as it always has. +export function withPlatform(bundle, platform) { + if (!bundle || !platform) return bundle + const nodes = bundle.map.nodes + if (!nodes.some((n) => n.captures)) return bundle // single-platform bundle + return { + ...bundle, + map: { + ...bundle.map, + nodes: nodes.map((n) => { + // a node captured on only one platform keeps the capture it has rather + // than blanking out — a screen that exists on iOS and not Android is + // better shown than hidden + const cap = n.captures?.[platform] + const capBase = n.capturesBase?.[platform] + return { + ...n, + ...(cap ? { capture: cap } : {}), + ...(n.capturesBase ? { captureBase: capBase ?? null } : {}), + } + }), + }, + } +} diff --git a/apps/visualiser/src/lib/useBundles.js b/apps/visualiser/src/lib/useBundles.js index 28688e3..c5b8eb9 100644 --- a/apps/visualiser/src/lib/useBundles.js +++ b/apps/visualiser/src/lib/useBundles.js @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo, useState } from 'react' -import { loadBundle, mergeBundles, sameApp } from './loadBundle' +import { loadBundle, mergeBundles, sameApp, platformsOf, withPlatform } from './loadBundle' // A viewer session holds up to two bundles: the Map (a plain .scrmap) and the // Changes overlay (a .diff.scrmap). When both describe the same app, Changes @@ -31,6 +31,7 @@ export function useBundles() { const [error, setError] = useState(null) const [demoAvailable, setDemoAvailable] = useState(false) const [gen, setGen] = useState(0) + const [platform, setPlatform] = useState(null) // null = whichever the bundle lists first const place = useCallback((loaded) => { if (loaded.diff) { @@ -152,29 +153,41 @@ export function useBundles() { } }, [open]) + // Platforms come from whichever bundle is loaded; the Map wins when both are, + // since it is the fuller one. Selecting happens BEFORE the merge so the merge, + // the diff annotations and every component below keep reading plain `capture`. + const platforms = useMemo(() => platformsOf(plain ?? changes), [plain, changes]) + const activePlatform = useMemo( + () => (platforms.some((p) => p.platform === platform) ? platform : platforms[0]?.platform ?? null), + [platforms, platform] + ) + const plainP = useMemo(() => withPlatform(plain, activePlatform), [plain, activePlatform]) + const changesP = useMemo(() => withPlatform(changes, activePlatform), [changes, activePlatform]) + // Is there a map to lay the overlay over? Only then does the Changes view get // the map's captures behind it; without one it still resolves what it can // from the diff's own base side. - const overlaid = useMemo(() => !!(plain && changes && sameApp(plain, changes)), [plain, changes]) + const overlaid = useMemo(() => !!(plainP && changesP && sameApp(plainP, changesP)), [plainP, changesP]) const merged = useMemo( - () => (changes ? mergeBundles(overlaid ? plain : null, changes) : null), - [plain, changes, overlaid] + () => (changesP ? mergeBundles(overlaid ? plainP : null, changesP) : null), + [plainP, changesP, overlaid] ) // what the graph renders for the current mode const bundle = useMemo(() => { if (mode === 'changes' && merged) return merged - if (plain) return plain + if (plainP) return plainP if (merged) { // Map view of a lone Changes bundle: the head graph, undecorated return { ...merged, diff: null, map: { ...merged.map, nodes: merged.map.nodes.filter((n) => n.diff?.status !== 'D') } } } return null - }, [mode, plain, merged]) + }, [mode, plainP, merged]) return { plain, changes, bundle, mode, setMode, hasChanges: !!changes, overlaid, + platforms, platform: activePlatform, setPlatform, open, closeChanges, loadDemo, busy, error, demoAvailable, gen, } } diff --git a/docs/diff-scrmap-format.md b/docs/diff-scrmap-format.md index dcd2c6b..361cc08 100644 --- a/docs/diff-scrmap-format.md +++ b/docs/diff-scrmap-format.md @@ -1,4 +1,4 @@ -# .diff.scrmap bundle format (v1) +# .diff.scrmap bundle format (v2) A `.diff.scrmap` file is a plain **zip** describing how an app's navigation map changed between two revisions — typically the base and head of a pull request. It follows git @@ -10,7 +10,7 @@ static analysis says the change could touch — not a full re-capture of the app ``` myapp-pr123.diff.scrmap (zip) -├── manifest.json # formatVersion: 1, kind: "diff", base/head/pr metadata +├── manifest.json # formatVersion: 1 (2 when multi-platform), kind: "diff", base/head/pr metadata ├── diff.json # the verdict: nodes/edges/states classified A/M/D ├── base/map.json # full graph of the base revision (map.json schema from screenmap v2) ├── head/map.json # full graph of the head revision @@ -131,3 +131,30 @@ The screenmap skill assembles diffs under `/.screenmap/out/diff// ├── suspects.json # diff-map.mjs suspects → capture work-list └── diff.json # diff-map.mjs pack ``` + + +## Platforms (`formatVersion: 2`) + +A diff captured on one platform is the v1 layout above, unchanged. One captured +on several becomes `formatVersion: 2`, gaining the same platform axis as a v3 +`.scrmap` — under each side rather than instead of it: + +``` +├── base/ +│ ├── map.json +│ └── screens/{ios,android}/*.png +└── head/ + ├── map.json + └── screens/{ios,android}/*.png +``` + +`manifest.app.platforms` lists them in capture order, and each node in a side's +`map.json` carries `captures` alongside a `capture` that mirrors the first +platform — so a v1 reader still renders the diff as a single-platform one. Each +side's `capture-status.json` is keyed by platform first. + +The diff verdicts themselves (`diff.json`: nodes, edges, states) stay +platform-independent: a screen is "changed" because the change set touches its +file or import closure, which is static analysis of one commit and says nothing +about which device it renders on. A state variant captured on either platform +counts as a state of that screen. diff --git a/docs/scrmap-format.md b/docs/scrmap-format.md index a2ee9a7..e5f5048 100644 --- a/docs/scrmap-format.md +++ b/docs/scrmap-format.md @@ -1,4 +1,4 @@ -# .scrmap bundle format (v2) +# .scrmap bundle format (v3) A `.scrmap` file is a plain **zip** containing everything needed to render an application's navigation map: the graph, screenshots, capture verdicts, and replayable flows. @@ -27,6 +27,49 @@ myapp-2026-08-08.scrmap (zip) (v1 bundles inlined JSON flows in `map.json`; viewers should keep reading them.) +## Platforms (`formatVersion: 3`) + +A bundle captured on one platform is exactly the v2 layout above and stays +`formatVersion: 2` — there is no platform axis to add. A bundle captured on +several becomes `formatVersion: 3`: + +``` +├── manifest.json # app.platforms lists them, in capture order +├── map.json # each node gains `captures` +└── screens/ + ├── ios/*.png + └── android/*.png +``` + +`manifest.app` keeps `platform` and `device` pointing at the **first** platform, +and every node keeps a `capture` mirroring that same platform, so a v2 reader +renders a v3 bundle as a single-platform map rather than failing or showing every +screen as missing. A v3-aware reader uses `app.platforms` and `node.captures`: + +```jsonc +"app": { + "platform": "ios-simulator", // first platform, for v2 readers + "device": "iPhone 17 Pro", // ditto + "platforms": [ + { "platform": "ios", "label": "ios-simulator", "device": "iPhone 17 Pro" }, + { "platform": "android", "label": "android-emulator", "device": "Pixel 7" } + ] +} +``` + +```jsonc +"captures": { // same shape as `capture`, one per platform + "ios": { "status": "ok", "screenshot": "screens/ios/Profile.png", "states": [...] }, + "android": { "status": "error-boundary", "note": "crashes on deep link", + "screenshot": "screens/android/Profile.png", "states": [] } +} +``` + +Status is per platform, which is the point: a screen that renders on iOS and +crashes on Android is one node with two verdicts, not two maps to compare by eye. +`capture-status.json` is keyed by platform first in a v3 bundle +(`{"android": {"": {…}}}`) and by route id in a v2 one. + ## manifest.json ```jsonc diff --git a/plugins/screenmap/skills/screenmap/SKILL.md b/plugins/screenmap/skills/screenmap/SKILL.md index 46d07c1..152afe1 100644 --- a/plugins/screenmap/skills/screenmap/SKILL.md +++ b/plugins/screenmap/skills/screenmap/SKILL.md @@ -1,16 +1,34 @@ --- name: screenmap -description: Generate a visual navigation map of an Expo app. Statically parses expo-router routes and links for full coverage, then deep-links through every screen in the iOS simulator capturing screenshots — including runtime states like bottom sheet snap points and modals — and renders a self-contained HTML map. Also diffs two revisions into a PR preview (.diff.scrmap) showing which screens/edges were added, removed, or changed. Use when the user asks to map an Expo/React Native app's navigation, screens, or routes, wants a visual sitemap of their app, or wants to preview/review what a PR changes on-screen. +description: Generate a visual navigation map of an Expo app. Statically parses expo-router routes and links for full coverage, then deep-links through every screen in the iOS simulator or Android emulator capturing screenshots — including runtime states like bottom sheet snap points and modals — and renders a self-contained HTML map. Both platforms can go into one map with a platform switcher. Also diffs two revisions into a PR preview (.diff.scrmap) showing which screens/edges were added, removed, or changed. Use when the user asks to map an Expo/React Native app's navigation, screens, or routes, wants a visual sitemap of their app, or wants to preview/review what a PR changes on-screen. --- # screenmap Produce a visual map of an Expo app's navigation: every expo-router route as a card with a screenshot, runtime state variants (bottom sheets at each snap point, modals), and navigation edges between screens. -**Arguments:** optional path to the Expo project (default: current working directory). `--static` = skip the simulator phases and render a screenshot-less map. `pr ` or `diff ..` = PR diff mode (see bottom). +**Arguments:** optional path to the Expo project (default: current working directory). `--static` = skip the device phases and render a screenshot-less map. `--platform ios|android|both` (default `ios`) = which device(s) to capture on. `pr ` or `diff ..` = PR diff mode (see bottom). **Working directory contract:** all outputs go to `/.screenmap/out/` — `graph.json`, `screens/*.png`, `flows/*.yaml` + `flows/*.meta.json`, `map.html`. Suggest adding `.screenmap/out/` to the project's `.gitignore` at the end. +**Platform contract:** with one platform, screenshots go to `screens/.png` as they always have. With `--platform both`, they go to `screens/ios/.png` and `screens/android/.png`, `capture-status.json` is keyed by platform first (`{"android": {"": …}}`), and the bundle carries both so the viewer gets an iOS / Android switcher. Run the device phases once per platform, all the way through, before starting the next — never interleave them. + +### Device command table + +Everything below that touches a device has a form per platform. `` is the iOS UDID or the Android serial (`adb devices`); with one device attached you can use `booted` on iOS and omit `-s ` on Android. + +| | iOS | Android | +|---|---|---| +| list devices | `xcrun simctl list devices booted` | `adb devices -l` | +| deep link | `xcrun simctl openurl ""` | `adb -s shell am start -a android.intent.action.VIEW -d ''` | +| screenshot to disk | `xcrun simctl io screenshot ` | `adb -s exec-out screencap -p > ` | +| relaunch | `xcrun simctl terminate ` then `launch` | `adb -s shell am force-stop ` then `monkey -p -c android.intent.category.LAUNCHER 1` | +| freeze status bar | `xcrun simctl status_bar override …` | SystemUI demo mode (see D3) | +| reach host Metro | works as-is | `adb -s reverse tcp:8081 tcp:8081` **first**, or nothing loads | +| taps / swipes | argent (`--udid `) | argent (`--udid ` — it takes an Android serial too) | + +Quote the Android deep link in single quotes: `adb shell` runs the string on the device, so an unquoted `&` in a query string backgrounds the command instead of passing it. + ## Flow recording (do this throughout Phases 4–5) Every interaction sequence you perform is recorded as a **replayable flow**, written at the moment you perform it — not reconstructed afterwards. Flows use the **argent flow format** (argent.swmansion.com — Software Mansion's agentic mobile toolkit): a `.yaml` argent flow plus a `.meta.json` cartography sidecar, both in `/.screenmap/out/flows/`. Anyone replays a flow headlessly, no LLM in the loop: `npx @swmansion/argent flow run .screenmap/out/flows/.yaml`. Full pair schema: `docs/scrmap-format.md` in the skill repo. @@ -59,6 +77,10 @@ If `--static` was requested, jump to Phase 6. ## Phase 3 — boot the app +Run this phase once per platform. + +**iOS** + 1. `xcrun simctl list devices booted` — check for a booted simulator. 2. Call the iOS simulator MCP `attach` action FIRST so the user can watch (harmless error if nothing is booted yet — boot/build, then retry attach). 3. Get the app running, preferring what already exists: @@ -70,14 +92,24 @@ If `--static` was requested, jump to Phase 6. - Expo Go: `xcrun simctl openurl booted "exp://127.0.0.1:8081/--/"` Take an MCP `screenshot` to confirm the app rendered (not a crash/error screen). Use whichever URL form worked for the rest of the run. +**Android** + +There is no Android equivalent of the iOS simulator MCP, so the user watches the emulator window itself — say so rather than promising a live panel. The `android-debugging` skill, if available, covers adb troubleshooting in more depth. + +1. `adb devices -l` — check for an attached device or a running emulator. If none, list AVDs with `emulator -list-avds` and start one in the background: `emulator -avd -no-snapshot -no-boot-anim &`. If `adb`/`emulator` are not on PATH, they are under `$ANDROID_HOME/platform-tools` and `$ANDROID_HOME/emulator`. +2. Wait for the boot to finish — `adb wait-for-device` only waits for adb to see it, so poll until `adb shell getprop sys.boot_completed` returns `1`, then dismiss the lock screen with `adb shell input keyevent 82`. Installing before that fails in ways that read as a broken APK. +3. **`adb reverse tcp:8081 tcp:8081`.** The emulator's `localhost` is the emulator; without the tunnel the app cannot reach Metro on the host and nothing will load. Redo it after any emulator restart. +4. Get the app running: start Metro in the background as above, then launch the installed dev build (`adb shell monkey -p -c android.intent.category.LAUNCHER 1`). If no dev build is installed, `npx expo run:android` (warn the user this builds and takes minutes). +5. **Verify deep linking before sweeping**, same as iOS: `adb shell am start -a android.intent.action.VIEW -d '://'`, then screenshot to disk and look at it. + ## Phase 4 — route sweep For each route in `graph.json` (substituting params from Phase 2; for `+not-found`, deep-link a garbage path like `/definitely-not-a-route`): -1. `open_url` (or `xcrun simctl openurl booted ""`) with the route's deep link. -2. Wait ~1–1.5s for the transition (MCP `wait`). Content screens that fetch over the network need 3–4s — a capture showing a spinner or loading skeleton means the wait was too short, not that the route is broken; the Phase 4b review catches these, and you re-capture with a longer wait. -3. Capture to disk: `xcrun simctl io booted screenshot /.screenmap/out/screens/.png` — use the exact `slug` from `graph.json`; the renderer depends on this naming. (MCP `screenshot` is for your own eyes only; it doesn't save a file.) -4. Every few routes, sanity-check via MCP `screenshot` that you're capturing real screens. If a route shows a red error screen, an error boundary, or redirected somewhere else, still keep the capture but note it for the final report. +1. Open the route's deep link (see the device command table). +2. Wait ~1–1.5s for the transition (MCP `wait`, or just `sleep`). Content screens that fetch over the network need 3–4s — a capture showing a spinner or loading skeleton means the wait was too short, not that the route is broken; the Phase 4b review catches these, and you re-capture with a longer wait. +3. Capture to disk at `/.screenmap/out/screens/.png` (or `screens//.png` when capturing both) — use the exact `slug` from `graph.json`; the renderer depends on this naming. (The iOS MCP `screenshot` is for your own eyes only; it doesn't save a file.) +4. Every few routes, sanity-check that you're capturing real screens — read a capture back with the Read tool, or on iOS take an MCP `screenshot`. If a route shows a red error screen, an error boundary, or redirected somewhere else, still keep the capture but note it for the final report. ### Phase 4b — review and recover (do not skip) @@ -132,11 +164,14 @@ These flows are what make edges *pinnable*: a nav flow tapping through a transit ## Phase 6 — pack, render, deliver ```bash -for f in /.screenmap/out/screens/*.png; do sips -Z 800 "$f" >/dev/null; done # downscale +# downscale (sips is macOS; on Linux use `mogrify -resize '800x800>' `) +for f in /.screenmap/out/screens/*.png; do sips -Z 800 "$f" >/dev/null; done node /scripts/pack-map.mjs # → .screenmap/out/-.scrmap bundle node /scripts/render-map.mjs /.screenmap/out/graph.json # static HTML fallback ``` +Capturing both platforms: downscale each `screens//` directory, then pass the platforms to the packer — `node .../pack-map.mjs --platforms ios,android`. The bundle then carries both and the viewer shows an iOS / Android switcher; report coverage per platform, since a screen can be fine on one and broken on the other. + The `.scrmap` bundle (zip: manifest.json + map.json + screens/) is the primary deliverable — see `docs/scrmap-format.md` in the skill repo. Open it in the **map viewer** (`apps/visualiser` in the skill repo, `npm run dev`, drag the bundle in): interactive graph, flow playback, click-to-copy replay commands. Send the bundle with SendUserFile; send `map.html` too as the no-tooling fallback (display: render). Report: routes captured / total, state variants captured, anything skipped (error screens, auth redirects, un-triggerable sheets), unresolved edges. Offer to publish as an Artifact (if so, load the artifact-design skill first and rebuild the page body-only per Artifact rules — don't publish the full-document HTML as-is). Suggest adding `.screenmap/out/` to `.gitignore`. ## Replay mode — `/screenmap replay ` @@ -147,7 +182,9 @@ Flows are argent YAML, so the primary replay is **headless**: npx @swmansion/argent flow run /.screenmap/out/flows/.yaml ``` -Run that first (it needs no LLM and reports pass/fail per step). Fall back to manual replay only when argent isn't installed and can't be (`npx` unavailable) or when the flow fails and the user wants a diagnosis: execute the YAML steps yourself — `open-url`/`wait` via `xcrun simctl`, taps/swipes via the simulator MCP using the sidecar's `target` labels as the source of truth (recorded coordinates are hints that may have drifted). Verify each step with an MCP screenshot; if a target can't be found in 3 attempts, stop and report which step failed and what the screen showed instead. Same safety rules as Phase 5: never trigger destructive or submitting controls. +Run that first (it needs no LLM and reports pass/fail per step; add `--device ` to pick a device, and argent takes an Android serial wherever it takes an iOS UDID). Fall back to manual replay only when argent isn't installed and can't be (`npx` unavailable) or when the flow fails and the user wants a diagnosis: execute the YAML steps yourself — `open-url`/`wait` via the device command table, taps/swipes via the simulator MCP (iOS) or `adb shell input tap ` in device pixels (Android), using the sidecar's `target` labels as the source of truth (recorded coordinates are hints that may have drifted). + +A flow recorded on one platform is not guaranteed to replay on the other: coordinates are normalized, but layouts, system chrome heights and back-navigation differ. Record per platform when you capture both, and name the platform in the sidecar's `device` field. Verify each step with an MCP screenshot; if a target can't be found in 3 attempts, stop and report which step failed and what the screen showed instead. Same safety rules as Phase 5: never trigger destructive or submitting controls. ## PR diff mode — `/screenmap pr ` or `/screenmap diff ..` @@ -174,6 +211,8 @@ not input to the classification. - **Native guard:** if changed files touch `ios/`, `android/`, `patches/`, or change native deps in `package.json`, warn the user that the installed dev build may not match both sides — JS-only diffs are the supported case. Proceed only if they accept. + A change under `ios/` only affects the iOS side and one under `android/` only the + Android side, so say which platform's captures to distrust rather than both. - The project must have a clean tree (or the user agrees to `git stash`). Remember the original ref; **restore it at the end, always** — even after failures. @@ -229,7 +268,16 @@ Boot the app (Phase 3), then freeze the status bar so both sides capture identic (clock noise otherwise pollutes every pixel comparison): ```bash +# iOS xcrun simctl status_bar booted override --time "9:41" --dataNetwork wifi --wifiMode active --wifiBars 3 --cellularMode active --cellularBars 4 --batteryState charged --batteryLevel 100 + +# Android — SystemUI demo mode is the equivalent +adb shell settings put global sysui_demo_allowed 1 +adb shell am broadcast -a com.android.systemui.demo -e command enter +adb shell am broadcast -a com.android.systemui.demo -e command clock -e hhmm 0941 +adb shell am broadcast -a com.android.systemui.demo -e command battery -e level 100 -e plugged false +adb shell am broadcast -a com.android.systemui.demo -e command network -e wifi show -e level 4 +adb shell am broadcast -a com.android.systemui.demo -e command notifications -e visible false ``` Then for each side in order **base → head**: @@ -251,6 +299,10 @@ Keep both sides comparable: same device, same account, same waits. ```bash for f in /{base,head}/screens/*.png; do sips -Z 800 "$f" >/dev/null; done node /scripts/diff-map.mjs pack --device "" + +# both platforms: screens live at //screens//, and +# --device takes one name per platform in the same order +node /scripts/diff-map.mjs pack --platforms ios,android --device "iPhone 17 Pro,Pixel 7" ``` Restore the original ref. Send the `.diff.scrmap` with SendUserFile; report the diff @@ -261,7 +313,7 @@ the map viewer overlays the diff on the full map, so unchanged screens keep thei screenshots (dimmed) and changed screens flip base⇄head in place (hover for a red changed-pixels render). -## Web fallback (no macOS simulator available, or user asks for web) +## Web fallback (no simulator or emulator available, or user asks for web) - Start `npx expo start --web`, confirm `http://localhost:8081/_sitemap` lists the same routes as the parse (good cross-check). - Capture each route with `npx playwright screenshot --viewport-size=390,844 "http://localhost:8081" /.screenmap/out/screens/.png` (needs `npx playwright install chromium` once; ask before installing). diff --git a/plugins/screenmap/skills/screenmap/scripts/diff-map.mjs b/plugins/screenmap/skills/screenmap/scripts/diff-map.mjs index 60212f7..34abe31 100644 --- a/plugins/screenmap/skills/screenmap/scripts/diff-map.mjs +++ b/plugins/screenmap/skills/screenmap/scripts/diff-map.mjs @@ -6,9 +6,14 @@ // reads /base/graph.json, head/graph.json, changed-files.txt // writes /suspects.json (the capture work-list) // -// node diff-map.mjs pack [--out ] +// node diff-map.mjs pack [--out ] [--platforms ios,android] // reads the above + pr.json (optional) + base|head/screens + base|head/capture-status.json (optional) // writes /diff.json and the .appmapdiff zip +// +// With --platforms naming more than one, screenshots are read from and +// written to /screens// and each node carries a +// `captures` map; with one (the default) the single-platform layout is +// unchanged, so existing bundles and viewers keep working. import { execFileSync } from 'node:child_process' import fs from 'node:fs' @@ -232,12 +237,24 @@ const edges = [] for (const [k, e] of headEdges) if (!baseEdges.has(k)) edges.push({ from: e.from, to: e.to, status: 'A', raw: e.raw ?? null, target: e.target ?? null }) for (const [k, e] of baseEdges) if (!headEdges.has(k)) edges.push({ from: e.from, to: e.to, status: 'D', raw: e.raw ?? null, target: e.target ?? null }) -const shots = (side) => { - const d = path.join(diffDir, side, 'screens') +const PLATFORM_LABELS = { ios: 'ios-simulator', android: 'android-emulator' } +const platforms = String(opts.platforms ?? 'ios').split(',').map((s) => s.trim()).filter(Boolean) +const multi = platforms.length > 1 +const shotDir = (side, platform) => path.join(diffDir, side, 'screens', ...(multi ? [platform] : [])) +const shots = (side, platform) => { + const d = shotDir(side, platform) return fs.existsSync(d) ? fs.readdirSync(d).filter((f) => /\.(png|jpe?g|webp)$/i.test(f)) : [] } -const baseShots = shots('base') -const headShots = shots('head') +// per-platform file lists, plus the union each side's state-diff works from: a +// variant captured on either platform is a variant of that screen +const shotsByPlatform = { base: {}, head: {} } +for (const side of ['base', 'head']) for (const pf of platforms) shotsByPlatform[side][pf] = shots(side, pf) +const allShots = (side) => [...new Set(platforms.flatMap((pf) => shotsByPlatform[side][pf]))] +const baseShots = allShots('base') +const headShots = allShots('head') +// allShots() dedupes by filename across platforms because a state variant is a +// state of the screen wherever it was captured; the packed file count is the sum +const countShots = (side) => platforms.reduce((n, pf) => n + shotsByPlatform[side][pf].length, 0) const annotated = suspects.capture.map(({ side, slug, urlPath, ...n }) => ({ ...n, ...noteFor(n.id) })) @@ -295,27 +312,36 @@ const diff = { } fs.writeFileSync(path.join(diffDir, 'diff.json'), JSON.stringify(diff, null, 2)) -// per-side map.json, same node mapping as pack-map.mjs -const sideMap = (graph, side, shotFiles) => { - const captureStatus = readJson(path.join(diffDir, side, 'capture-status.json'), {}) +// per-side map.json, same node mapping as pack-map.mjs. With several platforms +// each node carries a `captures` map and `capture` mirrors the first platform, +// so a viewer that predates multi-platform still renders the side. +const captureOf = (r, cs, shotFiles, prefix) => { + const baseShot = shotFiles.find((f) => f.replace(/\.\w+$/, '') === r.slug) + const stateShots = shotFiles + .filter((f) => f.startsWith(r.slug + '--')) + .map((f) => ({ name: f.replace(/\.\w+$/, '').slice(r.slug.length + 2), screenshot: prefix + f })) + .sort((a, b) => a.name.localeCompare(b.name)) + return { + status: cs.status ?? (baseShot ? 'ok' : 'missing'), + note: cs.note ?? null, + needsNavigation: cs.needsNavigation ?? false, + screenshot: baseShot ? prefix + baseShot : null, + states: stateShots, + } +} +const sideMap = (graph, side) => { + const raw = readJson(path.join(diffDir, side, 'capture-status.json'), {}) + // per-platform status when multi, flat when not + const statusFor = (pf) => (multi ? raw[pf] ?? {} : raw) const nodes = graph.routes.map((r) => { - const cs = captureStatus[r.id] ?? {} - const baseShot = shotFiles.find((f) => f.replace(/\.\w+$/, '') === r.slug) - const stateShots = shotFiles - .filter((f) => f.startsWith(r.slug + '--')) - .map((f) => ({ name: f.replace(/\.\w+$/, '').slice(r.slug.length + 2), screenshot: 'screens/' + f })) - .sort((a, b) => a.name.localeCompare(b.name)) + const per = Object.fromEntries(platforms.map((pf) => [pf, + captureOf(r, statusFor(pf)[r.id] ?? {}, shotsByPlatform[side][pf], multi ? `screens/${pf}/` : 'screens/')])) return { id: r.id, urlPath: r.urlPath, file: r.file ?? null, slug: r.slug, group: r.layoutDir ?? '', navigator: r.navigator ?? null, params: r.params ?? [], presentation: r.presentation ?? null, stateHints: r.stateHints ?? [], - capture: { - status: cs.status ?? (baseShot ? 'ok' : 'missing'), - note: cs.note ?? null, - needsNavigation: cs.needsNavigation ?? false, - screenshot: baseShot ? 'screens/' + baseShot : null, - states: stateShots, - }, + capture: per[platforms[0]], + ...(multi ? { captures: per } : {}), } }) return { nodes, edges: graph.edges ?? [], flows: [] } @@ -327,16 +353,20 @@ const sideMeta = (graph, side) => ({ commit: pr?.[side + 'Sha'] ?? null, generatedAt: graph.generatedAt ?? null, }) +// --device may name one device or, with several platforms, a comma-separated +// list in the same order as --platforms +const deviceNames = String(opts.device ?? '').split(',').map((s) => s.trim()) const manifest = { - formatVersion: 1, + formatVersion: multi ? 2 : 1, kind: 'diff', - generator: 'expo-map/2.0', + generator: 'screenmap/2.0', app: { name: appName, scheme: headGraph.scheme ?? null, - platform: 'ios-simulator', - device: opts.device ?? null, + platform: PLATFORM_LABELS[platforms[0]] ?? platforms[0], + device: deviceNames[0] || null, mode: headGraph.mode ?? null, + ...(multi ? { platforms: platforms.map((pf, i) => ({ platform: pf, label: PLATFORM_LABELS[pf] ?? pf, device: deviceNames[i] || null })) } : {}), }, base: sideMeta(baseGraph, 'base'), head: sideMeta(headGraph, 'head'), @@ -348,10 +378,14 @@ const stage = fs.mkdtempSync(path.join(diffDir, '.pack-')) try { fs.writeFileSync(path.join(stage, 'manifest.json'), JSON.stringify(manifest, null, 2)) fs.writeFileSync(path.join(stage, 'diff.json'), JSON.stringify(diff, null, 2)) - for (const [side, graph, files] of [['base', baseGraph, baseShots], ['head', headGraph, headShots]]) { + for (const [side, graph] of [['base', baseGraph], ['head', headGraph]]) { fs.mkdirSync(path.join(stage, side, 'screens'), { recursive: true }) - fs.writeFileSync(path.join(stage, side, 'map.json'), JSON.stringify(sideMap(graph, side, files), null, 2)) - for (const f of files) fs.copyFileSync(path.join(diffDir, side, 'screens', f), path.join(stage, side, 'screens', f)) + fs.writeFileSync(path.join(stage, side, 'map.json'), JSON.stringify(sideMap(graph, side), null, 2)) + for (const pf of platforms) { + const dest = path.join(stage, side, 'screens', ...(multi ? [pf] : [])) + fs.mkdirSync(dest, { recursive: true }) + for (const f of shotsByPlatform[side][pf]) fs.copyFileSync(path.join(shotDir(side, pf), f), path.join(dest, f)) + } } const slug = pr?.number ? `pr${pr.number}` : `${(pr?.baseSha ?? 'base').slice(0, 7)}..${(pr?.headSha ?? 'head').slice(0, 7)}` const outPath = path.resolve(opts.out ?? path.join(diffDir, `${appName}-${slug}.appmapdiff`)) @@ -359,7 +393,7 @@ try { execFileSync('zip', ['-r', '-q', outPath, 'manifest.json', 'diff.json', 'base', 'head'], { cwd: stage }) const kb = Math.round(fs.statSync(outPath).size / 1024) const n = (s) => diff.nodes.filter((c) => c.status === s).length - console.log(`wrote ${outPath} (${kb} KB) — nodes: ${n('A')}A/${n('M')}M/${n('D')}D · edges: ${edges.filter((e) => e.status === 'A').length}A/${edges.filter((e) => e.status === 'D').length}D · states: ${states.length} · shots: ${baseShots.length} base + ${headShots.length} head`) + console.log(`wrote ${outPath} (${kb} KB) — nodes: ${n('A')}A/${n('M')}M/${n('D')}D · edges: ${edges.filter((e) => e.status === 'A').length}A/${edges.filter((e) => e.status === 'D').length}D · states: ${states.length} · shots: ${countShots('base')} base + ${countShots('head')} head`) } finally { fs.rmSync(stage, { recursive: true, force: true }) } diff --git a/plugins/screenmap/skills/screenmap/scripts/pack-map.mjs b/plugins/screenmap/skills/screenmap/scripts/pack-map.mjs index e293170..01028c6 100644 --- a/plugins/screenmap/skills/screenmap/scripts/pack-map.mjs +++ b/plugins/screenmap/skills/screenmap/scripts/pack-map.mjs @@ -1,6 +1,10 @@ #!/usr/bin/env node // Packs an .screenmap/out/ working directory into a distributable .scrmap bundle (zip). -// Usage: node pack-map.mjs [projectRoot] [--out ] +// Usage: node pack-map.mjs [projectRoot] [--out ] [--platforms ios,android] +// +// Screens come from .screenmap/out/screens/. With several platforms they come +// from .screenmap/out/screens// instead, each node gains a `captures` +// map, and the bundle is formatVersion 3. // See docs/scrmap-format.md for the format contract. import { execFileSync } from 'node:child_process' @@ -10,10 +14,14 @@ import path from 'node:path' const args = process.argv.slice(2) let projectRoot = '.' let outPath = null +let platforms = ['ios'] for (let i = 0; i < args.length; i++) { if (args[i] === '--out') outPath = args[++i] + else if (args[i] === '--platforms') platforms = args[++i].split(',').map((s) => s.trim()).filter(Boolean) else projectRoot = args[i] } +const multi = platforms.length > 1 +const PLATFORM_LABELS = { ios: 'ios-simulator', android: 'android-emulator' } projectRoot = path.resolve(projectRoot) const base = path.join(projectRoot, '.screenmap', 'out') @@ -41,20 +49,37 @@ const flows = flowFiles }) .filter(Boolean) -const shotsDir = path.join(base, 'screens') -const shotFiles = fs.existsSync(shotsDir) - ? fs.readdirSync(shotsDir).filter((f) => /\.(png|jpe?g|webp)$/i.test(f)) - : [] +const shotsDirFor = (pf) => path.join(base, 'screens', ...(multi ? [pf] : [])) +const shotsByPlatform = Object.fromEntries(platforms.map((pf) => { + const d = shotsDirFor(pf) + return [pf, fs.existsSync(d) ? fs.readdirSync(d).filter((f) => /\.(png|jpe?g|webp)$/i.test(f)) : []] +})) const appName = path.basename(graph.projectRoot ?? projectRoot) -const nodes = graph.routes.map((r) => { - const cs = captureStatus[r.id] ?? {} +// capture-status.json is keyed by platform when several are packed, by route +// id when one is — the same shape the CI packer writes +const statusFor = (pf) => (multi ? captureStatus[pf] ?? {} : captureStatus) +const captureOf = (r, pf) => { + const cs = statusFor(pf)[r.id] ?? {} + const shotFiles = shotsByPlatform[pf] + const prefix = multi ? `screens/${pf}/` : 'screens/' const baseShot = shotFiles.find((f) => f.replace(/\.\w+$/, '') === r.slug) const states = shotFiles .filter((f) => f.startsWith(r.slug + '--')) - .map((f) => ({ name: f.replace(/\.\w+$/, '').slice(r.slug.length + 2), screenshot: 'screens/' + f })) + .map((f) => ({ name: f.replace(/\.\w+$/, '').slice(r.slug.length + 2), screenshot: prefix + f })) .sort((a, b) => a.name.localeCompare(b.name)) + return { + status: cs.status ?? (baseShot ? 'ok' : 'missing'), + note: cs.note ?? null, + needsNavigation: cs.needsNavigation ?? false, + screenshot: baseShot ? prefix + baseShot : null, + states, + } +} + +const nodes = graph.routes.map((r) => { + const per = Object.fromEntries(platforms.map((pf) => [pf, captureOf(r, pf)])) return { id: r.id, urlPath: r.urlPath, @@ -65,27 +90,28 @@ const nodes = graph.routes.map((r) => { params: r.params ?? [], presentation: r.presentation ?? null, stateHints: r.stateHints ?? [], - capture: { - status: cs.status ?? (baseShot ? 'ok' : 'missing'), - note: cs.note ?? null, - needsNavigation: cs.needsNavigation ?? false, - screenshot: baseShot ? 'screens/' + baseShot : null, - states, - }, + // `capture` mirrors the first platform so a pre-multi-platform viewer still + // renders the map; `captures` is the full set + capture: per[platforms[0]], + ...(multi ? { captures: per } : {}), } }) const map = { nodes, edges: graph.edges ?? [], flows: [] } +// A flow sidecar records the device it was recorded on; with several platforms +// prefer the one whose sidecar names that platform, else fall back to any. +const deviceFor = (pf) => flows.find((f) => f.platform === pf && f.device)?.device ?? (multi ? null : flows.find((f) => f.device)?.device ?? null) const manifest = { - formatVersion: 2, + formatVersion: multi ? 3 : 2, flowFormat: 'argent', // flows/*.yaml runnable via `argent flow run` generator: 'screenmap/2.0', app: { name: appName, scheme: graph.scheme ?? null, - platform: 'ios-simulator', - device: flows.find((f) => f.device)?.device ?? null, + platform: PLATFORM_LABELS[platforms[0]] ?? platforms[0], + device: deviceFor(platforms[0]) ?? (multi ? null : flows.find((f) => f.device)?.device ?? null), mode: graph.mode ?? null, + ...(multi ? { platforms: platforms.map((pf) => ({ platform: pf, label: PLATFORM_LABELS[pf] ?? pf, device: deviceFor(pf) })) } : {}), }, generatedAt: new Date().toISOString(), } @@ -95,7 +121,11 @@ try { fs.writeFileSync(path.join(stage, 'manifest.json'), JSON.stringify(manifest, null, 2)) fs.writeFileSync(path.join(stage, 'map.json'), JSON.stringify(map, null, 2)) fs.mkdirSync(path.join(stage, 'screens')) - for (const f of shotFiles) fs.copyFileSync(path.join(shotsDir, f), path.join(stage, 'screens', f)) + for (const pf of platforms) { + const dest = path.join(stage, 'screens', ...(multi ? [pf] : [])) + fs.mkdirSync(dest, { recursive: true }) + for (const f of shotsByPlatform[pf]) fs.copyFileSync(path.join(shotsDirFor(pf), f), path.join(dest, f)) + } fs.mkdirSync(path.join(stage, 'flows')) for (const f of flowFiles) fs.copyFileSync(path.join(flowsDir, f), path.join(stage, 'flows', f)) @@ -104,7 +134,8 @@ try { fs.rmSync(outPath, { force: true }) execFileSync('zip', ['-r', '-q', outPath, 'manifest.json', 'map.json', 'screens', 'flows'], { cwd: stage }) const kb = Math.round(fs.statSync(outPath).size / 1024) - console.log(`wrote ${outPath} (${kb} KB, ${nodes.length} nodes, ${map.edges.length} edges, ${flows.length} flows, ${shotFiles.length} screenshots)`) + const shotCount = platforms.reduce((n, pf) => n + shotsByPlatform[pf].length, 0) + console.log(`wrote ${outPath} (${kb} KB, ${nodes.length} nodes, ${map.edges.length} edges, ${flows.length} flows, ${shotCount} screenshots across ${platforms.join('+')})`) } finally { fs.rmSync(stage, { recursive: true, force: true }) } From 113178c764857b2a07eab52fa22892d7804df7b4 Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 21:14:39 +0200 Subject: [PATCH 02/11] Install the emulator packages the Android step actually uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The step assumed the runner image carried platform-tools and emulator, and called adb and avdmanager before checking either existed. GitHub's ubuntu images ship platform-tools, but emulator is not guaranteed, and a missing one surfaced as a bare "command not found" several steps later. sdkmanager now installs platform-tools, emulator and the system image together — a no-op when they are already there — and the two binaries are checked with a message that names what is missing and where it was looked for. Co-Authored-By: Claude Opus 5 --- action.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index 2e6b6e9..2ebd1fc 100644 --- a/action.yml +++ b/action.yml @@ -199,17 +199,26 @@ runs: export PATH="$sdk/platform-tools:$sdk/emulator:$sdk/cmdline-tools/latest/bin:$PATH" echo "$sdk/platform-tools" >> "$GITHUB_PATH" echo "$sdk/emulator" >> "$GITHUB_PATH" - # a device already attached (self-hosted runner, or a previous step) wins - if adb devices | awk 'NR>1 && $2=="device"' | grep -q .; then + command -v sdkmanager >/dev/null || { echo "::error::sdkmanager not found under $sdk/cmdline-tools/latest/bin"; exit 1; } + # a device already attached (self-hosted runner, or an earlier step) wins, + # and needs none of the SDK packages below + if command -v adb >/dev/null && adb devices | awk 'NR>1 && $2=="device"' | grep -q .; then echo "using the attached device: $(adb devices | awk 'NR>1 && $2=="device" {print $1}' | head -1)" else + # Install what we actually use rather than trusting the runner image to + # carry it: GitHub's ubuntu images ship platform-tools, but `emulator` + # is not guaranteed, and a missing one surfaces as a bare + # "command not found" three steps later. Already-present packages are + # a no-op, so this costs nothing on a warm image. + yes | sdkmanager --install "platform-tools" "emulator" "${{ inputs.android_system_image }}" >/dev/null + command -v avdmanager >/dev/null || { echo "::error::avdmanager not found under $sdk/cmdline-tools/latest/bin"; exit 1; } if ! avdmanager list avd -c | grep -qx "${{ inputs.avd }}"; then - yes | sdkmanager --install "${{ inputs.android_system_image }}" >/dev/null echo no | avdmanager create avd -n "${{ inputs.avd }}" -k "${{ inputs.android_system_image }}" --force fi - # the driver (lib/android.mjs) boots and waits for sys.boot_completed; + # the driver (lib/android.mjs) boots it and waits for sys.boot_completed; # starting it here would just duplicate that logic echo "AVD ${{ inputs.avd }} ready — the run boots it" + "$sdk/emulator/emulator" -version | head -1 || true fi - name: Dev client (EAS or provided) From 55b95dc25fb22e254fb4beb5fc7a6fb106b0443c Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 21:15:38 +0200 Subject: [PATCH 03/11] Record that screenmaps paths carry no platform A baseline publishes to main/latest.scrmap and a PR restores from it, with no platform anywhere in the path. In a repo that already maps iOS, an Android baseline overwrites the iOS map and the next iOS PR run restores the wrong device's captures. The screenmaps_branch input isolates a first Android experiment, which is enough to test with and not an answer to keep. Written down with the two ways it could resolve and the back-compat constraint either one has to meet. Co-Authored-By: Claude Opus 5 --- TODOS.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/TODOS.md b/TODOS.md index 30f6407..5356710 100644 --- a/TODOS.md +++ b/TODOS.md @@ -54,6 +54,36 @@ which platforms they hold (`platformsIn()`), a side that is not there reports input whose platform does not match the one it is being merged as rather than silently mislabelling it. +## The screenmaps branch has no platform in its paths + +Found while planning the first Android CI run, on 2026-09-08. Not fixed. + +A baseline publishes to `main/.scrmap` and `main/latest.scrmap`, and a PR +run restores `main/.scrmap` falling back to `main/latest.scrmap`. +Nothing in either path names a platform. So in a repo that already maps iOS, an +Android baseline overwrites the iOS map, and the next iOS PR run restores an +Android baseline: the diff's static verdicts still hold, but every base-side +capture is missing or belongs to the wrong device. + +The workaround for a first experiment is the existing `screenmaps_branch` input +— point the Android runs at their own branch and nothing collides. That is fine +for a trial and wrong as an answer, because a repo mapping both platforms wants +one map, not two branches. + +Deciding it properly means picking where the merge happens: + +1. **Per-platform paths plus a merged map.** Each platform publishes + `main//latest.scrmap`; a merge job writes `main/latest.scrmap` from + them. PR runs restore the merged one, so the viewer keeps getting a single + bundle. Costs a third job and makes `latest.scrmap` a derived artifact. +2. **Per-platform paths only**, with the viewer loading two bundles. Cheaper in + CI, but it pushes the merge onto every reader and the PR comment can only + show one. + +Whichever wins, existing repos have an unprefixed `main/latest.scrmap` that must +keep resolving, or every repo loses its baseline on upgrade — the same hazard as +the `appmaps` -> `screenmaps` branch rename. + ## OCR recall on the Linux lane Measured on 2026-09-01 against six real captures (downscaled 368x800) plus a From 10dde16e995c183573bca99ddac4a0d315ec4fff Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 21:25:17 +0200 Subject: [PATCH 04/11] Stop pipefail failing the SDK install that succeeded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub runs `shell: bash` with -eo pipefail. `yes | sdkmanager` therefore fails the step the moment sdkmanager stops reading and SIGPIPE kills `yes` — even on a clean install. The first real ubuntu run died here, on the line after adb started its daemon, with nothing but "yes: standard output: Broken pipe". Take sdkmanager's own status out of PIPESTATUS, and keep its output so a real install failure says something instead of being swallowed by >/dev/null. Co-Authored-By: Claude Opus 5 --- action.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 2ebd1fc..0646a3c 100644 --- a/action.yml +++ b/action.yml @@ -210,7 +210,20 @@ runs: # is not guaranteed, and a missing one surfaces as a bare # "command not found" three steps later. Already-present packages are # a no-op, so this costs nothing on a warm image. - yes | sdkmanager --install "platform-tools" "emulator" "${{ inputs.android_system_image }}" >/dev/null + # + # `yes |` needs care: GitHub runs `shell: bash` with -eo pipefail, and + # `yes` is killed by SIGPIPE the moment sdkmanager stops reading, so + # pipefail reports the whole pipeline as failed even on a clean + # install. Take sdkmanager's own status out of PIPESTATUS instead. + set +o pipefail + yes 2>/dev/null | sdkmanager --install "platform-tools" "emulator" "${{ inputs.android_system_image }}" >/tmp/sdkmanager.log 2>&1 + rc=${PIPESTATUS[1]} + set -o pipefail + if [ "$rc" -ne 0 ]; then + tail -30 /tmp/sdkmanager.log + echo "::error::sdkmanager failed to install platform-tools/emulator/${{ inputs.android_system_image }}" + exit 1 + fi command -v avdmanager >/dev/null || { echo "::error::avdmanager not found under $sdk/cmdline-tools/latest/bin"; exit 1; } if ! avdmanager list avd -c | grep -qx "${{ inputs.avd }}"; then echo no | avdmanager create avd -n "${{ inputs.avd }}" -k "${{ inputs.android_system_image }}" --force From 59569d18e95e42b5e9494b0f7e182933815b8343 Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 21:46:12 +0200 Subject: [PATCH 05/11] Give the emulator the libraries it needs, and fail where the cause is MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first ubuntu run got all the way through an EAS Android build and then died on "no Android device connected and no AVD defined". The AVD was fine. The emulator binary could not start at all: qemu-system-x86_64: error while loading shared libraries: libpulse.so.0: cannot open shared object file GitHub's ubuntu images do not carry libpulse0, so the Linux dependency step now installs it for android runs, along with the X libs different system images pull in — those one at a time and best-effort, since the names drift between ubuntu releases and one missing optional package should not fail a run. Two diagnosis fixes so this cannot mislead again. The `emulator -version` check was `|| true`, which hid the broken binary and let the failure surface seventeen minutes and one EAS build later, pointing at the wrong thing; it is now a hard gate that prints qemu's own complaint. And listAvds() returned an empty array both when the emulator failed to run and when there were genuinely no AVDs, so it now reports which, and ensureBooted says so. Co-Authored-By: Claude Opus 5 --- action.yml | 41 +++++++++++++++++++++++++++++++------- action/cli/lib/android.mjs | 16 ++++++++++++--- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/action.yml b/action.yml index 0646a3c..286bf2a 100644 --- a/action.yml +++ b/action.yml @@ -133,14 +133,31 @@ runs: echo "Android SDK: ${ANDROID_HOME:-$ANDROID_SDK_ROOT}" fi - # Screen text drives the landing checks, deep-link verification and system-alert - # dismissal. macOS uses Apple Vision (compiled on demand by lib/ocr.mjs); Linux - # has no Vision, so without tesseract every one of those checks silently goes - # dark and a drifted flow stops being detectable. - - name: Install OCR backend (Linux) + # Two things Linux needs that macOS brings in the box. + # + # OCR: screen text drives the landing checks, deep-link verification and + # system-alert dismissal. macOS uses Apple Vision (compiled on demand by + # lib/ocr.mjs); Linux has no Vision, so without tesseract every one of those + # checks silently goes dark and a drifted flow stops being detectable. + # + # libpulse0: the SDK's qemu binary links against it and GitHub's ubuntu + # images do not carry it, so `emulator` cannot start at all — it fails with + # "libpulse.so.0: cannot open shared object file". The rest are the X libs + # different system images pull in; they are installed one at a time and + # best-effort because the names drift between ubuntu releases (libasound2 vs + # libasound2t64) and one missing optional package should not fail the run. + - name: Install Linux dependencies if: runner.os == 'Linux' shell: bash - run: sudo apt-get update -qq && sudo apt-get install -y -qq tesseract-ocr imagemagick + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq tesseract-ocr imagemagick + if [ "${{ inputs.platform }}" = "android" ]; then + sudo apt-get install -y -qq libpulse0 + for pkg in libnss3 libxcursor1 libxdamage1 libxcomposite1 libxi6 libxtst6 libasound2t64 libasound2; do + sudo apt-get install -y -qq "$pkg" 2>/dev/null || true + done + fi - name: Install screenmap-ci shell: bash @@ -231,7 +248,17 @@ runs: # the driver (lib/android.mjs) boots it and waits for sys.boot_completed; # starting it here would just duplicate that logic echo "AVD ${{ inputs.avd }} ready — the run boots it" - "$sdk/emulator/emulator" -version | head -1 || true + # Gate on the emulator actually running. This used to be `|| true`, + # which hid a qemu that could not load libpulse: the run went on to + # spend seventeen minutes on an EAS build and only then failed, with + # "no AVD defined" — the one thing that was not wrong. Fail here, where + # the message can name the real cause. + if ! "$sdk/emulator/emulator" -version >/tmp/emulator-version.log 2>&1; then + tail -5 /tmp/emulator-version.log + echo "::error::the emulator binary cannot run on this runner (see above — a missing shared library is the usual cause)" + exit 1 + fi + head -1 /tmp/emulator-version.log fi - name: Dev client (EAS or provided) diff --git a/action/cli/lib/android.mjs b/action/cli/lib/android.mjs index fb0f091..05726c0 100644 --- a/action/cli/lib/android.mjs +++ b/action/cli/lib/android.mjs @@ -75,10 +75,19 @@ export function listBooted() { }) } +// Defined AVDs, plus why the answer is empty when it is. `emulator -list-avds` +// failing and there genuinely being no AVDs look identical from the outside, +// and the two need very different fixes — a runner whose qemu cannot load its +// shared libraries reported "no AVD defined" for seventeen minutes before this +// distinction existed. export function listAvds() { const bin = emulatorPath() - if (!bin) return [] - try { return sh(bin, ['-list-avds']).split('\n').map((s) => s.trim()).filter(Boolean) } catch { return [] } + if (!bin) return { avds: [], error: 'no emulator binary found (set ANDROID_HOME)' } + try { + return { avds: sh(bin, ['-list-avds']).split('\n').map((s) => s.trim()).filter(Boolean), error: null } + } catch (e) { + return { avds: [], error: `emulator -list-avds failed: ${(e.stderr || e.message || '').trim().split('\n').slice(-2).join(' ')}` } + } } // Wait for the device to finish booting. `wait-for-device` only waits for adb @@ -112,7 +121,8 @@ export async function ensureBooted(config) { } const bin = emulatorPath() if (!bin) throw new Error('no Android device connected and no emulator binary found (set ANDROID_HOME)') - const avds = listAvds() + const { avds, error } = listAvds() + if (error) throw new Error(`no Android device connected, and the emulator could not be queried — ${error}`) if (!avds.length) throw new Error('no Android device connected and no AVD defined — create one with avdmanager') const avd = avds.find((a) => a === config.device) ?? avds[0] log(`booting AVD ${avd}`) From caed025a348555622cf89ce990d157425a5eba5a Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 21:49:53 +0200 Subject: [PATCH 06/11] Make avdmanager and the emulator agree on where AVDs live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run three created the AVD, printed the emulator version, and then failed a step later with "no AVD defined". Both were true: avdmanager wrote one and `emulator -list-avds` could not see it, because the two resolve the AVD directory through different chains — $ANDROID_AVD_HOME, then $ANDROID_SDK_HOME/.android/avd for one and $HOME/.android/avd for the other — and a runner that sets ANDROID_SDK_HOME sends them to different places. Pin ANDROID_AVD_HOME for both and carry it to the boot step through GITHUB_ENV. Then verify it rather than assume: the step now requires the AVD to appear in `emulator -list-avds` before declaring it ready, and on failure prints the AVD home, its contents and what the emulator actually reported. Creating an AVD the emulator cannot see is this step's most likely failure and the next step can only describe it as the thing that is not wrong. Co-Authored-By: Claude Opus 5 --- action.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 286bf2a..ae41956 100644 --- a/action.yml +++ b/action.yml @@ -217,6 +217,16 @@ runs: echo "$sdk/platform-tools" >> "$GITHUB_PATH" echo "$sdk/emulator" >> "$GITHUB_PATH" command -v sdkmanager >/dev/null || { echo "::error::sdkmanager not found under $sdk/cmdline-tools/latest/bin"; exit 1; } + # Pin where AVDs live. avdmanager and emulator each resolve this from a + # different chain ($ANDROID_AVD_HOME, then $ANDROID_SDK_HOME/.android/avd + # for one and $HOME/.android/avd for the other), so on a runner that sets + # ANDROID_SDK_HOME they write and read different directories: the AVD is + # created successfully and `emulator -list-avds` still comes back empty. + # Setting it explicitly makes both agree, and GITHUB_ENV carries it to + # the step that actually boots the device. + export ANDROID_AVD_HOME="$HOME/.android/avd" + mkdir -p "$ANDROID_AVD_HOME" + echo "ANDROID_AVD_HOME=$ANDROID_AVD_HOME" >> "$GITHUB_ENV" # a device already attached (self-hosted runner, or an earlier step) wins, # and needs none of the SDK packages below if command -v adb >/dev/null && adb devices | awk 'NR>1 && $2=="device"' | grep -q .; then @@ -247,7 +257,6 @@ runs: fi # the driver (lib/android.mjs) boots it and waits for sys.boot_completed; # starting it here would just duplicate that logic - echo "AVD ${{ inputs.avd }} ready — the run boots it" # Gate on the emulator actually running. This used to be `|| true`, # which hid a qemu that could not load libpulse: the run went on to # spend seventeen minutes on an EAS build and only then failed, with @@ -259,6 +268,17 @@ runs: exit 1 fi head -1 /tmp/emulator-version.log + # …and on it agreeing that the AVD exists. Creating one successfully + # and having the emulator not see it is the failure this step is most + # prone to, and one the next step can only report as "no AVD defined". + if ! "$sdk/emulator/emulator" -list-avds | grep -qx "${{ inputs.avd }}"; then + echo "ANDROID_AVD_HOME=$ANDROID_AVD_HOME" + ls -la "$ANDROID_AVD_HOME" 2>&1 | head -20 + echo "emulator -list-avds says: $("$sdk/emulator/emulator" -list-avds 2>&1 | tr '\n' ' ')" + echo "::error::AVD '${{ inputs.avd }}' was created but the emulator cannot see it" + exit 1 + fi + echo "AVD ${{ inputs.avd }} ready — the run boots it" fi - name: Dev client (EAS or provided) From f8ded1b988f98c33faf14a1d6f89af686b308329 Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 21:57:00 +0200 Subject: [PATCH 07/11] Keep Android's "isn't responding" dialog out of every capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first green Android baseline captured all eight screens of the app behind a grey scrim reading "Pixel Launcher isn't responding". An emulator on software rendering is slow enough to trip Android's watchdog, the dialog is modal, and nothing dismisses it — so it appears in every capture taken after it, the same poisoned tail a sticky error boundary causes. Two layers, matching how the iOS side handles system alerts. hide_error_dialogs (plus anr_show_background) stops the system drawing them at all, set right after sys.boot_completed alongside the keyguard dismissal. And ALERT_HINTS learns the ANR wording so dismissAlert() can clear one that still gets through, ordered so "Wait" is chosen over "Close app" — the app under test is what is being mapped, and killing it ends the run. Co-Authored-By: Claude Opus 5 --- action/cli/lib/android.mjs | 16 ++++++++++++++++ action/cli/lib/ocr.mjs | 10 ++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/action/cli/lib/android.mjs b/action/cli/lib/android.mjs index 05726c0..be72248 100644 --- a/action/cli/lib/android.mjs +++ b/action/cli/lib/android.mjs @@ -104,6 +104,7 @@ async function waitBootComplete(id, timeoutMs = 300000) { // capture would otherwise be the lock screen adbOk(id, ['shell', 'input', 'keyevent', '82']) adbOk(id, ['shell', 'wm', 'dismiss-keyguard']) + quietSystemDialogs(id) return true } await sleep(2000) @@ -140,6 +141,21 @@ export async function ensureBooted(config) { throw new Error(`AVD ${avd} did not come up within 5 minutes`) } +// An emulator on software rendering is slow enough that the launcher and the app +// itself trip Android's "isn't responding" watchdog. The dialog is modal, it is +// drawn over whatever is on screen, and nothing dismisses it — so it lands in +// every remaining capture of the run. The first green Android baseline came back +// with all eight screens behind a grey scrim reading "Pixel Launcher isn't +// responding". This is the switch that stops the system drawing them at all; +// `dismissAlert()` in replay.mjs handles one that still gets through. +export function quietSystemDialogs(id) { + adbOk(id, ['shell', 'settings', 'put', 'global', 'hide_error_dialogs', '1']) + adbOk(id, ['shell', 'settings', 'put', 'global', 'anr_show_background', '0']) + // long-press power / "system UI isn't responding" variants come from the same + // watchdog and are suppressed by the same setting on modern images + adbOk(id, ['shell', 'settings', 'put', 'secure', 'immersive_mode_confirmations', 'confirmed']) +} + // SystemUI demo mode is Android's answer to `simctl status_bar override`: // identical clock/battery/signal on every capture, so base and head // screenshots only differ where the app differs. diff --git a/action/cli/lib/ocr.mjs b/action/cli/lib/ocr.mjs index 77e3143..2fd6d51 100644 --- a/action/cli/lib/ocr.mjs +++ b/action/cli/lib/ocr.mjs @@ -149,12 +149,18 @@ export const containment = (needles, hay) => { if (!needles.size) return null; l const ALERT_HINTS = [/would like/i, /don['’]t allow/i, /^allow$/i, /allow while using/i, /allow once/i, /^not now$/i, /turn on/i, /^ok$/i, // Android's runtime permission dialog wording - /while using the app/i, /only this time/i, /^deny$/i, /don['’]t allow/i] + /while using the app/i, /only this time/i, /^deny$/i, /don['’]t allow/i, + // Android's ANR dialog. hide_error_dialogs should stop these being drawn at + // all (see android.mjs), but a slow emulator is exactly where they appear and + // exactly where a capture cannot afford one — it is modal and stays up. + /isn['’]t responding/i, /has stopped/i, /keeps stopping/i, /^wait$/i, /^close app$/i] // a system permission/alert is up if several of its tell-tale strings are visible export function alertButtons(items) { const texts = items.map((i) => i.text.trim()) const hits = texts.filter((t) => ALERT_HINTS.some((re) => re.test(t))) if (hits.length < 2) return [] - const order = [/don['’]t allow/i, /^deny$/i, /^not now$/i, /^ok$/i, /only this time/i, /allow once/i, /while using the app/i, /allow while using/i, /^allow$/i, /limit access/i] + // most conservative first. "Wait" beats "Close app" on an ANR: the app under + // test is the thing being mapped, and killing it ends the run. + const order = [/^wait$/i, /don['’]t allow/i, /^deny$/i, /^not now$/i, /^ok$/i, /only this time/i, /allow once/i, /while using the app/i, /allow while using/i, /^allow$/i, /limit access/i] return items.filter((i) => order.some((re) => re.test(i.text.trim()))).sort((a, b) => order.findIndex((re) => re.test(a.text)) - order.findIndex((re) => re.test(b.text))) } From 60e63833ac15fa0cebbf0a989ca96cad513c3ea8 Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 22:00:48 +0200 Subject: [PATCH 08/11] Make the baseline workflow's "full" input do something The baseline template has offered a workflow_dispatch input described as "Rebuild the whole map instead of incrementally" since it was written, but action.yml never read it and never passed --full to the CLI. Dispatching with full=true did an ordinary incremental run. Found while trying to re-test an Android fix: the run reused all eight captures from the previous baseline, never booted the emulator, and reported success without exercising the change at all. The action takes `full` as an input and forwards it; the template passes its dispatch input through. Co-Authored-By: Claude Opus 5 --- action.yml | 11 ++++++++++- action/templates/screenmap-baseline.yml | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index ae41956..730f696 100644 --- a/action.yml +++ b/action.yml @@ -92,6 +92,13 @@ inputs: ios (needs developmentClient + ios.simulator), development-emulator for android (needs developmentClient + android.buildType "apk" — an .aab cannot be installed on an emulator). default: "" + full: + description: >- + (baseline) Recapture every screen instead of reusing the unchanged ones from the previous + map. The baseline workflow template has exposed this as a workflow_dispatch input since the + beginning, but nothing ever passed it through, so "Rebuild the whole map" quietly did an + incremental run. + default: "false" flows_pr: description: "(baseline) Open a PR against the default branch with flows the agent recorded" default: "true" @@ -418,7 +425,9 @@ runs: --pr "${{ github.event.pull_request.number }}" --title "${{ github.event.pull_request.title }}" --url "${{ github.event.pull_request.html_url }}" \ --base-ref "${{ github.event.pull_request.base.ref }}" --head-ref "${{ github.event.pull_request.head.ref }}" > .screenmap-ci/summary.json else - screenmap-ci baseline --project "$proj" $( [ "${{ steps.baseline.outputs.found }}" = "true" ] && echo "--previous .screenmap-ci/baseline.scrmap" ) > .screenmap-ci/summary.json + screenmap-ci baseline --project "$proj" \ + $( [ "${{ steps.baseline.outputs.found }}" = "true" ] && echo "--previous .screenmap-ci/baseline.scrmap" ) \ + $( [ "${{ inputs.full }}" = "true" ] && echo "--full" ) > .screenmap-ci/summary.json fi bundle=$(python3 -c 'import json;print(json.load(open(".screenmap-ci/summary.json"))["bundle"])') echo "bundle=$bundle" >> "$GITHUB_OUTPUT" diff --git a/action/templates/screenmap-baseline.yml b/action/templates/screenmap-baseline.yml index 8983014..70c23ca 100644 --- a/action/templates/screenmap-baseline.yml +++ b/action/templates/screenmap-baseline.yml @@ -33,6 +33,7 @@ jobs: - uses: aleqsio/screenmap@v1 with: mode: baseline + full: ${{ inputs.full }} agent_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # agent_provider: claude # or codex | gemini | opencode — pass the matching key above expo_token: ${{ secrets.EXPO_TOKEN }} # EAS builds/serves the dev client (or pass app_path instead) From f6ad78b2f3ca3705047ccff77639e18563f27874 Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 22:08:46 +0200 Subject: [PATCH 09/11] Say what the Android CI runs actually proved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver is no longer unverified: it ran green on GitHub Actions against screenmap-test, capturing and publishing eight screens. Notably it needed no changes of its own after the first successful boot — every fix was in the Action or in how a failure was reported. Replaces the pre-run list of guesses with what the six runs cost and why, keeps the three things still genuinely unproven (muteDevMenu, the status-bar clock across runs, and the PR lane), and records the incidental finds: EAS needs no manual keystore, fingerprint reuse works, and `full` was never wired up. Co-Authored-By: Claude Opus 5 --- TODOS.md | 101 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/TODOS.md b/TODOS.md index 5356710..244f1bc 100644 --- a/TODOS.md +++ b/TODOS.md @@ -6,53 +6,60 @@ the full write-up of what that turned up is in `site/docs/setup-instruction-fixe --- -## Android: what has and has not been run against a real device - -Android support landed on 2026-09-01. Everything that does not need an Android -device was exercised end to end, most of it against `sample-app` and its real -captures: config resolution, the multi-platform bundle (v3) and diff (v2), -incremental reuse, `screenmap-ci merge`, the viewer's platform switcher, the -tesseract OCR backend, and the refactored iOS driver against a live simulator. -**The `adb` driver in `action/cli/lib/android.mjs` has not been run against a -live emulator**, because the machine it was written on has no Android SDK. Treat -the first real run as the test, and check these in order — they are ranked by how -likely they are to be wrong: - -1. **`muteDevMenu()` is a guess.** It writes - `expo.modules.devmenu.sharedpreferences.xml` through `run-as`. The file name and - the three boolean keys were inferred from the iOS `EXDevMenu*` defaults, not read - off a device. If the dev-menu overlay shows up in captures, dump the real prefs - (`adb shell run-as ls /data/data//shared_prefs`) and fix the names. - Deliberately non-fatal: a wrong guess costs a floating button in the corner of - every screenshot, not a failed run. -2. **`appIdOf()` depends on `aapt2` being findable.** It searches PATH then every - `build-tools/*` under the SDK roots. A runner that ships the SDK without - build-tools falls through to a clear error telling the user to set - `android.packageName`, but that path has never actually fired. -3. **The connect loop's cold-start nudge.** `openUrl` on a force-stopped app is - assumed to launch it straight into the deep link, the way `simctl openurl` does. - If the first bundle never arrives, that assumption is the first thing to check — - diagnostics land in `.screenmap/out/ci/diag/android/` (screenshot, logcat tail, - package list, Metro log). -4. **SystemUI demo mode** is broadcast-based and silently does nothing on a build - where `sysui_demo_allowed` is not settable. A base/head pair with a drifting - clock in the diff means it did not take. - -Also unproven: the KVM setup in `action.yml` on a real ubuntu runner, and whether -`emulator -no-window` plus `-gpu swiftshader_indirect` boots fast enough there to -beat the macOS rate it exists to avoid. - -### The bug the sample-app run caught - -Worth recording because it was invisible and would have shipped a plausible-looking -wrong map. `baselineSide()` originally let a single-platform bundle answer for any -platform asked of it. A repo whose previous baseline was iOS-only, turning Android -on, therefore "reused" all six iOS screenshots as Android captures — labelled -Android in the viewer, and indistinguishable from a real run. Bundles now declare -which platforms they hold (`platformsIn()`), a side that is not there reports -`exists: false` so the new platform captures in full, and `merge` refuses an -input whose platform does not match the one it is being merged as rather than -silently mislabelling it. +## Android: verified on CI, and what the first runs cost + +Android support landed on 2026-09-01 and ran green on GitHub Actions on +2026-09-08 against `aleqsio/screenmap-test`: emulator booted, APK installed, +`adb reverse` tunnelled, `brew://` deep links resolved, first JS bundle served, +eight screens captured and published. **The adb driver itself needed no changes +after the first successful boot** — every fix was in `action.yml` or in how the +run reported failure. What it took, in order, because none of it was reachable +from a laptop: + +1. **`yes | sdkmanager` failed a step that succeeded.** GitHub runs `shell: bash` + with `-eo pipefail`; `yes` dies of SIGPIPE the moment sdkmanager stops + reading, and pipefail takes that as the pipeline's status. Read + `PIPESTATUS[1]` instead. +2. **`libpulse.so.0` is not on GitHub's ubuntu images**, and the SDK's qemu links + against it, so `emulator` could not start at all. Installed for android runs + along with the X libs, those best-effort since the names drift between + releases. +3. **avdmanager and emulator disagreed on where AVDs live.** Different resolution + chains (`ANDROID_AVD_HOME`, then `$ANDROID_SDK_HOME/.android/avd` for one and + `$HOME/.android/avd` for the other). The AVD was created and invisible. + `ANDROID_AVD_HOME` is now pinned for both. +4. **Every capture came back behind "Pixel Launcher isn't responding".** An + emulator on software rendering trips the ANR watchdog, the dialog is modal, + and it lands in every capture after it. `hide_error_dialogs` stops the system + drawing them; `ALERT_HINTS` learned the wording as a second layer. + +Two of those cost far more than they should have because the run reported the +wrong thing, which is the lesson worth keeping: `emulator -version` was `|| true`, +so a broken binary surfaced seventeen minutes and one EAS build later as "no AVD +defined" — the one thing that was not wrong. Both are now hard gates that print +the underlying tool's own complaint. + +Incidental finds along the way: EAS generates Android credentials +non-interactively, so no keystore setup is needed; fingerprint reuse works +(a rebuild collapsed to an 8-second download); and the baseline workflow's +`full` input had never been wired to `--full`. + +### Still unverified + +- **`muteDevMenu()` remains a guess.** No dev-menu overlay appeared in the + captures, but this app may simply not show one where iOS would, so the + SharedPreferences filename and keys are still unconfirmed. It stays + best-effort and non-fatal. +- **The status bar clock.** Demo mode's pinned icons show, and the run that had + the ANR dialog up also showed `9:41`; the run without it shows no clock at + all. The likeliest reading is that the app draws edge-to-edge over the clock + area and the dialog was changing the window insets — but that is a guess, and + the alternative is that a demo-mode broadcast is being dropped. It does not + affect a single run, where all eight captures agree. It would matter across + base and head: a clock present in one and absent in the other marks every + screen changed. Worth settling before the Android PR lane is trusted. +- **The PR lane itself.** Only the baseline has run. The diff, its comment, and + the tesseract OCR line in it are still untested on a real PR. ## The screenmaps branch has no platform in its paths From 9197f40df36fe77e1948f7dc99dd986ea3b1a40e Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 22:23:20 +0200 Subject: [PATCH 10/11] The Android PR lane works, and the clock was a red herring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit screenmap-test#17 exercised the diff lane on Android: suspects narrowed to the one changed screen, only that screen was captured on the head side, the base side came from the Android baseline, and the comment rendered with the before/after pair and the tesseract line. It also answers the status-bar question, which turns out not to be one. Base and head agree — neither shows a clock, because the app draws edge-to-edge over that area. The 9:41 in the earlier run was the anomaly: the ANR dialog was changing the window insets. What a diff needs is the two sides agreeing, and they do. Replaces those two entries with what is genuinely still untested on Android: flow replay, and the agent lane. Co-Authored-By: Claude Opus 5 --- TODOS.md | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/TODOS.md b/TODOS.md index 244f1bc..9ecb182 100644 --- a/TODOS.md +++ b/TODOS.md @@ -44,22 +44,37 @@ non-interactively, so no keystore setup is needed; fingerprint reuse works (a rebuild collapsed to an 8-second download); and the baseline workflow's `full` input had never been wired to `--full`. +### The PR lane, and the clock + +Both settled on 2026-09-08 by +[screenmap-test#17](https://github.com/aleqsio/screenmap-test/pull/17), a +one-file copy change to `/grind`. + +The diff lane works: suspects narrowed to exactly one node, only that screen was +captured on the head side, the base side came from the Android baseline, and the +sticky comment rendered with the before/after pair, the right reason +("its own source changed"), the Android device name, and the tesseract line. + +**The clock is a non-issue.** Base and head agree: neither shows one. The +status-bar strip in both contains only the app's own eyebrow text, because this +app draws edge-to-edge over that area. The `9:41` seen in the run with the ANR +dialog up was the anomaly — the dialog was changing the window insets. Demo mode +is still worth setting for the icons, but on an edge-to-edge app the clock it +pins may never be visible, and that is fine: what a diff needs is base and head +agreeing, which they do. + ### Still unverified -- **`muteDevMenu()` remains a guess.** No dev-menu overlay appeared in the - captures, but this app may simply not show one where iOS would, so the +- **`muteDevMenu()` remains a guess.** No dev-menu overlay appeared in any + capture, but this app may simply not show one where iOS would, so the SharedPreferences filename and keys are still unconfirmed. It stays best-effort and non-fatal. -- **The status bar clock.** Demo mode's pinned icons show, and the run that had - the ANR dialog up also showed `9:41`; the run without it shows no clock at - all. The likeliest reading is that the app draws edge-to-edge over the clock - area and the dialog was changing the window insets — but that is a guess, and - the alternative is that a demo-mode broadcast is being dropped. It does not - affect a single run, where all eight captures agree. It would matter across - base and head: a clock present in one and absent in the other marks every - screen changed. Worth settling before the Android PR lane is trusted. -- **The PR lane itself.** Only the baseline has run. The diff, its comment, and - the tesseract OCR line in it are still untested on a real PR. +- **Flow replay on Android.** Every run so far was flowless — `0 by flow + replay`, everything deep-linked. argent's device tools take an Android serial, + but no committed flow has actually been replayed on one, so `replayFlow()` and + `verifyLanding()`'s landmark check are still untested on this platform. +- **The agent lane on Android.** Deliberately off (no key) for these runs, so the + platform-specific prompt in `agent.mjs` has never been exercised. ## The screenmaps branch has no platform in its paths From d024719b61dfc371921fb2d69b8596467dc670da Mon Sep 17 00:00:00 2001 From: aleqsio Date: Tue, 8 Sep 2026 23:07:19 +0200 Subject: [PATCH 11/11] Boot the device the Action provisioned, and fail when it never boots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from review on #8, both real. ensureBooted discarded waitBootComplete's result on both paths, so an emulator adb could see but that never flipped sys.boot_completed was returned anyway. The session then ran install and launch against a half-booted system and failed later with opaque package-manager errors. Both paths now throw, saying which device and that it was the boot that timed out. The `avd` input created an AVD nothing then selected: ensureBooted reads config.android.device, which defaulted to null and was never set from the input, so the driver took avds[0]. With one AVD that happens to be right; on a runner with others it boots an image the Action never provisioned. The action now passes both provisioned devices as SCREENMAP_DEVICE_IOS / SCREENMAP_DEVICE_ANDROID, loadConfig reads them per platform, and an explicit config..device still wins. The driver also logs which AVD it picked, and warns when the requested one was not among those defined. Also fixes the platform switcher, which this found in passing: passing asChild straight from TooltipTrigger to ToggleGroupItem let the tooltip's data-state overwrite the item's, so the selected style never rendered and Radix reported the wrong item as checked — the viewer opened on Android with neither button highlighted. Wrapping the item in a span, as the Changes item already does, restores both. Co-Authored-By: Claude Opus 5 --- action.yml | 4 ++++ action/cli/lib/android.mjs | 19 ++++++++++++++++--- action/cli/lib/util.mjs | 7 +++++++ apps/visualiser/src/components/TopBar.jsx | 16 +++++++++------- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/action.yml b/action.yml index 730f696..2936ab9 100644 --- a/action.yml +++ b/action.yml @@ -344,6 +344,10 @@ runs: AGENT_API_KEY: ${{ inputs.agent_api_key }} SCREENMAP_APP_PATH: ${{ steps.app.outputs.app }} SCREENMAP_PLATFORMS: ${{ inputs.platform }} + # the device this action actually provisioned, so the driver boots that + # one rather than whichever happens to be listed first + SCREENMAP_DEVICE_IOS: ${{ inputs.simulator }} + SCREENMAP_DEVICE_ANDROID: ${{ inputs.avd }} GH_TOKEN: ${{ inputs.github_token }} run: | set -e diff --git a/action/cli/lib/android.mjs b/action/cli/lib/android.mjs index be72248..f047ad5 100644 --- a/action/cli/lib/android.mjs +++ b/action/cli/lib/android.mjs @@ -117,7 +117,12 @@ export async function ensureBooted(config) { if (booted.length) { const pick = config.device ? booted.find((d) => d.name === config.device) ?? booted[0] : booted[0] log(`android device already available: ${pick.name} (${pick.id})`) - await waitBootComplete(pick.id, 60000) + // adb seeing a device is not the device being usable. Returning a + // half-booted one means install and launch fail later with opaque + // package-manager errors instead of a boot timeout that says what happened. + if (!(await waitBootComplete(pick.id, 60000))) { + throw new Error(`${pick.name} (${pick.id}) is visible to adb but never finished booting (sys.boot_completed)`) + } return pick } const bin = emulatorPath() @@ -126,7 +131,10 @@ export async function ensureBooted(config) { if (error) throw new Error(`no Android device connected, and the emulator could not be queried — ${error}`) if (!avds.length) throw new Error('no Android device connected and no AVD defined — create one with avdmanager') const avd = avds.find((a) => a === config.device) ?? avds[0] - log(`booting AVD ${avd}`) + if (config.device && avd !== config.device) { + log(`warning: AVD "${config.device}" not found among [${avds.join(', ')}] — booting ${avd} instead`) + } + log(`booting AVD ${avd}${avds.length > 1 ? ` (of ${avds.length} defined)` : ''}`) // detached: the emulator runs for the whole session and must outlive this call const proc = spawn(bin, ['-avd', avd, '-no-snapshot', '-no-boot-anim', '-no-audio', ...(process.env.SCREENMAP_EMULATOR_WINDOW === '1' ? [] : ['-no-window']), @@ -135,7 +143,12 @@ export async function ensureBooted(config) { const deadline = Date.now() + 300000 while (Date.now() < deadline) { const now = listBooted() - if (now.length) { await waitBootComplete(now[0].id); return now[0] } + if (now.length) { + if (!(await waitBootComplete(now[0].id))) { + throw new Error(`AVD ${avd} started but never finished booting (sys.boot_completed) — a software-rendered emulator can be too slow to come up`) + } + return now[0] + } await sleep(3000) } throw new Error(`AVD ${avd} did not come up within 5 minutes`) diff --git a/action/cli/lib/util.mjs b/action/cli/lib/util.mjs index 186d7ba..89af221 100644 --- a/action/cli/lib/util.mjs +++ b/action/cli/lib/util.mjs @@ -135,6 +135,13 @@ export function loadConfig(projectDir) { if (defined.packageName && !defined.appId) defined.appId = defined.packageName const envPath = process.env[`SCREENMAP_APP_PATH_${p.toUpperCase()}`] || (merged.platforms.length === 1 ? process.env.SCREENMAP_APP_PATH : null) if (envPath) defined.appPath = envPath // a prebuilt client (e.g. from EAS) beats on-disk discovery + // The Action provisions a specific simulator/AVD and has to be able to say + // which one: without this the driver falls back to "whatever is first", + // which on a runner that already has other devices boots something the + // Action never set up. Config still wins — this is the CI default, not an + // override of an explicit choice. + const envDevice = process.env[`SCREENMAP_DEVICE_${p.toUpperCase()}`] + if (envDevice && !user[p]?.device) defined.device = envDevice merged[p] = defined } return merged diff --git a/apps/visualiser/src/components/TopBar.jsx b/apps/visualiser/src/components/TopBar.jsx index da8a79b..11df0db 100644 --- a/apps/visualiser/src/components/TopBar.jsx +++ b/apps/visualiser/src/components/TopBar.jsx @@ -113,13 +113,15 @@ export default function TopBar({ manifest, mode, setMode, hasChanges, overlaid, {platforms.map((p) => ( - - {PLATFORM_LABEL[p.platform] ?? p.platform} - + + + {PLATFORM_LABEL[p.platform] ?? p.platform} + + {p.device ?? p.label ?? p.platform}