From abc493c18820f4097f983ce26963a5880711812b Mon Sep 17 00:00:00 2001 From: Sanjevi Rau Date: Thu, 12 Nov 2026 15:32:23 +0800 Subject: [PATCH 1/7] fix(console-front): stop rebuilding the engine on every cargo build The build script watched console/dist/index.html, which is missing until the console is built. Cargo re-runs a build script on every build while a watched path is missing, so in such a checkout every `cargo build` recompiled console-front, suite-runtime and the CLI: about 43 s each time on a CI runner, in most steps of the Differential harness and of the client replays. It now watches the folder only (Cargo walks it), and dates a folder it creates to the script, so a fresh checkout builds once. A no-change build goes from 5.6 s to 0.15 s locally; adding, changing or removing a console file still re-embeds. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GojiTpvSY8YywZFwh2ZJg2 --- crates/console-front/build.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/console-front/build.rs b/crates/console-front/build.rs index 423c2337..3a1fea7f 100644 --- a/crates/console-front/build.rs +++ b/crates/console-front/build.rs @@ -2,18 +2,30 @@ //! creates. A checkout without one (the differential harness, a fresh //! `cargo build`) must still compile: the folder is guaranteed here, empty, //! and the router then serves its build-needed page. Cargo re-runs the build -//! when the folder's entries change so a release binary re-embeds a new -//! console build. +//! when anything in the folder changes (it walks a watched folder), so a +//! release binary re-embeds a new console build. +//! +//! Only the folder is watched. Cargo re-runs a build script on every build +//! while a watched path is missing, and `index.html` is missing until the +//! console is built: watching it recompiled this crate and the two above it +//! on every `cargo build` in such a checkout, about 43 s each time on a CI +//! runner, and the harness builds the engine in most of its steps. use std::path::PathBuf; fn main() { let manifest = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("manifest dir")); let dist = manifest.join("../../console/dist"); + let created = !dist.exists(); std::fs::create_dir_all(&dist).expect("console/dist can be created"); + if created { + // A folder made during this run is newer than the run, which the + // next build takes for a change: date it to this script instead. + // Best effort (Windows does not open a folder as a file), at worst + // one more rebuild. + if let Ok(time) = std::fs::metadata(manifest.join("build.rs")).and_then(|m| m.modified()) { + let _ = std::fs::File::open(&dist).and_then(|folder| folder.set_modified(time)); + } + } println!("cargo:rerun-if-changed={}", dist.display()); - println!( - "cargo:rerun-if-changed={}", - dist.join("index.html").display() - ); } From 0a07f093bcece6a505dac8d082352f9dfb419140 Mon Sep 17 00:00:00 2001 From: Sanjevi Rau Date: Thu, 12 Nov 2026 15:32:26 +0800 Subject: [PATCH 2/7] test(auth-front): settle the two Auth tests that failed now and then The sign-in p99 test measures up to three rounds and passes on the first within both bounds; the p99 of 200 sign-ins is the third slowest, and a busy Intel macOS runner failed its ratio about one run in ten. A sign-in that waited for the encode still fails every round. Fixes #190. The persistence test flushes before it removes the state directory, so the writer thread cannot create a temporary file in it meanwhile ("Directory not empty"). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GojiTpvSY8YywZFwh2ZJg2 --- crates/auth-front/src/console/tests.rs | 43 ++++++++++++++++---------- crates/auth-front/src/persist_tests.rs | 4 +++ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/crates/auth-front/src/console/tests.rs b/crates/auth-front/src/console/tests.rs index 7ac99a23..754ead97 100644 --- a/crates/auth-front/src/console/tests.rs +++ b/crates/auth-front/src/console/tests.rs @@ -2387,25 +2387,36 @@ async fn sign_in_latencies( /// A sign-in never waits for the state to be encoded: at 10,000 users with /// persistence on its p99 stays within 50 ms and within three times (plus -/// 2 ms of scheduling slack) the 100-user p99 measured in the same run. +/// 2 ms of scheduling slack) the 100-user p99 measured in the same round. +/// +/// The p99 of 200 sign-ins is the third slowest, so a few sign-ins slowed +/// by a busy runner failed it now and then (the Intel macOS runner, about +/// one run in ten): up to three rounds are measured and the first one +/// within both bounds passes. A sign-in that waited for the encode would +/// fail every round. #[tokio::test] async fn a_sign_in_with_persistence_on_does_not_grow_with_the_user_count() { - let mut p99 = Vec::new(); - for users in [100, 10_000] { - let (runtime, directory, credentials) = persisted(users, 31); - let mut latencies = sign_in_latencies(&runtime, &credentials, 200).await; - p99.push(percentiles(&mut latencies).2); - drop(runtime); - let _ = std::fs::remove_dir_all(&directory); + const ROUNDS: usize = 3; + let mut rounds = Vec::new(); + for _ in 0..ROUNDS { + let mut p99 = Vec::new(); + for users in [100, 10_000] { + let (runtime, directory, credentials) = persisted(users, 31); + let mut latencies = sign_in_latencies(&runtime, &credentials, 200).await; + p99.push(percentiles(&mut latencies).2); + drop(runtime); + let _ = std::fs::remove_dir_all(&directory); + } + let (small, large) = (p99[0], p99[1]); + rounds.push(format!("{large:?} at 10,000 users, {small:?} at 100")); + if large <= Duration::from_millis(50) && large <= small * 3 + Duration::from_millis(2) { + return; + } } - let (small, large) = (p99[0], p99[1]); - assert!( - large <= Duration::from_millis(50), - "p99 {large:?} at 10,000 users" - ); - assert!( - large <= small * 3 + Duration::from_millis(2), - "p99 {large:?} at 10,000 users, {small:?} at 100" + panic!( + "sign-in p99 out of bounds (at most 50 ms, and three times the 100-user p99 plus 2 ms) \ + in all {ROUNDS} rounds: {}", + rounds.join("; ") ); } diff --git a/crates/auth-front/src/persist_tests.rs b/crates/auth-front/src/persist_tests.rs index 809b5468..07163e30 100644 --- a/crates/auth-front/src/persist_tests.rs +++ b/crates/auth-front/src/persist_tests.rs @@ -328,6 +328,10 @@ async fn an_operation_answers_once_its_change_is_on_disk() { ); // The state's directory becomes a file: every write fails from here on. + // The flush first waits out a write the writer thread may have started + // and leaves nothing for it to write: a temporary file it created while + // the directory was being removed failed this with "Directory not empty". + runtime.flush().expect("flush"); std::fs::remove_dir_all(&state).expect("remove the state directory"); std::fs::write(&state, "not a directory").expect("block the state path"); let (status, body) = sign_up(&runtime, "grace@example.test").await; From c679c980699a906aa1c19af0129fbe819320a5c8 Mon Sep 17 00:00:00 2001 From: Sanjevi Rau Date: Thu, 12 Nov 2026 15:32:28 +0800 Subject: [PATCH 3/7] ci(packages): build darwin-x64 on Apple silicon, test it under Rosetta 2 The Intel macOS runner spent 18-25 minutes on the release build alone and 20-47 on the job, twice the next slowest platform, and it set the length of every packages and release run. The engine is now built for x86_64-apple-darwin on the Apple silicon runner (checked with lipo), and its release-mode tests, the npm and Bun installs and the suite smoke run the x86_64 engine with an x64 Node under Rosetta 2. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GojiTpvSY8YywZFwh2ZJg2 --- .github/workflows/packages.yml | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 16c74ba6..460a6563 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -34,22 +34,37 @@ jobs: - {runner: ubuntu-24.04, platform: linux-x64} - {runner: ubuntu-24.04-arm, platform: linux-arm64} - {runner: macos-15, platform: darwin-arm64} - - {runner: macos-15-intel, platform: darwin-x64} + # Built on Apple silicon for x86_64, then tested and installed + # with the x86_64 engine and an x64 Node under Rosetta 2. The + # Intel runner spent 18-25 minutes on the release build alone and + # 20-47 on this job, twice the next slowest platform. + - {runner: macos-15, platform: darwin-x64, rust-target: x86_64-apple-darwin, node-architecture: x64, target-directory: 'x86_64-apple-darwin/'} - {runner: windows-2025, platform: win32-x64, extension: .exe, target-directory: 'x86_64-pc-windows-msvc/'} runs-on: ${{ matrix.runner }} - # The release build plus the release-mode suite tests: about 14 minutes - # on the Intel macOS runner before the engine embedded its own console, - # past 28 since (Windows 21). The job reruns everything from a clean - # checkout, so the limit only has to catch a hang. + # The release build plus the release-mode suite tests: 9-17 minutes on + # Linux and Apple silicon, up to 22 on Windows (the Intel macOS runner, + # before darwin-x64 moved to Apple silicon, took up to 47). The job + # reruns everything from a clean checkout, so the limit only has to + # catch a hang. timeout-minutes: 50 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Rosetta 2 for the x86_64 engine and Node + if: matrix.node-architecture == 'x64' && runner.os == 'macOS' + run: | + if ! arch -x86_64 /usr/bin/true 2> /dev/null; then + sudo softwareupdate --install-rosetta --agree-to-license + fi + arch -x86_64 /usr/bin/true - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 with: node-version: 24.20.0 + architecture: ${{ matrix.node-architecture }} package-manager-cache: false + - name: The Node the packages are installed with + run: node -p '`${process.platform}-${process.arch} ${process.version}`' # No Java and no firebase-tools on the packed-install hosts: the suite # evaluates Storage rules natively and runs Functions on its own runtime. - name: Pinned package managers and unit contracts @@ -74,10 +89,15 @@ jobs: env: TOOLCHAIN: ${{ steps.engine.outputs.toolchain }} EXPECTED_REVISION: ${{ steps.engine.outputs.revision }} + RUST_TARGET: ${{ matrix.rust-target }} run: | test "$(git -C accepted-engine rev-parse HEAD)" = "$EXPECTED_REVISION" rustup toolchain install "$TOOLCHAIN" --profile minimal --no-self-update cargo=(cargo "+$TOOLCHAIN") + if [[ -n "$RUST_TARGET" ]]; then + rustup target add --toolchain "$TOOLCHAIN" "$RUST_TARGET" + export CARGO_BUILD_TARGET="$RUST_TARGET" + fi if [[ "$RUNNER_OS" == "Windows" ]]; then export CARGO_BUILD_TARGET=x86_64-pc-windows-msvc export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS='-C target-feature=+crt-static' @@ -87,6 +107,9 @@ jobs: # Ubuntu 20.04 and node:24-slim. The tests run there too. if [[ "$RUNNER_OS" == "Linux" ]]; then cargo=(node packaging/cargo-glibc.mjs "$TOOLCHAIN"); fi "${cargo[@]}" build --manifest-path accepted-engine/Cargo.toml --locked --release --bin firenook + if [[ "$RUST_TARGET" == x86_64-apple-darwin ]]; then + test "$(lipo -archs accepted-engine/target/x86_64-apple-darwin/release/firenook)" = x86_64 + fi "${cargo[@]}" test --manifest-path accepted-engine/Cargo.toml --locked --release -p firenook-suite-runtime --lib control::tests # Every Auth unit test in release mode: the filters these replaced # (password_tests, tests::oauth) had matched none since a rename. From f96c9d9f96904be6d854ffe0d0532ea9a5b5cdc6 Mon Sep 17 00:00:00 2001 From: Sanjevi Rau Date: Thu, 12 Nov 2026 15:32:31 +0800 Subject: [PATCH 4/7] ci(release): draft the GitHub release with this version's notes only The draft took the whole changelog, and the release owner cut it down to the version's section by hand. packaging/release-notes.mjs takes that section; on 0.2.3 it gives exactly the notes that were published. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GojiTpvSY8YywZFwh2ZJg2 --- .github/workflows/release-npm.yml | 4 ++- packaging/PUBLISHING.md | 4 ++- packaging/release-notes.mjs | 37 ++++++++++++++++++++++++ packaging/release-notes.test.mjs | 47 +++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 packaging/release-notes.mjs create mode 100644 packaging/release-notes.test.mjs diff --git a/.github/workflows/release-npm.yml b/.github/workflows/release-npm.yml index 0e1e1c81..75de1d38 100644 --- a/.github/workflows/release-npm.yml +++ b/.github/workflows/release-npm.yml @@ -140,8 +140,10 @@ jobs: RELEASE_TAG: ${{ inputs.tag }} SOURCE_RUN: ${{ needs.authorize.outputs.source_run || github.run_id }} run: | - gh release create "$RELEASE_TAG" --verify-tag --draft --title "Firenook ${RELEASE_TAG#npm-v}" --notes-file packaging/CHANGELOG.md VERSION="${RELEASE_TAG#npm-v}" + # This version's section of the changelog, not the whole file. + node packaging/release-notes.mjs "$VERSION" > "$RUNNER_TEMP/release-notes.md" + gh release create "$RELEASE_TAG" --verify-tag --draft --title "Firenook $VERSION" --notes-file "$RUNNER_TEMP/release-notes.md" mkdir release-receipts for PLATFORM in linux-x64 linux-arm64 darwin-arm64 darwin-x64 win32-x64; do for RECEIPT in npm-smoke bun-smoke suite-smoke; do diff --git a/packaging/PUBLISHING.md b/packaging/PUBLISHING.md index bdd2c6b3..a8f7968d 100644 --- a/packaging/PUBLISHING.md +++ b/packaging/PUBLISHING.md @@ -121,7 +121,9 @@ macOS notarization and more platforms remain separate supported-platform work. automatic `latest` on a prerelease stops for owner review; a CLI published to `latest` on request holds it on purpose. 6. After successful npm publication, a **draft** GitHub release gets the exact - tarballs/manifests/test receipts. Review and publish those notes. Without + tarballs/manifests/test receipts, with this version's section of + `packaging/CHANGELOG.md` as its notes (`packaging/release-notes.mjs`). + Review and publish those notes. Without `latest`, the workflow publishes to `next` only. Until the first stable release, `latest` follows the newest verified prerelease so that `npx firenook` resolves to a real engine rather than the `0.0.0` diff --git a/packaging/release-notes.mjs b/packaging/release-notes.mjs new file mode 100644 index 00000000..1c2ece78 --- /dev/null +++ b/packaging/release-notes.mjs @@ -0,0 +1,37 @@ +// The notes of one release: its section of packaging/CHANGELOG.md, from its +// `# VERSION` heading to the next release's. The release job drafts the +// GitHub release with these instead of the whole changelog, which the +// release owner had to cut down by hand. +// +// node packaging/release-notes.mjs VERSION [CHANGELOG] + +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +/** The section of `changelog` whose heading names `version`, trimmed. */ +export function releaseNotes(changelog, version) { + const lines = changelog.split('\n'); + const heading = line => /^# \S/.test(line); + const named = line => line === `# ${version}` || line.startsWith(`# ${version} `); + let fenced = false; + let start = -1; + let end = lines.length; + for (const [index, line] of lines.entries()) { + if (/^\s*(```|~~~)/.test(line)) fenced = !fenced; + if (fenced || !heading(line)) continue; + if (start === -1) { + if (named(line)) start = index; + } else { + end = index; + break; + } + } + if (start === -1) throw new Error(`The changelog has no "# ${version}" section`); + return `${lines.slice(start, end).join('\n').trim()}\n`; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const [version, file = new URL('CHANGELOG.md', import.meta.url)] = process.argv.slice(2); + if (!version) throw new Error('Usage: node packaging/release-notes.mjs VERSION [CHANGELOG]'); + process.stdout.write(releaseNotes(readFileSync(file, 'utf8'), version)); +} diff --git a/packaging/release-notes.test.mjs b/packaging/release-notes.test.mjs new file mode 100644 index 00000000..2e19b4e4 --- /dev/null +++ b/packaging/release-notes.test.mjs @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { releaseNotes } from './release-notes.mjs'; + +const changelog = [ + '# 0.2.4 — newer', + '', + '- four', + '', + '# 0.2.3 — the one asked for', + '', + '- three', + '', + '```sh', + '# a shell comment, not a heading', + '```', + '', + '## a smaller heading inside it', + '', + '# 0.2.3-next.1 — a prerelease of it', + '', + '- earlier', + '', +].join('\n'); + +test('a release gets its own section, up to the next release', () => { + assert.equal( + releaseNotes(changelog, '0.2.3'), + '# 0.2.3 — the one asked for\n\n- three\n\n```sh\n# a shell comment, not a heading\n```\n\n## a smaller heading inside it\n', + ); + assert.equal(releaseNotes(changelog, '0.2.4'), '# 0.2.4 — newer\n\n- four\n'); + assert.equal(releaseNotes(changelog, '0.2.3-next.1'), '# 0.2.3-next.1 — a prerelease of it\n\n- earlier\n'); +}); + +test('a version is matched whole, and a missing one stops the release', () => { + assert.throws(() => releaseNotes(changelog, '0.2'), /no "# 0\.2" section/); + assert.throws(() => releaseNotes(changelog, '0.2.5'), /no "# 0\.2\.5" section/); +}); + +test('the newest section of the real changelog is found and ends before the one after it', () => { + const real = readFileSync(new URL('CHANGELOG.md', import.meta.url), 'utf8'); + const newest = real.match(/^# (\S+)/m)[1]; + const notes = releaseNotes(real, newest); + assert.ok(notes.startsWith(`# ${newest}`)); + assert.equal(notes.match(/^# \S/gm).length, 1); +}); From 1138f0d43cb3be23c8d019d7fa5f21af7df89e87 Mon Sep 17 00:00:00 2001 From: Sanjevi Rau Date: Thu, 12 Nov 2026 15:54:33 +0800 Subject: [PATCH 5/7] ci(packages): install the x64 Bun directly on the darwin-x64 job Under Rosetta, Bun's npm installer picks its Apple silicon build, which an x64 Node did not install, and fails. The x64 baseline build is installed directly (Rosetta offers no AVX), so Bun installs the darwin-x64 package as on an Intel Mac; the step checks that Bun reports x64. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GojiTpvSY8YywZFwh2ZJg2 --- .github/workflows/packages.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 460a6563..9ac4a8b4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -68,8 +68,20 @@ jobs: # No Java and no firebase-tools on the packed-install hosts: the suite # evaluates Storage rules natively and runs Functions on its own runtime. - name: Pinned package managers and unit contracts + env: + NODE_ARCHITECTURE: ${{ matrix.node-architecture }} run: | - npm install --global npm@12.0.2 bun@1.3.14 + if [[ "$RUNNER_OS" == macOS && "$NODE_ARCHITECTURE" == x64 ]]; then + # Bun's installer picks its Apple silicon build under Rosetta, + # and Rosetta offers no AVX: the x64 baseline build is installed + # directly, so Bun installs the darwin-x64 package as on an + # Intel Mac. It warns that the CPU lacks AVX; baseline uses none. + npm install --global npm@12.0.2 @oven/bun-darwin-x64-baseline@1.3.14 + ln -s "$(npm root --global)/@oven/bun-darwin-x64-baseline/bin/bun" "$(npm prefix --global)/bin/bun" + test "$(bun -e 'console.log(process.arch)')" = x64 + else + npm install --global npm@12.0.2 bun@1.3.14 + fi node --test packaging/*.test.mjs node packaging/check-release.mjs - name: Resolve pinned engine source From 43c58c55ba52748ac870bdf38c7554f0e184e56a Mon Sep 17 00:00:00 2001 From: Sanjevi Rau Date: Thu, 12 Nov 2026 15:54:36 +0800 Subject: [PATCH 6/7] ci: start the journeys without the console checks, in four jobs The journeys need the engine, not the console checks: the engine is built in a job of its own beside them, so the journeys start 2.5-4.5 minutes sooner. The six short journey projects (3-6 minutes each) run in pairs and the Functions journeys on Node 22 and 26 in one job, so a run asks for four fewer runners and waits less for them. Shell stays alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GojiTpvSY8YywZFwh2ZJg2 --- .github/workflows/ci.yml | 85 ++++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f36613c5..d1238a42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,7 @@ jobs: if-no-files-found: error console: - name: Console checks and the engine the journeys run against + name: Console checks needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest @@ -125,15 +125,25 @@ jobs: npm run test --prefix console npm run build --prefix console npm run budget --prefix console + + journey-engine: + # Built once and handed to every journey job, beside the console checks + # rather than after them: the journeys need the engine, not the checks + # (2.5-4.5 minutes). The debug binary reads console/dist from this + # checkout's path at run time, which every journey job's checkout + # shares; each builds the console itself. + name: The engine the journeys run against + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install latest stable Rust run: | rustup update stable --no-self-update rustup default stable rustc --version - # Built once here and handed to every journey shard: a cold debug - # build is most of this job, and seven shards building it would be - # seven times that. The debug binary reads console/dist from this - # checkout's path at run time, which every shard's checkout shares. - name: Build the engine the browser journeys run against run: cargo build --locked --bin firenook - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 @@ -145,16 +155,22 @@ jobs: console-journeys: # The whole suite is about 17 minutes on a 12-core machine; on a 4-vCPU - # runner it would not fit one job's budget, so each Playwright project - # (one engine profile) is its own job with its own limit. - name: Console journeys (${{ matrix.project }}) - needs: console + # runner it would not fit one job's budget. Each Playwright project (one + # engine profile) ran as a job of its own; the six short ones (3-6 + # minutes) now run in pairs, so the run asks for three fewer runners and + # waits less for them, and shell (about 10) stays alone. + name: Console journeys (${{ matrix.name }}) + needs: journey-engine runs-on: ubuntu-latest timeout-minutes: 35 strategy: fail-fast: false matrix: - project: [data, auth, storage, compute, messaging, observe, shell] + include: + - {name: data and storage, key: data-storage, projects: --project data --project storage} + - {name: auth and messaging, key: auth-messaging, projects: --project auth --project messaging} + - {name: compute and observe, key: compute-observe, projects: --project compute --project observe} + - {name: shell, key: shell, projects: --project shell} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 @@ -179,13 +195,13 @@ jobs: run: npx playwright install --with-deps chromium - name: Install the pinned Functions SDK the journeys' Functions fixture links in run: npm run e2e:sdk --prefix console - - name: Run the ${{ matrix.project }} journeys against the real engine + - name: Run the ${{ matrix.name }} journeys against the real engine working-directory: console - run: npx playwright test --project ${{ matrix.project }} + run: npx playwright test ${{ matrix.projects }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 if: failure() with: - name: console-playwright-report-${{ matrix.project }} + name: console-playwright-report-${{ matrix.key }} path: | console/playwright-report console/test-results @@ -194,23 +210,27 @@ jobs: # Functions run on the Node that runs Firenook, and every other job runs # Node 24. The Functions journeys again, with the engine running the # Functions on each other supported Node; Playwright and the console - # build stay on the pinned 24.20.0. - name: Console Functions journeys (Node ${{ matrix.node }}) - needs: console + # build stay on the pinned 24.20.0. One job runs both (about 3 minutes + # each), so the setup is paid once. + name: Console Functions journeys (Node 22 and 26) + needs: journey-engine runs-on: ubuntu-latest timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - node: [22, 26] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: - node-version: ${{ matrix.node }} - - name: Keep the Node the engine runs the Functions on + node-version: 22 + - name: Keep Node 22 for the Functions + run: | + echo "NODE22=$(command -v node)" >> "$GITHUB_ENV" + node --version + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 26 + - name: Keep Node 26 for the Functions run: | - echo "FIRENOOK_NODE=$(command -v node)" >> "$GITHUB_ENV" + echo "NODE26=$(command -v node)" >> "$GITHUB_ENV" node --version - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: @@ -234,15 +254,28 @@ jobs: run: npx playwright install --with-deps chromium - name: Install the pinned Functions SDK the journeys' Functions fixture links in run: npm run e2e:sdk --prefix console - - name: Run the compute journeys with the Functions on Node ${{ matrix.node }} + - name: Run the compute journeys with the Functions on Node 22 working-directory: console - run: npx playwright test --project compute + env: + FIRENOOK_NODE: ${{ env.NODE22 }} + run: npx playwright test --project compute --output test-results/node22 + - name: Keep the Node 22 report + if: ${{ !cancelled() }} + working-directory: console + run: if [ -d playwright-report ]; then mv playwright-report playwright-report-node22; fi + - name: Run the compute journeys with the Functions on Node 26 + if: ${{ !cancelled() }} + working-directory: console + env: + FIRENOOK_NODE: ${{ env.NODE26 }} + run: npx playwright test --project compute --output test-results/node26 - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 if: failure() with: - name: console-playwright-report-compute-node${{ matrix.node }} + name: console-playwright-report-compute-node22-26 path: | console/playwright-report + console/playwright-report-node22 console/test-results functions-node: From a5f0abf78e94677748a462d32d6f174724b2941e Mon Sep 17 00:00:00 2001 From: Sanjevi Rau Date: Thu, 12 Nov 2026 15:54:38 +0800 Subject: [PATCH 7/7] ci: run the oracle corpus replays beside the Differential harness The Functions, Cloud Tasks, Auth and Pub/Sub replays were the harness's longest steps, about 9 of its 19 minutes once the engine stopped rebuilding. They compare recorded answers and start no Java, so they run in a job of their own with their own ledger records and evidence. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GojiTpvSY8YywZFwh2ZJg2 --- .github/workflows/ci.yml | 128 +++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1238a42..29998612 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,7 +423,10 @@ jobs: runs-on: ubuntu-latest # The 0.2.0 replays (who a request is from, Storage targets without # buckets, rules-unit-testing on both emulators) brought this job to - # 24-30 minutes; 30 was cancelling it on slow runners. + # 24-30 minutes; 30 was cancelling it on slow runners. Most of that was + # the engine rebuilt in every step (fixed in console-front's build + # script), and the oracle corpus replays run in a job of their own: + # about 10 minutes now. timeout-minutes: 45 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -510,40 +513,6 @@ jobs: target/debug/firenook "$RUNNER_TEMP/firenook-devtools" \ "$HOME/.cache/firebase/emulators" "$RUNNER_TEMP/firenook-native-upgrade-evidence" \ "$RUNNER_TEMP/firenook-previous-engine/node_modules/@fireside-dev/linux-x64/bin/fireside" - - name: Replay the Functions runtime and Extensions oracle corpus - env: - FIREBASE_FUNCTIONS_7_2_ROOT: ${{ runner.temp }}/firenook-devtools/node_modules/firebase-functions - FIRENOOK_OFFLINE: "1" - run: | - NODE24="$(command -v node)" npm run replay:functions:runtime --prefix conformance -- \ - --binary ../target/debug/firenook \ - --profiles main,v1-blocking,inspect,ext-missing-param \ - --output "$RUNNER_TEMP/firenook-functions-runtime-replay.json" - - name: Replay the Cloud Tasks oracle corpus against the suite - env: - FIREBASE_FUNCTIONS_7_2_ROOT: ${{ runner.temp }}/firenook-devtools/node_modules/firebase-functions - FIRENOOK_OFFLINE: "1" - run: | - NODE24="$(command -v node)" npm run replay:functions:runtime --prefix conformance -- \ - --binary ../target/debug/firenook \ - --profiles tasks \ - --fixture fixtures/tasks-v1/emulator-programs.json \ - --output "$RUNNER_TEMP/firenook-tasks-replay.json" - - name: Replay the Auth oracle corpus against the standalone Auth service - run: | - npm run replay:auth --prefix conformance -- \ - --binary ../target/debug/firenook \ - --output "$RUNNER_TEMP/firenook-auth-replay.json" - - name: Replay the Pub/Sub oracle corpus over gRPC and HTTP/JSON - run: | - npm run replay:pubsub --prefix conformance -- \ - --binary ../target/debug/firenook \ - --output "$RUNNER_TEMP/firenook-pubsub-replay.json" - - name: Verify the pinned Pub/Sub client library against Firenook - run: | - npm run verify:pubsub:client --prefix conformance -- \ - --binary ../target/debug/firenook \ - --output "$RUNNER_TEMP/firenook-pubsub-client.json" - name: Verify suite start without Java on PATH run: >- node conformance/src/suite/verify-no-java.mjs @@ -570,10 +539,6 @@ jobs: ${{ runner.temp }}/firenook-developer-ui-evidence/failure.json ${{ runner.temp }}/firenook-developer-ui-evidence/browser-failure.json ${{ runner.temp }}/firenook-developer-ui-evidence/*.png - ${{ runner.temp }}/firenook-functions-runtime-replay.json - ${{ runner.temp }}/firenook-auth-replay.json - ${{ runner.temp }}/firenook-pubsub-replay.json - ${{ runner.temp }}/firenook-pubsub-client.json ${{ runner.temp }}/firenook-no-java-evidence/result.json ${{ runner.temp }}/firenook-no-java-evidence/suite.log if-no-files-found: ignore @@ -627,6 +592,89 @@ jobs: retention-days: 7 if-no-files-found: error + # The Functions, Cloud Tasks, Auth and Pub/Sub oracle corpus replays + # (about 9 minutes) beside the harness rather than in it, where they were + # its longest steps: they compare recorded answers and start no Java. + oracle-replays: + name: Oracle corpus replays + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Note the recorded cases each replay compares + run: echo "FIRENOOK_CONFORMANCE_LEDGER=$RUNNER_TEMP/conformance-ledger/oracle-replays" >> "$GITHUB_ENV" + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24.20.0 + cache: npm + cache-dependency-path: conformance/package-lock.json + - name: Install current npm + run: npm install --global npm@12.0.2 + - name: Install latest stable Rust + run: | + rustup update stable --no-self-update + rustup default stable + rustc --version + - name: Build the engine the replays run against + run: cargo build --locked --bin firenook + - name: Install pinned harness dependencies and the Functions peers + run: | + npm ci --prefix conformance + npm install --prefix "$RUNNER_TEMP/firenook-devtools" --ignore-scripts --no-audit --no-fund firebase-functions@7.2.5 firebase-admin@13.10.0 + - name: Replay the Functions runtime and Extensions oracle corpus + env: + FIREBASE_FUNCTIONS_7_2_ROOT: ${{ runner.temp }}/firenook-devtools/node_modules/firebase-functions + FIRENOOK_OFFLINE: "1" + run: | + NODE24="$(command -v node)" npm run replay:functions:runtime --prefix conformance -- \ + --binary ../target/debug/firenook \ + --profiles main,v1-blocking,inspect,ext-missing-param \ + --output "$RUNNER_TEMP/firenook-functions-runtime-replay.json" + - name: Replay the Cloud Tasks oracle corpus against the suite + env: + FIREBASE_FUNCTIONS_7_2_ROOT: ${{ runner.temp }}/firenook-devtools/node_modules/firebase-functions + FIRENOOK_OFFLINE: "1" + run: | + NODE24="$(command -v node)" npm run replay:functions:runtime --prefix conformance -- \ + --binary ../target/debug/firenook \ + --profiles tasks \ + --fixture fixtures/tasks-v1/emulator-programs.json \ + --output "$RUNNER_TEMP/firenook-tasks-replay.json" + - name: Replay the Auth oracle corpus against the standalone Auth service + run: | + npm run replay:auth --prefix conformance -- \ + --binary ../target/debug/firenook \ + --output "$RUNNER_TEMP/firenook-auth-replay.json" + - name: Replay the Pub/Sub oracle corpus over gRPC and HTTP/JSON + run: | + npm run replay:pubsub --prefix conformance -- \ + --binary ../target/debug/firenook \ + --output "$RUNNER_TEMP/firenook-pubsub-replay.json" + - name: Verify the pinned Pub/Sub client library against Firenook + run: | + npm run verify:pubsub:client --prefix conformance -- \ + --binary ../target/debug/firenook \ + --output "$RUNNER_TEMP/firenook-pubsub-client.json" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + if: always() + with: + name: oracle-replays + path: | + ${{ runner.temp }}/firenook-functions-runtime-replay.json + ${{ runner.temp }}/firenook-tasks-replay.json + ${{ runner.temp }}/firenook-auth-replay.json + ${{ runner.temp }}/firenook-pubsub-replay.json + ${{ runner.temp }}/firenook-pubsub-client.json + if-no-files-found: ignore + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: conformance-ledger-oracle-replays + path: ${{ runner.temp }}/conformance-ledger + retention-days: 7 + if-no-files-found: error + # The 0.2.2 replays and real-SDK checks run beside the harness: in it they # added 12-14 minutes and took it past its 45. client-replays: @@ -687,7 +735,7 @@ jobs: # names a case cases.mjs does not list. conformance-ledger: name: Conformance ledger - needs: [changes, rust, conformance, client-replays] + needs: [changes, rust, conformance, oracle-replays, client-replays] if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest timeout-minutes: 10