test(ci): qualify parallel runners on balanced shards

.github/workflows/node-ci.yml

@@ -80,9 +80,16 @@ jobs:
                   ;;
                 .github/workflows/node-ci.yml | .github/actions/setup-tools/*)
                   container_validate=true
-                  test_quality=true
                   ;;
-                .github/workflows/test-quality.yml)
+              esac
+              case "$path" in
+                .github/workflows/node-ci.yml | .github/workflows/test-quality.yml | \
+                  .github/actions/setup-tools/* | sdk/typescript/bunfig.toml | \
+                  sdk/typescript/scripts/compare-test-reports.mts | \
+                  sdk/typescript/scripts/prepare-windows-test-root.ps1 | \
+                  sdk/typescript/scripts/run-ci-tests.mts | \
+                  sdk/typescript/scripts/test-shards.mts | \
+                  sdk/typescript/scripts/test-timings.json)
                   test_quality=true
                   ;;
               esac

.github/workflows/test-quality.yml

@@ -30,68 +30,33 @@ jobs:
   runner:
     needs: native
     if: ${{ !cancelled() && (inputs.native-artifacts-ready || needs.native.result == 'success') }}
-    name: ${{ matrix.os }} / ${{ matrix.mode }}
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: ${{ matrix.os == 'windows-latest' && 90 || 30 }}
+    name: ${{ matrix.target.os }} / ${{ matrix.mode.name }} / shard-${{ matrix.target.shard }}
+    runs-on: ${{ matrix.target.os }}
+    timeout-minutes: ${{ matrix.target.os == 'windows-latest' && 90 || 30 }}
     env:
       CODEX_SECURITY_PROPERTY_SEED: ${{ github.event_name == 'pull_request' && 1 || github.run_number }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        mode: [baseline, isolated, parallel]
-        exclude:
-          - os: windows-latest
-            mode: isolated
-        include:
-          - mode: baseline
-            args: ""
-          - mode: isolated
-            args: --isolate
-          - mode: parallel
-            args: --parallel=2
-          - os: windows-latest
-            mode: isolated-1
-            args: --isolate --shard=1/7
-          - os: windows-latest
-            mode: isolated-2
-            args: --isolate --shard=2/7
-          - os: windows-latest
-            mode: isolated-3
-            args: --isolate --shard=3/7
-          - os: windows-latest
-            mode: isolated-4
-            args: --isolate --shard=4/7
-          - os: windows-latest
-            mode: isolated-5
-            args: --isolate --shard=5/7
-          - os: windows-latest
-            mode: isolated-6
-            args: --isolate --shard=6/7
-          - os: windows-latest
-            mode: isolated-7
-            args: --isolate --shard=7/7
-          - os: windows-latest
-            mode: shard-1
-            args: --shard=1/7
-          - os: windows-latest
-            mode: shard-2
-            args: --shard=2/7
-          - os: windows-latest
-            mode: shard-3
-            args: --shard=3/7
-          - os: windows-latest
-            mode: shard-4
-            args: --shard=4/7
-          - os: windows-latest
-            mode: shard-5
-            args: --shard=5/7
-          - os: windows-latest
-            mode: shard-6
-            args: --shard=6/7
-          - os: windows-latest
-            mode: shard-7
-            args: --shard=7/7
+        # Every mode uses the same production shard count and timing snapshot.
+        target:
+          - { os: ubuntu-latest, shard: 1, count: 3 }
+          - { os: ubuntu-latest, shard: 2, count: 3 }
+          - { os: ubuntu-latest, shard: 3, count: 3 }
+          - { os: macos-latest, shard: 1, count: 3 }
+          - { os: macos-latest, shard: 2, count: 3 }
+          - { os: macos-latest, shard: 3, count: 3 }
+          - { os: windows-latest, shard: 1, count: 7 }
+          - { os: windows-latest, shard: 2, count: 7 }
+          - { os: windows-latest, shard: 3, count: 7 }
+          - { os: windows-latest, shard: 4, count: 7 }
+          - { os: windows-latest, shard: 5, count: 7 }
+          - { os: windows-latest, shard: 6, count: 7 }
+          - { os: windows-latest, shard: 7, count: 7 }
+        mode:
+          - { name: baseline, args: "" }
+          - { name: isolated, args: --isolate }
+          - { name: parallel, args: --parallel=2 }
     steps:
       - &actions-checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -116,17 +81,9 @@ jobs:
             sdk/typescript/pnpm-lock.yaml
             plugins/codex-security/mcp-app/pnpm-lock.yaml
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
-        if: runner.os == 'Windows'
         with:
-          bun-version: "1.4.2"
-      - name: Set up Bun on Linux
-        if: runner.os == 'Linux'
-        shell: bash
-        run: |
-          # Includes the subprocess fix from https://github.com/oven-sh/bun/pull/44581
-          # for the isolated-runner hang tracked in https://github.com/oven-sh/bun/issues/34069.
-          npm install --prefix "$RUNNER_TEMP/bun" --ignore-scripts --no-audit --no-fund @oven/bun-linux-x64@1.4.2-canary.20261008.1
-          echo "$RUNNER_TEMP/bun/node_modules/@oven/bun-linux-x64/bin" >> "$GITHUB_PATH"
+          # Includes the isolated spawnSync fix from oven-sh/bun#44581.
+          bun-version: "1.4.3"
       - name: Install dependencies
         run: |
           pnpm --dir sdk/typescript install --frozen-lockfile
@@ -136,20 +93,25 @@ jobs:
         id: windows-temp
         shell: pwsh
         run: ./sdk/typescript/scripts/prepare-windows-test-root.ps1
+      - name: Build bundled plugin
+        run: pnpm --dir sdk/typescript run build:plugin
+      - name: Install ripgrep
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes ripgrep
       - name: Test runner mode
+        working-directory: sdk/typescript
         env:
-          CODEX_SECURITY_TEST_TIMEOUT_MS: ${{ runner.os == 'Windows' && '120000' || '30000' }}
           TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
           TMP: ${{ steps.windows-temp.outputs.path || runner.temp }}
           TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }}
-        run: |
-          node -e "require('node:fs').mkdirSync('sdk/typescript/reports',{recursive:true})"
-          pnpm --dir sdk/typescript run test ${{ runner.os == 'Windows' && '--timeout=120000' || '' }} ${{ matrix.args }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter=junit --reporter-outfile=reports/runner-${{ matrix.os }}-${{ matrix.mode }}.xml
+        run: node --experimental-strip-types scripts/run-ci-tests.mts ${{ matrix.target.shard }}/${{ matrix.target.count }} ${{ matrix.mode.args }} --seed=${{ env.CODEX_SECURITY_PROPERTY_SEED }} --reporter-outfile=reports/runner-${{ matrix.target.os }}-${{ matrix.mode.name }}-${{ matrix.target.shard }}.xml
       - name: Upload runner report
         if: always()
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: runner-${{ matrix.os }}-${{ matrix.mode }}
+          name: runner-${{ matrix.target.os }}-${{ matrix.mode.name }}-${{ matrix.target.shard }}
           overwrite: true
           path: sdk/typescript/reports/runner-*.xml
           if-no-files-found: error
@@ -178,12 +140,28 @@ jobs:
         shell: bash
         run: |
           comparison_status=0
-          for os in ubuntu-latest windows-latest; do
+          shopt -s nullglob
+          for os in ubuntu-latest macos-latest windows-latest; do
+            shards=3
+            if [[ "$os" == windows-latest ]]; then shards=7; fi
+            for mode in baseline isolated parallel; do
+              reports=(reports/runner-"$os"-"$mode"-*.xml)
+              if (( ${#reports[@]} != shards )); then
+                echo "Expected $shards reports for $os/$mode, found ${#reports[@]}" >&2
+                comparison_status=1
+              fi
+              for ((shard=1; shard<=shards; shard++)); do
+                report="reports/runner-$os-$mode-$shard.xml"
+                if [[ ! -f "$report" ]]; then
+                  echo "Missing shard report: $report" >&2
+                  comparison_status=1
+                fi
+              done
+            done
             for mode in isolated parallel; do
-              node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "reports/runner-$os-baseline.xml" "reports/runner-$os-$mode*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
+              node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts "reports/runner-$os-baseline-*.xml" "reports/runner-$os-$mode-*.xml" >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
             done
           done
-          node --experimental-strip-types sdk/typescript/scripts/compare-test-reports.mts reports/runner-windows-latest-baseline.xml 'reports/runner-windows-latest-shard-*.xml' >> "$GITHUB_STEP_SUMMARY" || comparison_status=1
           exit "$comparison_status"
 
   mutation:

sdk/typescript/TESTING.md

@@ -119,7 +119,8 @@ The requirements file is the single list of pinned Python test dependencies. Ruf
 - Add property tests for meaningful invariants, with accepted and rejected
   inputs. Keep example-based regression tests for readable failure cases.
 - Give parameterized cases distinct names. Use `%p`, `%j`, or `%#` for values
-  that are not strings. The JUnit comparison rejects duplicate identities.
+  that are not strings. Duplicate identities remain valid; the JUnit comparison
+  checks their occurrence and outcome counts.
 
 Property tests use a fixed default seed. Fast-check prints the seed, shrink
 path, and counterexample on failure. To replay one property, select its file
@@ -174,19 +175,40 @@ Failed tests and missing package artifacts block CI; failed diagnostic uploads
 do not. Use `python -m pytest -n 0` to reproduce Python failures serially.
 
 The separate `test-quality` workflow runs weekly, can be dispatched manually,
-and runs on pull requests that change its workflow file. It compares Bun's
-default runner, `--isolate`, `--parallel=2`, and seven-way Windows sharding.
-Every mode uses the same seed for property cases and test ordering: pull
-requests replay seed 1; scheduled and manual runs use the workflow run number.
-The workflow compares test identities and outcomes against the unsharded
-default run and records timings, including failed shards, in the job summary.
-It is not a required check or part of the release trigger.
-
-Runner trials use Bun 1.4.2 to avoid the
-[async-module initialization bug in 1.3.14](https://github.com/oven-sh/bun/issues/31410)
-that breaks the Ink UI tests under isolation. Keep the trial pin until a newer
-release passes the full SDK suite in every mode. Required CI and the mutation
-trial remain on Bun 1.3.14.
+and runs on pull requests that change its workflow, setup tools, Bun configuration,
+shard runner, timing metadata, report comparator, or Windows test-root helper.
+It compares Bun's default runner, `--isolate`, and `--parallel=2` on Linux, macOS,
+and Windows.
+Each mode uses the same duration-based file assignment and production shard count:
+three on Linux/macOS and seven on Windows. This gives 39 sharded runner jobs in place
+of 19 jobs that included an unsharded Windows baseline. It measures the production
+partition rather than cross-file interactions across one unsharded suite; local
+full-suite runs still cover that ordering. More jobs add setup overhead, so compare
+workflow wall time and runner minutes, alongside shard test timings, before
+adopting a runner change.
+
+Every mode uses the same seed for property cases and test ordering: pull requests
+replay seed 1; scheduled and manual runs use the workflow run number. Reports have
+unique OS/mode/shard names. The workflow requires every expected shard report
+before accepting comparisons, then compares complete shard groups by test
+identity, outcome, and occurrence count, retaining legitimate repeated names from
+parameterized tests while detecting missing or extra instances. It records every
+shard's timing, including failed shards, in the job summary. It is not a required
+check or part of the release trigger.
+
+Runner trials use Bun 1.4.3, including the
+[isolated subprocess fix](https://github.com/oven-sh/bun/pull/44581).
+`--parallel=2` runs files in two worker processes and isolates each file; it does
+not make tests inside one file concurrent. Required CI and the mutation trial
+remain on Bun 1.3.14 until a newer release passes the full SDK suite in every mode
+on all three platforms.
+
+Runner modes use the same property seed and coverage settings as each other.
+Required CI uses the default property seed and collects Linux coverage, while
+these diagnostics use the seed above and omit coverage instrumentation. Compare
+inventories across required CI and diagnostics, but measure scheduling speed
+between diagnostic modes to avoid attributing these differences to the runner.
+Both workflows install ripgrep on Linux so the inventory-path test executes.
 
 Keep the machine-policy test serial. Do not replace the full required suite
 with `--changed`: Python files, schemas, fixtures, and workflows loaded at

sdk/typescript/scripts/compare-test-reports.mts

@@ -114,7 +114,6 @@ function readReport(path: string) {
   );
   parser.write(content).close();
 
-  const identities = new Set<string>();
   const cases = new Map<string, number>();
   for (const { attributes, status } of records) {
     const identity: TestIdentity = [
@@ -124,18 +123,13 @@ function readReport(path: string) {
       attributes["classname"]?.value ?? "",
       attributes["name"]?.value ?? "",
     ];
-    const key = JSON.stringify(identity);
-    if (identities.has(key))
-      throw new Error(
-        `${path}: duplicate test identity: ${identity.join(" > ")}`,
-      );
-    identities.add(key);
-    cases.set(JSON.stringify([...identity, status]), 1);
+    const key = JSON.stringify([...identity, status]);
+    cases.set(key, (cases.get(key) ?? 0) + 1);
   }
   if (!cases.size) throw new Error(`${path}: no test cases`);
   if (
-    integer(root!.attributes["tests"]?.value ?? String(cases.size)) !==
-    BigInt(cases.size)
+    integer(root!.attributes["tests"]?.value ?? String(records.length)) !==
+    BigInt(records.length)
   ) {
     throw new Error(`${path}: reported test count does not match test cases`);
   }
@@ -150,7 +144,7 @@ function readReport(path: string) {
   const duration = seconds(root!.attributes["time"]?.value ?? "0");
   const skipped = records.filter(({ status }) => status === "skipped").length;
   console.log(
-    `| ${basename(path)} | ${cases.size} | ${skipped} | ${duration.toFixed(2)} |`,
+    `| ${basename(path)} | ${records.length} | ${skipped} | ${duration.toFixed(2)} |`,
   );
   return {
     cases,
@@ -159,6 +153,26 @@ function readReport(path: string) {
   };
 }
 
+function readReports(patterns: string[]) {
+  const cases = new Map<string, number>();
+  const durations: number[] = [];
+  let failed = false;
+  for (const pattern of patterns) {
+    const paths = matchingReports(pattern).sort((a, b) =>
+      Buffer.compare(Buffer.from(a), Buffer.from(b)),
+    );
+    if (!paths.length) throw new Error(`No reports match ${pattern}`);
+    for (const path of paths) {
+      const report = readReport(path);
+      failed ||= report.failed;
+      for (const [identity, count] of report.cases)
+        cases.set(identity, (cases.get(identity) ?? 0) + count);
+      durations.push(report.duration);
+    }
+  }
+  return { cases, durations, failed };
+}
+
 function main(): number {
   let args: { values: { help?: boolean }; positionals: string[] };
   try {
@@ -176,32 +190,18 @@ function main(): number {
   }
   if (args.values.help) {
     console.log(
-      "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nCandidates are JUnit files or glob patterns.",
+      "Compare Bun JUnit inventories before changing the required CI runner.\n\nUsage: node --experimental-strip-types compare-test-reports.mts baseline candidates [candidates ...]\nBaseline and candidates are JUnit files or glob patterns.",
     );
     return 0;
   }
   console.log("| Report | Cases | Skipped | Seconds |");
   console.log("| --- | ---: | ---: | ---: |");
-  const baseline = readReport(args.positionals[0]!);
-  let failed = baseline.failed;
-  const candidates = new Map<string, number>();
-  const durations: number[] = [];
-  for (const pattern of args.positionals.slice(1)) {
-    const paths = matchingReports(pattern).sort((a, b) =>
-      Buffer.compare(Buffer.from(a), Buffer.from(b)),
-    );
-    if (!paths.length) throw new Error(`No reports match ${pattern}`);
-    for (const path of paths) {
-      const report = readReport(path);
-      failed ||= report.failed;
-      for (const [identity, count] of report.cases)
-        candidates.set(identity, (candidates.get(identity) ?? 0) + count);
-      durations.push(report.duration);
-    }
-  }
+  const baseline = readReports([args.positionals[0]!]);
+  const candidate = readReports(args.positionals.slice(1));
+  let failed = baseline.failed || candidate.failed;
   for (const [label, left, right] of [
-    ["Missing", baseline.cases, candidates],
-    ["Extra", candidates, baseline.cases],
+    ["Missing", baseline.cases, candidate.cases],
+    ["Extra", candidate.cases, baseline.cases],
   ] as const) {
     const differences = [...left]
       .map(([identity, count]): [TestOutcome, number] => [
@@ -225,8 +225,13 @@ function main(): number {
     }
   }
   if (failed) return 1;
+  if (baseline.durations.length > 1) {
+    console.log(
+      `\nBaseline shards: slowest ${Math.max(...baseline.durations).toFixed(2)}s; combined test time: ${baseline.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.`,
+    );
+  }
   console.log(
-    `\nIdentical test inventory and outcomes. Slowest candidate: ${Math.max(...durations).toFixed(2)}s; combined test time: ${durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.\n`,
+    `\nIdentical test inventory and outcomes. Slowest candidate: ${Math.max(...candidate.durations).toFixed(2)}s; combined test time: ${candidate.durations.reduce((sum, duration) => sum + duration, 0).toFixed(2)}s.\n`,
   );
   return 0;
 }