Batch registered research CI without dropping package coverage

.github/workflows/research-package-batch.yml

@@ -0,0 +1,34 @@
+name: Registered research package batch
+on:
+  workflow_call:
+    inputs:
+      packages:
+        description: JSON array of at most 256 registered package paths
+        required: true
+        type: string
+permissions:
+  contents: read
+jobs:
+  replay:
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        package: ${{ fromJSON(inputs.packages) }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      PYTHONDONTWRITEBYTECODE: '1'
+      RESEARCH_PACKAGE: ${{ matrix.package }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
+        with:
+          python-version: '3.12'
+      - name: Explicit bounded package replay
+        run: python -B scripts/research_package.py --package "$RESEARCH_PACKAGE" --install-deps
+      - name: Committed inputs remain unchanged
+        run: git diff --exit-code

.github/workflows/research-packages.yml

@@ -37,28 +37,16 @@ jobs:
         name: Select changed packages and declared input dependents
         env:
           REVIEW_BASE: ${{ github.event.pull_request.base.sha || github.event.before || inputs.base }}
-        run: python -B scripts/research_package.py --changed-since "$REVIEW_BASE" --github-output "$GITHUB_OUTPUT"
+        run: python -B scripts/research_package.py --changed-since "$REVIEW_BASE" --matrix-batch-size 256 --github-output "$GITHUB_OUTPUT"
   replay:
     needs: plan
     if: needs.plan.outputs.count != '0'
     strategy:
       fail-fast: false
-      max-parallel: 4
+      # Each called workflow owns at most 256 package jobs; batches run in
+      # sequence so the inner four-worker concurrency budget is unchanged.
+      max-parallel: 1
       matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    env:
-      PYTHONDONTWRITEBYTECODE: '1'
-      RESEARCH_PACKAGE: ${{ matrix.package }}
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
-        with:
-          python-version: '3.12'
-      - name: Explicit bounded package replay
-        run: python -B scripts/research_package.py --package "$RESEARCH_PACKAGE" --install-deps
-      - name: Committed inputs remain unchanged
-        run: git diff --exit-code
+    uses: ./.github/workflows/research-package-batch.yml
+    with:
+      packages: ${{ toJSON(matrix.packages) }}

docs/research/research-package-guide.md

@@ -95,6 +95,17 @@ Each gets a fresh runner and explicit commands. It does not run every historical
 `verify.py`. New hooks do not replace existing cold-replay obligations, and a
 passing metadata check is not acceptance of a mathematical result.
 
+The planner groups at most 256 package paths into each reusable-workflow batch,
+so a shared-runner change can select more than GitHub's single-matrix limit.
+Batches run one at a time and each batch runs at most four isolated package jobs;
+the per-package fresh checkout, environment and 30-minute job bound are retained.
+Changing either workflow selects all registered packages. Selection is never
+truncated: more than 256 batches fails explicitly. The ordinary planner CLI keeps
+its flat matrix format unless `--matrix-batch-size 256` is requested.
+This is a matrix-count bound, not a promise that the largest grouping fits every
+GitHub execution limit; the 1 MB job-output limit (estimated in UTF-16) and other
+platform limits still apply. The current 450-package matrix is well below it.
+
 ## Optional workflow feedback
 
 If something helped or obstructed the session, add a few sentences to your PR

scripts/research_package.py

@@ -247,7 +247,8 @@ def run_package(root, package, install=False):
 def plan(root, changed):
     changed = set(changed)
     replay_changed = bool(changed.intersection({
-        "scripts/research_package.py", ".github/workflows/research-packages.yml"}))
+        "scripts/research_package.py", ".github/workflows/research-packages.yml",
+        ".github/workflows/research-package-batch.yml"}))
     packages = []
     for path in sorted((root / "research").glob("*/VERIFY.json")):
         package = path.parent.relative_to(root).as_posix()
@@ -264,13 +265,27 @@ def plan(root, changed):
     return packages
 
 
+def replay_matrix(packages, batch_size=None):
+    """Retain the flat CLI format, or group complete coverage into bounded matrices."""
+    if batch_size is None:
+        return {"include": [{"package": p} for p in packages]}
+    require(type(batch_size) is int and 1 <= batch_size <= 256,
+            "matrix batch size must be 1..256")
+    batches = [packages[i:i + batch_size] for i in range(0, len(packages), batch_size)]
+    require(len(batches) <= 256,
+            "more than 256 matrix batches required; no packages were truncated")
+    return {"include": [{"packages": batch} for batch in batches]}
+
+
 def main():
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("--package")
     ap.add_argument("--check-only", action="store_true")
     ap.add_argument("--install-deps", action="store_true", help="explicit opt-in to install manifest's allowed pinned dependencies")
     ap.add_argument("--changed-since", help="plan affected opt-in packages relative to this Git commit")
     ap.add_argument("--github-output", type=Path)
+    ap.add_argument("--matrix-batch-size", type=int,
+                    help="group planned packages for bounded reusable-workflow matrices (1..256)")
     args = ap.parse_args()
     try:
         if args.changed_since:
@@ -281,7 +296,7 @@ def main():
                 # A rename must invalidate both the removed input and its new path.
                 changed = git(ROOT, "diff", "--no-renames", "--name-only", "-z", base, "HEAD").decode().split("\0")
             packages = plan(ROOT, changed)
-            matrix = json.dumps({"include": [{"package": p} for p in packages]})
+            matrix = json.dumps(replay_matrix(packages, args.matrix_batch_size))
             print(matrix)
             if args.github_output:
                 with args.github_output.open("a", encoding="utf-8") as out:

tests/test_research_matrix_batches.py

@@ -0,0 +1,58 @@
+"""Complete package coverage at both GitHub matrix boundaries."""
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location('matrix_package', ROOT / 'scripts/research_package.py')
+package = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(package)
+
+
+class MatrixBatchTests(unittest.TestCase):
+    def test_exact_coverage_and_both_matrix_limits(self):
+        for count in (0, 1, 255, 256, 257, 450, 65536):
+            with self.subTest(count=count):
+                names = ['research/p' + str(i) for i in range(count)]
+                rows = package.replay_matrix(names, 256)['include']
+                self.assertLessEqual(len(rows), 256)
+                self.assertTrue(all(0 < len(row['packages']) <= 256 for row in rows))
+                self.assertEqual([p for row in rows for p in row['packages']], names)
+
+    def test_oversized_or_invalid_matrix_fails_without_truncation(self):
+        with self.assertRaisesRegex(ValueError, 'no packages were truncated'):
+            package.replay_matrix(['research/a'] * 65537, 256)
+        for size in (0, -1, 257, True, 1.5):
+            with self.subTest(size=size), self.assertRaises(ValueError):
+                package.replay_matrix(['research/a'], size)
+
+    def test_legacy_cli_shape_is_unchanged(self):
+        self.assertEqual(package.replay_matrix(['research/a', 'research/b']),
+                         {'include': [{'package': 'research/a'}, {'package': 'research/b'}]})
+
+    def test_nested_workflow_change_selects_every_registered_package(self):
+        expected = sorted(p.parent.relative_to(ROOT).as_posix()
+                          for p in (ROOT / 'research').glob('*/VERIFY.json'))
+        self.assertEqual(package.plan(ROOT, ['.github/workflows/research-package-batch.yml']), expected)
+
+    def test_grouped_cli_output_keeps_total_package_count(self):
+        expected = package.plan(ROOT, ['scripts/research_package.py'])
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory) / 'github-output'
+            result = subprocess.run([sys.executable, '-B', str(ROOT / 'scripts/research_package.py'),
+                                     '--changed-since', '0', '--matrix-batch-size', '256',
+                                     '--github-output', str(output)], cwd=ROOT, text=True,
+                                    capture_output=True, check=True)
+            fields = dict(line.split('=', 1) for line in output.read_text().splitlines())
+            matrix = json.loads(result.stdout)
+            self.assertEqual(json.loads(fields['matrix']), matrix)
+            self.assertEqual(int(fields['count']), len(expected))
+            self.assertEqual([p for row in matrix['include'] for p in row['packages']], expected)
+
+
+if __name__ == '__main__':
+    unittest.main()