diff --git a/.github/workflows/research-packages.yml b/.github/workflows/research-packages.yml index 70e228e66b..3c2bd0ee9b 100644 --- a/.github/workflows/research-packages.yml +++ b/.github/workflows/research-packages.yml @@ -1,30 +1,10 @@ name: Registered research packages on: + # Declared package inputs may live anywhere in the repository. Always plan; + # the manifest-aware selector keeps unrelated changes out of replay jobs. pull_request: - paths: - - 'research/**' - - 'scripts/find_overlap.py' - - 'scripts/new_research_package.py' - - 'scripts/research_package.py' - - 'scripts/choose_research_task.py' - - 'scripts/research_baseline.py' - - 'tests/test_research_*.py' - - 'docs/research/agent-task-catalog.json' - - 'autoresearch.md' - - '.github/workflows/research-packages.yml' push: branches: [main] - paths: - - 'research/**' - - 'scripts/find_overlap.py' - - 'scripts/new_research_package.py' - - 'scripts/research_package.py' - - 'scripts/choose_research_task.py' - - 'scripts/research_baseline.py' - - 'tests/test_research_*.py' - - 'docs/research/agent-task-catalog.json' - - 'autoresearch.md' - - '.github/workflows/research-packages.yml' workflow_dispatch: inputs: base: diff --git a/docs/research/research-package-guide.md b/docs/research/research-package-guide.md index f8c3b3340b..f8b01e4fa6 100644 --- a/docs/research/research-package-guide.md +++ b/docs/research/research-package-guide.md @@ -76,7 +76,12 @@ Each command has a maximum 600-second timeout and the package total is at most bounds limit a verification job, not the mathematical search budget. The [registered-package workflow](../../.github/workflows/research-packages.yml) -selects **only opt-in packages** whose directory or declared inputs changed. +selects **only opt-in packages**, using their directories, declared inputs, and shared replay controls. +Its planning job runs for pull requests and pushes to `main` without path filters, +because declared inputs can live anywhere in the repository. Unrelated changes +produce no replay jobs. Git renames count as removal of the old path and addition +of the new path, so packages that depend on either path receive a replay. +Changes to the package runner or this workflow select every registered package. Each gets a fresh runner and explicit commands. It does not run every historical `verify.py`. New hooks do not replace existing cold-replay obligations, and a passing metadata check is not acceptance of a mathematical result. diff --git a/scripts/research_package.py b/scripts/research_package.py index 14e75666ac..606b2bd4b0 100644 --- a/scripts/research_package.py +++ b/scripts/research_package.py @@ -143,6 +143,8 @@ def run_package(root, package, install=False): def plan(root, changed): changed = set(changed) + replay_changed = bool(changed.intersection({ + "scripts/research_package.py", ".github/workflows/research-packages.yml"})) packages = [] for path in sorted((root / "research").glob("*/VERIFY.json")): package = path.parent.relative_to(root).as_posix() @@ -150,7 +152,7 @@ def plan(root, changed): # Parse only for selection; validate fully in the isolated replay job. inputs = manifest.get("inputs", []) if (changed.intersection(inputs) or any(p.startswith(package + "/") for p in changed) - or "scripts/research_package.py" in changed): + or replay_changed): packages.append(package) return packages @@ -169,7 +171,8 @@ def main(): changed = git(ROOT, "ls-files", "-z").decode().split("\0") else: base = git(ROOT, "rev-parse", "--verify", args.changed_since + "^{commit}").decode().strip() - changed = git(ROOT, "diff", "--name-only", "-z", base, "HEAD").decode().split("\0") + # A rename must invalidate both the removed input and its new path. + changed = git(ROOT, "diff", "--no-renames", "--name-only", "-z", base, "HEAD").decode().split("\0") packages = plan(ROOT, changed) matrix = json.dumps({"include": [{"package": p} for p in packages]}) print(matrix) diff --git a/tests/test_research_replay_selection.py b/tests/test_research_replay_selection.py new file mode 100644 index 0000000000..bb173497c5 --- /dev/null +++ b/tests/test_research_replay_selection.py @@ -0,0 +1,120 @@ +"""Regression controls for registered replay invalidation across workflow and Git. + +Charles Bethin, with OpenAI Codex assistance. Apache-2.0. +""" +import json +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ".github/workflows/research-packages.yml" + + +class ReplaySelectionTests(unittest.TestCase): + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.root = Path(temp.name) / "repo" + self.root.mkdir() + self.output = Path(temp.name) / "github-output.txt" + self.git("init", "-q") + for key, value in (("user.name", "Fixture"), ("user.email", "fixture@example.invalid"), + ("core.autocrlf", "false"), ("diff.renames", "true")): + self.git("config", key, value) + for name in ("scripts/research_package.py", WORKFLOW): + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((ROOT / name).read_bytes()) + self.inputs = ["certificates/input.json", "docs/research/proof.md", + "src/solver.py", "fixtures/space name.json"] + for name in self.inputs + ["data/other.json", "README.md"]: + self.write(name, "original fixture\n") + # Routing fixtures only: planning deliberately does not validate or run a package. + for slug, inputs in (("alpha", self.inputs), ("beta", ["data/other.json"])): + self.write(f"research/{slug}/VERIFY.json", json.dumps({"inputs": inputs})) + self.commit() + + def git(self, *args): + return subprocess.check_output(["git", *args], cwd=self.root).decode().strip() + + def write(self, name, text): + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + + def commit(self): + self.git("add", "--all") + self.git("-c", "maintenance.auto=false", "-c", "gc.auto=0", "commit", "-qm", "fixture") + return self.git("rev-parse", "HEAD") + + def selected(self, base): + output = self.output + output.unlink(missing_ok=True) + result = subprocess.run( + [sys.executable, "-B", str(self.root / "scripts/research_package.py"), + "--changed-since", base, "--github-output", str(output)], + cwd=self.root, capture_output=True, text=True, encoding="utf-8", timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + matrix = json.loads(result.stdout) + fields = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8").splitlines()) + self.assertEqual(json.loads(fields["matrix"]), matrix) + self.assertEqual(int(fields["count"]), len(matrix["include"])) + return [entry["package"] for entry in matrix["include"]] + + def test_declared_inputs_outside_research_reach_the_planner(self): + for name in self.inputs: + with self.subTest(name=name): + base = self.git("rev-parse", "HEAD") + self.write(name, "changed input\n") + self.commit() + self.assertEqual(self.selected(base), ["research/alpha"]) + + def test_deleted_input_still_selects_its_consumer(self): + base = self.git("rev-parse", "HEAD") + (self.root / self.inputs[0]).unlink() + self.commit() + self.assertEqual(self.selected(base), ["research/alpha"]) + + def test_renamed_input_still_selects_its_old_consumer(self): + base = self.git("rev-parse", "HEAD") + self.git("mv", self.inputs[0], "data/renamed.json") + self.commit() + # Establish the adversarial case: Git's ordinary name-only output loses the old path. + self.assertEqual(self.git("diff", "--name-only", base, "HEAD"), "data/renamed.json") + self.assertEqual(self.selected(base), ["research/alpha"]) + + def test_global_controls_select_all_registered_packages(self): + for name in (WORKFLOW, "scripts/research_package.py"): + with self.subTest(name=name): + base = self.git("rev-parse", "HEAD") + with (self.root / name).open("a", encoding="utf-8") as stream: + stream.write("\n# changed replay control\n") + self.commit() + self.assertEqual(self.selected(base), ["research/alpha", "research/beta"]) + + def test_unrelated_change_has_no_replay_and_package_addition_does(self): + base = self.git("rev-parse", "HEAD") + self.write("README.md", "unrelated edit\n") + self.commit() + self.assertEqual(self.selected(base), []) + base = self.git("rev-parse", "HEAD") + self.write("research/beta/evidence.txt", "new local artifact\n") + self.commit() + self.assertEqual(self.selected(base), ["research/beta"]) + + def test_workflow_does_not_filter_out_declared_input_paths(self): + text = (ROOT / WORKFLOW).read_text(encoding="utf-8") + # Static policy guard, not a YAML interpreter. actionlint checks the YAML separately. + events = text.split("\npermissions:", 1)[0] + self.assertIn("\n pull_request:", events) + self.assertIn("\n push:\n branches: [main]", events) + self.assertNotRegex(events, re.compile(r"^\s*['\"]?paths(?:-ignore)?['\"]?\s*:", re.M)) + + +if __name__ == "__main__": + unittest.main()