From 8bb9080f7288b62cd5e2ea42b4400748d4603a02 Mon Sep 17 00:00:00 2001 From: Toni Nowak Date: Sun, 27 Sep 2026 14:23:16 +0200 Subject: [PATCH] test(pilot): stage query migration repair case --- PILOT.md | 6 + TASKS.md | 6 + scripts/pilot_oracles/query_migration.py | 97 ++++++++++++++++ .../fixtures/query_migration_triage.case.json | 80 +++++++++++++ .../query_migration_triage/CHANGELOG.md | 5 + .../query_migration_triage/CONTRACT_v1.md | 3 + .../query_migration_triage/CONTRACT_v2.md | 3 + .../fixtures/query_migration_triage/README.md | 12 ++ .../legacy_query_golden.json | 5 + .../query_migration_triage/query_encoder.py | 9 ++ .../tests/test_query_encoder.py | 21 ++++ tests/test_query_migration_fixture.py | 107 ++++++++++++++++++ 12 files changed, 354 insertions(+) create mode 100644 scripts/pilot_oracles/query_migration.py create mode 100644 tests/fixtures/query_migration_triage.case.json create mode 100644 tests/fixtures/query_migration_triage/CHANGELOG.md create mode 100644 tests/fixtures/query_migration_triage/CONTRACT_v1.md create mode 100644 tests/fixtures/query_migration_triage/CONTRACT_v2.md create mode 100644 tests/fixtures/query_migration_triage/README.md create mode 100644 tests/fixtures/query_migration_triage/legacy_query_golden.json create mode 100644 tests/fixtures/query_migration_triage/query_encoder.py create mode 100644 tests/fixtures/query_migration_triage/tests/test_query_encoder.py create mode 100644 tests/test_query_migration_fixture.py diff --git a/PILOT.md b/PILOT.md index 3b86a86..aa96fe2 100644 --- a/PILOT.md +++ b/PILOT.md @@ -1731,3 +1731,9 @@ This integration is an offline measurement capability, not delivery or efficacy ## VCR391 — Profile advice acknowledgment timing The profile event parser now observes acknowledgment after both legacy and configured valid triage results. A real synthetic event stream verifies acknowledgment before the next tool and distinguishes missing or late acknowledgment. This fixes delivery measurement only; it does not establish native delivery or efficacy. + +## VCR389 — Prospective query migration repair fixture + +Staged a new synthetic URL query migration task with versioned contracts, historical evidence, immutable tests, and an external hash-pinned oracle. Offline checks establish the expected initial failure, a source-only reference repair, and rejection of protected-file tampering or unexpected files. The oracle independently runs focused/full checks and checks the source patch. + +This is preparation, not a launched pair or efficacy evidence. Launch remains gated on symmetric arm Git initialization and observed agent diff validation. The accepted diagnostic metadata includes contract confirmation, but the current runner does not yet request that candidate; causal ranking and diagnostic selection require separate integration. diff --git a/TASKS.md b/TASKS.md index 1525bc3..42815ec 100644 --- a/TASKS.md +++ b/TASKS.md @@ -1135,3 +1135,9 @@ The runner now writes the bounded redacted measurement to its private output dir ## VCR391 — Profile advice acknowledgment timing The profile event parser now observes acknowledgment after both legacy and configured valid triage results. A real synthetic event stream verifies acknowledgment before the next tool and distinguishes missing or late acknowledgment. This fixes delivery measurement only; it does not establish native delivery or efficacy. + +## VCR389 — Prospective query migration repair fixture + +Staged a new synthetic URL query migration task with versioned contracts, historical evidence, immutable tests, and an external hash-pinned oracle. Offline checks establish the expected initial failure, a source-only reference repair, and rejection of protected-file tampering or unexpected files. The oracle independently runs focused/full checks and checks the source patch. + +This is preparation, not a launched pair or efficacy evidence. Launch remains gated on symmetric arm Git initialization and observed agent diff validation. The accepted diagnostic metadata includes contract confirmation, but the current runner does not yet request that candidate; causal ranking and diagnostic selection require separate integration. diff --git a/scripts/pilot_oracles/query_migration.py b/scripts/pilot_oracles/query_migration.py new file mode 100644 index 0000000..dd09a3b --- /dev/null +++ b/scripts/pilot_oracles/query_migration.py @@ -0,0 +1,97 @@ +"""Independent oracle and immutable-check verifier for the query migration fixture.""" +from __future__ import annotations + +import argparse +import hashlib +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +IMMUTABLE_SHA256 = { + "CHANGELOG.md": "ccfb22eb8fb0c75773bea70f1f7d3566ec7d6c1f5743388664bd92655e21835d", + "CONTRACT_v1.md": "2aa3178710bc90b6c073518e87652c760ff5357674b2f7153f605e0074215412", + "CONTRACT_v2.md": "c10decdef269169115650ff34c76b1c1be3517756869466e7194561591af4d96", + "README.md": "15cb774071942bc09b8d1fd32f88053a9d4be4fdeed71b207eb5b8d68c56d2cc", + "legacy_query_golden.json": "e12b45986a7bace7f792aba8a49ca7e1749f7ec94e433a620daf90726695c870", + "tests/test_query_encoder.py": "61e8874a8dbdc45e790ccf0a357fa7de812e235a2e8bfaeb6146efa98341da32", +} +BASELINE_SOURCE = '''"""Serialize ordered query pairs for the request client.""" +from __future__ import annotations + +from urllib.parse import quote, urlencode + + +def serialize_query(pairs: list[tuple[str, str]]) -> str: + """Return query pairs in order, preserving the v1 quoting implementation.""" + return urlencode(pairs, quote_via=quote) +''' + + +def run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> None: + result = subprocess.run(command, cwd=cwd, env=env, text=True, capture_output=True) + if result.returncode != 0: + raise SystemExit( + f"independent_check_failed:{command[1:3]}:exit={result.returncode}:" + f"{result.stderr[-1200:]}" + ) + + +parser = argparse.ArgumentParser() +parser.add_argument("--fixture-dir", required=True, type=Path) +args = parser.parse_args() +fixture = args.fixture_dir.resolve(strict=True) + +# The only allowed source edit is query_encoder.py; all task/test evidence is frozen. +actual_paths = { + path.relative_to(fixture).as_posix() + for path in fixture.rglob("*") + if path.is_file() + and ".git" not in path.relative_to(fixture).parts + and "__pycache__" not in path.relative_to(fixture).parts +} +expected_paths = set(IMMUTABLE_SHA256) | {"query_encoder.py"} +if actual_paths != expected_paths: + raise SystemExit("fixture_file_set_changed") +for relative, expected in IMMUTABLE_SHA256.items(): + actual = hashlib.sha256((fixture / relative).read_bytes()).hexdigest() + if actual != expected: + raise SystemExit(f"immutable_fixture_file_changed:{relative}") + +spec_source = fixture / "query_encoder.py" +# Independent behavior oracle. +spec_text = spec_source.read_text(encoding="utf-8") +namespace: dict[str, object] = {} +exec(compile(spec_text, str(spec_source), "exec"), namespace) +serialize_query = namespace["serialize_query"] +cases = [ + ([ ("q", "blue sky"), ("tag", "a+b") ], "q=blue+sky&tag=a%2Bb"), + ([ ("city", "Ålesund café") ], "city=%C3%85lesund+caf%C3%A9"), + ([ ("b", "2"), ("a", "") ], "b=2&a="), +] +for pairs, expected in cases: + actual = serialize_query(pairs) # type: ignore[operator] + if actual != expected: + raise SystemExit(f"oracle_mismatch:{expected!r}:{actual!r}") + +# Re-run the exact focused and full fixture commands independently. +env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") +run([sys.executable, "-m", "unittest", "discover", "-s", "tests", "-p", "test_query_encoder.py", "-v"], cwd=fixture, env=env) +run([sys.executable, "-m", "unittest", "discover", "-s", "tests", "-v"], cwd=fixture, env=env) + +# The copied arm has no .git directory. Build a throwaway baseline index so the +# exact candidate patch can still receive an independent `git diff --check`. +with tempfile.TemporaryDirectory(prefix="query-migration-diff-") as temp: + work = Path(temp) + candidate = work / "query_encoder.py" + candidate.write_text(BASELINE_SOURCE, encoding="utf-8") + run(["git", "init", "-q"], cwd=work) + run(["git", "config", "user.name", "fixture-oracle"], cwd=work) + run(["git", "config", "user.email", "fixture-oracle@invalid"], cwd=work) + run(["git", "add", "query_encoder.py"], cwd=work) + run(["git", "commit", "-q", "-m", "baseline"], cwd=work) + candidate.write_bytes(spec_source.read_bytes()) + run(["git", "diff", "--check"], cwd=work) + +print("oracle_passed:3 cases; focused/full tests passed; immutable files and diff check passed") diff --git a/tests/fixtures/query_migration_triage.case.json b/tests/fixtures/query_migration_triage.case.json new file mode 100644 index 0000000..1921997 --- /dev/null +++ b/tests/fixtures/query_migration_triage.case.json @@ -0,0 +1,80 @@ +{ + "case_id": "query_migration_triage", + "evidence_files": { + "changelog": "CHANGELOG.md", + "contract_v1": "CONTRACT_v1.md", + "contract_v2": "CONTRACT_v2.md", + "focused_test": "tests/test_query_encoder.py", + "implementation": "query_encoder.py", + "legacy_golden": "legacy_query_golden.json" + }, + "evidence_markers": { + "changelog": [ + "The v2 contract is authoritative for current requests" + ], + "contract_v1": [ + "Effective through 2025-06-30" + ], + "contract_v2": [ + "Effective 2025-07-01 and superseding v1", + "The v1 golden remains a historical record" + ], + "focused_test": [ + "q=blue+sky" + ], + "implementation": [ + "quote_via=quote" + ], + "legacy_golden": [ + "\"contract_version\": \"v1\"", + "q=blue%20sky&tag=a%2Bb" + ] + }, + "failure_markers": [ + "test_v2_space_encoding", + "q=blue%20sky", + "q=blue+sky" + ], + "fixture_source": "tests/fixtures/query_migration_triage", + "focused_command": [ + "python", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_query_encoder.py", + "-v" + ], + "focused_test_file": "tests/test_query_encoder.py", + "oracle_script": "scripts/pilot_oracles/query_migration.py", + "oracle_sha256": "da6110537d48a8bc497bf5e24d1dc9bd5584ff3950a6c3c8b6a650db0261b114", + "outcome_mode": "repair", + "schema_version": 1, + "source_file": "query_encoder.py", + "task_prompt_file": "README.md", + "triage": { + "accepted_ids": [ + "assertion_behavior_regression", + "assertion_expectation_drift", + "confirm_behavior_contract" + ], + "accepted_statuses": [ + "no-remote-choice" + ], + "hypotheses": [ + "assertion_behavior_regression", + "assertion_expectation_drift" + ], + "kinds": [ + "assertion" + ], + "observations": { + "assertion": [ + "legacy_fixture_conflict" + ] + }, + "rank_hypotheses": true + } +} diff --git a/tests/fixtures/query_migration_triage/CHANGELOG.md b/tests/fixtures/query_migration_triage/CHANGELOG.md new file mode 100644 index 0000000..eb7d53a --- /dev/null +++ b/tests/fixtures/query_migration_triage/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## 2025-07-01 — Query protocol v2 + +The upstream request format migrated from the v1 URI query convention to form-style query encoding. The v2 contract is authoritative for current requests. Preserve pair order and literal plus escaping; only space representation changes from `%20` to `+`. diff --git a/tests/fixtures/query_migration_triage/CONTRACT_v1.md b/tests/fixtures/query_migration_triage/CONTRACT_v1.md new file mode 100644 index 0000000..57c6947 --- /dev/null +++ b/tests/fixtures/query_migration_triage/CONTRACT_v1.md @@ -0,0 +1,3 @@ +# Query serialization contract, v1 + +Effective through 2025-06-30. Query pairs preserve caller order. For each value, encode UTF-8 bytes and percent-escape spaces as `%20`; a literal plus is `%2B`. This was the legacy RFC 3986-style representation. diff --git a/tests/fixtures/query_migration_triage/CONTRACT_v2.md b/tests/fixtures/query_migration_triage/CONTRACT_v2.md new file mode 100644 index 0000000..a8828c7 --- /dev/null +++ b/tests/fixtures/query_migration_triage/CONTRACT_v2.md @@ -0,0 +1,3 @@ +# Query serialization contract, v2 + +Effective 2025-07-01 and superseding v1 for current application requests. Preserve query pair order and values. Encode query pairs with `application/x-www-form-urlencoded` semantics: a space is `+`, while a literal plus remains `%2B`. The v1 golden remains a historical record and is not the v2 expected output. diff --git a/tests/fixtures/query_migration_triage/README.md b/tests/fixtures/query_migration_triage/README.md new file mode 100644 index 0000000..0ce0210 --- /dev/null +++ b/tests/fixtures/query_migration_triage/README.md @@ -0,0 +1,12 @@ +# Task: repair migrated query serialization + +The query serializer fails a new compatibility test after a protocol migration. Determine whether the implementation or the test expectation is out of date by inspecting the versioned local contract, historical fixture, current test, and implementation. Preserve the public function signature, input ordering, and value semantics. Do not edit or weaken tests, contract documents, changelog, or historical fixture; only `query_encoder.py` may change. + +Run these as separate commands and retain the observed exit from each: + +1. `python -m unittest discover -s tests -p 'test_query_encoder.py' -v` (before edits) +2. The same focused command again after any source edit +3. `python -m unittest discover -s tests -v` +4. `git diff --check` + +Inspect the historical fixture and current contract before deciding which representation is authoritative. Do not claim the initial failure passed because post-edit checks pass. diff --git a/tests/fixtures/query_migration_triage/legacy_query_golden.json b/tests/fixtures/query_migration_triage/legacy_query_golden.json new file mode 100644 index 0000000..92e502d --- /dev/null +++ b/tests/fixtures/query_migration_triage/legacy_query_golden.json @@ -0,0 +1,5 @@ +{ + "contract_version": "v1", + "pairs": [["q", "blue sky"], ["tag", "a+b"]], + "query": "q=blue%20sky&tag=a%2Bb" +} diff --git a/tests/fixtures/query_migration_triage/query_encoder.py b/tests/fixtures/query_migration_triage/query_encoder.py new file mode 100644 index 0000000..38c6bc6 --- /dev/null +++ b/tests/fixtures/query_migration_triage/query_encoder.py @@ -0,0 +1,9 @@ +"""Serialize ordered query pairs for the request client.""" +from __future__ import annotations + +from urllib.parse import quote, urlencode + + +def serialize_query(pairs: list[tuple[str, str]]) -> str: + """Return query pairs in order, preserving the v1 quoting implementation.""" + return urlencode(pairs, quote_via=quote) diff --git a/tests/fixtures/query_migration_triage/tests/test_query_encoder.py b/tests/fixtures/query_migration_triage/tests/test_query_encoder.py new file mode 100644 index 0000000..d09f2ff --- /dev/null +++ b/tests/fixtures/query_migration_triage/tests/test_query_encoder.py @@ -0,0 +1,21 @@ +import unittest + +from query_encoder import serialize_query + + +class QueryEncoderTests(unittest.TestCase): + def test_v2_space_encoding(self): + self.assertEqual(serialize_query([("q", "blue sky")]), "q=blue+sky") + + def test_literal_plus_remains_escaped(self): + self.assertEqual(serialize_query([("tag", "a+b")]), "tag=a%2Bb") + + def test_pair_order_and_empty_values_are_preserved(self): + self.assertEqual( + serialize_query([("b", "2"), ("a", "")]), + "b=2&a=", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_query_migration_fixture.py b/tests/test_query_migration_fixture.py new file mode 100644 index 0000000..7c56ce4 --- /dev/null +++ b/tests/test_query_migration_fixture.py @@ -0,0 +1,107 @@ +"""Integration guards for the staged synthetic query migration case.""" +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "tests/fixtures/query_migration_triage" +PROFILE_PATH = ROOT / "tests/fixtures/query_migration_triage.case.json" +ORACLE = ROOT / "scripts/pilot_oracles/query_migration.py" +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) +import pilot_contract_triage_pair as pair # noqa: E402 + + +def _run_oracle(fixture: Path) -> subprocess.CompletedProcess[str]: + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + return subprocess.run( + [sys.executable, str(ORACLE), "--fixture-dir", str(fixture)], + cwd=ROOT, env=env, text=True, capture_output=True, check=False, + ) + + +class QueryMigrationFixtureProfileTests(unittest.TestCase): + def test_profile_loads_and_keeps_diagnostic_candidates_separate_from_causal_order(self) -> None: + profile = pair.load_case_profile(PROFILE_PATH) + self.assertEqual(profile.case_id, "query_migration_triage") + self.assertEqual(profile.fixture_source, FIXTURE) + self.assertEqual(profile.source_file, "query_encoder.py") + self.assertEqual(profile.focused_test_file, "tests/test_query_encoder.py") + self.assertEqual(profile.oracle_script, ORACLE) + self.assertEqual( + set(profile.triage_hypotheses), + {"assertion_behavior_regression", "assertion_expectation_drift"}, + ) + self.assertEqual( + set(profile.triage_accepted_ids), + {*profile.triage_hypotheses, "confirm_behavior_contract"}, + ) + # Current runner sends only causal hypotheses; confirm remains accepted + # metadata but is not claimed as a requested diagnostic until the argv + # schema is split from causal ranking. + argv = pair._triage_argv(1, profile) + self.assertIn("assertion_behavior_regression", argv) + self.assertIn("assertion_expectation_drift", argv) + self.assertNotIn("confirm_behavior_contract", argv) + self.assertTrue(profile.rank_hypotheses) + self.assertEqual(profile.triage_accepted_statuses, ("no-remote-choice",)) + + def test_frozen_seed_has_the_expected_focused_failure(self) -> None: + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + result = subprocess.run( + [sys.executable, "-m", "unittest", "discover", "-s", "tests", + "-p", "test_query_encoder.py", "-v"], + cwd=FIXTURE, env=env, text=True, capture_output=True, check=False, + ) + self.assertEqual(result.returncode, 1) + self.assertIn("FAIL: test_v2_space_encoding", result.stderr) + self.assertIn("q=blue%20sky", result.stderr) + self.assertIn("Ran 3 tests", result.stderr) + + def test_oracle_accepts_source_repair_and_git_metadata_but_rejects_other_changes(self) -> None: + with tempfile.TemporaryDirectory(prefix="query-migration-test-") as temp: + root = Path(temp) + repaired = root / "repaired" + shutil.copytree(FIXTURE, repaired) + source = repaired / "query_encoder.py" + text = source.read_text(encoding="utf-8") + self.assertIn("return urlencode(pairs, quote_via=quote)", text) + source.write_text( + text.replace("from urllib.parse import quote, urlencode", + "from urllib.parse import urlencode") + .replace("return urlencode(pairs, quote_via=quote)", + "return urlencode(pairs)"), + encoding="utf-8", + ) + (repaired / ".git").mkdir() + (repaired / ".git/HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + accepted = _run_oracle(repaired) + self.assertEqual(accepted.returncode, 0, accepted.stderr) + self.assertIn("focused/full tests passed", accepted.stdout) + self.assertIn("diff check passed", accepted.stdout) + + tampered = root / "tampered" + shutil.copytree(repaired, tampered) + readme = tampered / "README.md" + readme.write_text(readme.read_text(encoding="utf-8") + "\nchanged\n", + encoding="utf-8") + rejected = _run_oracle(tampered) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("immutable_fixture_file_changed:README.md", rejected.stderr) + + extra = root / "extra" + shutil.copytree(repaired, extra) + (extra / "unexpected.txt").write_text("not part of the fixture\n", encoding="utf-8") + rejected_extra = _run_oracle(extra) + self.assertNotEqual(rejected_extra.returncode, 0) + self.assertIn("fixture_file_set_changed", rejected_extra.stderr) + + +if __name__ == "__main__": + unittest.main()