Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions PILOT.md
Original file line number Diff line number Diff line change
Expand Up @@ -1731,3 +1731,9 @@ This integration is an offline measurement capability, not delivery or efficacy
## VCR391 — Profile advice acknowledgment timing

The profile event parser now observes acknowledgment after both legacy and configured valid triage results. A real synthetic event stream verifies acknowledgment before the next tool and distinguishes missing or late acknowledgment. This fixes delivery measurement only; it does not establish native delivery or efficacy.

## VCR389 — Prospective query migration repair fixture

Staged a new synthetic URL query migration task with versioned contracts, historical evidence, immutable tests, and an external hash-pinned oracle. Offline checks establish the expected initial failure, a source-only reference repair, and rejection of protected-file tampering or unexpected files. The oracle independently runs focused/full checks and checks the source patch.

This is preparation, not a launched pair or efficacy evidence. Launch remains gated on symmetric arm Git initialization and observed agent diff validation. The accepted diagnostic metadata includes contract confirmation, but the current runner does not yet request that candidate; causal ranking and diagnostic selection require separate integration.
6 changes: 6 additions & 0 deletions TASKS.md
Original file line number Diff line number Diff line change
Expand Up @@ -1135,3 +1135,9 @@ The runner now writes the bounded redacted measurement to its private output dir
## VCR391 — Profile advice acknowledgment timing

The profile event parser now observes acknowledgment after both legacy and configured valid triage results. A real synthetic event stream verifies acknowledgment before the next tool and distinguishes missing or late acknowledgment. This fixes delivery measurement only; it does not establish native delivery or efficacy.

## VCR389 — Prospective query migration repair fixture

Staged a new synthetic URL query migration task with versioned contracts, historical evidence, immutable tests, and an external hash-pinned oracle. Offline checks establish the expected initial failure, a source-only reference repair, and rejection of protected-file tampering or unexpected files. The oracle independently runs focused/full checks and checks the source patch.

This is preparation, not a launched pair or efficacy evidence. Launch remains gated on symmetric arm Git initialization and observed agent diff validation. The accepted diagnostic metadata includes contract confirmation, but the current runner does not yet request that candidate; causal ranking and diagnostic selection require separate integration.
97 changes: 97 additions & 0 deletions scripts/pilot_oracles/query_migration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""Independent oracle and immutable-check verifier for the query migration fixture."""
from __future__ import annotations

import argparse
import hashlib
import os
import subprocess
import sys
import tempfile
from pathlib import Path

IMMUTABLE_SHA256 = {
"CHANGELOG.md": "ccfb22eb8fb0c75773bea70f1f7d3566ec7d6c1f5743388664bd92655e21835d",
"CONTRACT_v1.md": "2aa3178710bc90b6c073518e87652c760ff5357674b2f7153f605e0074215412",
"CONTRACT_v2.md": "c10decdef269169115650ff34c76b1c1be3517756869466e7194561591af4d96",
"README.md": "15cb774071942bc09b8d1fd32f88053a9d4be4fdeed71b207eb5b8d68c56d2cc",
"legacy_query_golden.json": "e12b45986a7bace7f792aba8a49ca7e1749f7ec94e433a620daf90726695c870",
"tests/test_query_encoder.py": "61e8874a8dbdc45e790ccf0a357fa7de812e235a2e8bfaeb6146efa98341da32",
}
BASELINE_SOURCE = '''"""Serialize ordered query pairs for the request client."""
from __future__ import annotations

from urllib.parse import quote, urlencode


def serialize_query(pairs: list[tuple[str, str]]) -> str:
"""Return query pairs in order, preserving the v1 quoting implementation."""
return urlencode(pairs, quote_via=quote)
'''


def run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> None:
result = subprocess.run(command, cwd=cwd, env=env, text=True, capture_output=True)
if result.returncode != 0:
raise SystemExit(
f"independent_check_failed:{command[1:3]}:exit={result.returncode}:"
f"{result.stderr[-1200:]}"
)


parser = argparse.ArgumentParser()
parser.add_argument("--fixture-dir", required=True, type=Path)
args = parser.parse_args()
fixture = args.fixture_dir.resolve(strict=True)

# The only allowed source edit is query_encoder.py; all task/test evidence is frozen.
actual_paths = {
path.relative_to(fixture).as_posix()
for path in fixture.rglob("*")
if path.is_file()
and ".git" not in path.relative_to(fixture).parts
and "__pycache__" not in path.relative_to(fixture).parts
}
expected_paths = set(IMMUTABLE_SHA256) | {"query_encoder.py"}
if actual_paths != expected_paths:
raise SystemExit("fixture_file_set_changed")
for relative, expected in IMMUTABLE_SHA256.items():
actual = hashlib.sha256((fixture / relative).read_bytes()).hexdigest()
if actual != expected:
raise SystemExit(f"immutable_fixture_file_changed:{relative}")

spec_source = fixture / "query_encoder.py"
# Independent behavior oracle.
spec_text = spec_source.read_text(encoding="utf-8")
namespace: dict[str, object] = {}
exec(compile(spec_text, str(spec_source), "exec"), namespace)
serialize_query = namespace["serialize_query"]
cases = [
([ ("q", "blue sky"), ("tag", "a+b") ], "q=blue+sky&tag=a%2Bb"),
([ ("city", "Ålesund café") ], "city=%C3%85lesund+caf%C3%A9"),
([ ("b", "2"), ("a", "") ], "b=2&a="),
]
for pairs, expected in cases:
actual = serialize_query(pairs) # type: ignore[operator]
if actual != expected:
raise SystemExit(f"oracle_mismatch:{expected!r}:{actual!r}")

# Re-run the exact focused and full fixture commands independently.
env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
run([sys.executable, "-m", "unittest", "discover", "-s", "tests", "-p", "test_query_encoder.py", "-v"], cwd=fixture, env=env)
run([sys.executable, "-m", "unittest", "discover", "-s", "tests", "-v"], cwd=fixture, env=env)

# The copied arm has no .git directory. Build a throwaway baseline index so the
# exact candidate patch can still receive an independent `git diff --check`.
with tempfile.TemporaryDirectory(prefix="query-migration-diff-") as temp:
work = Path(temp)
candidate = work / "query_encoder.py"
candidate.write_text(BASELINE_SOURCE, encoding="utf-8")
run(["git", "init", "-q"], cwd=work)
run(["git", "config", "user.name", "fixture-oracle"], cwd=work)
run(["git", "config", "user.email", "fixture-oracle@invalid"], cwd=work)
run(["git", "add", "query_encoder.py"], cwd=work)
run(["git", "commit", "-q", "-m", "baseline"], cwd=work)
candidate.write_bytes(spec_source.read_bytes())
run(["git", "diff", "--check"], cwd=work)

print("oracle_passed:3 cases; focused/full tests passed; immutable files and diff check passed")
80 changes: 80 additions & 0 deletions tests/fixtures/query_migration_triage.case.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
{
"case_id": "query_migration_triage",
"evidence_files": {
"changelog": "CHANGELOG.md",
"contract_v1": "CONTRACT_v1.md",
"contract_v2": "CONTRACT_v2.md",
"focused_test": "tests/test_query_encoder.py",
"implementation": "query_encoder.py",
"legacy_golden": "legacy_query_golden.json"
},
"evidence_markers": {
"changelog": [
"The v2 contract is authoritative for current requests"
],
"contract_v1": [
"Effective through 2025-06-30"
],
"contract_v2": [
"Effective 2025-07-01 and superseding v1",
"The v1 golden remains a historical record"
],
"focused_test": [
"q=blue+sky"
],
"implementation": [
"quote_via=quote"
],
"legacy_golden": [
"\"contract_version\": \"v1\"",
"q=blue%20sky&tag=a%2Bb"
]
},
"failure_markers": [
"test_v2_space_encoding",
"q=blue%20sky",
"q=blue+sky"
],
"fixture_source": "tests/fixtures/query_migration_triage",
"focused_command": [
"python",
"-m",
"unittest",
"discover",
"-s",
"tests",
"-p",
"test_query_encoder.py",
"-v"
],
"focused_test_file": "tests/test_query_encoder.py",
"oracle_script": "scripts/pilot_oracles/query_migration.py",
"oracle_sha256": "da6110537d48a8bc497bf5e24d1dc9bd5584ff3950a6c3c8b6a650db0261b114",
"outcome_mode": "repair",
"schema_version": 1,
"source_file": "query_encoder.py",
"task_prompt_file": "README.md",
"triage": {
"accepted_ids": [
"assertion_behavior_regression",
"assertion_expectation_drift",
"confirm_behavior_contract"
],
"accepted_statuses": [
"no-remote-choice"
],
"hypotheses": [
"assertion_behavior_regression",
"assertion_expectation_drift"
],
"kinds": [
"assertion"
],
"observations": {
"assertion": [
"legacy_fixture_conflict"
]
},
"rank_hypotheses": true
}
}
5 changes: 5 additions & 0 deletions tests/fixtures/query_migration_triage/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Changelog

## 2025-07-01 — Query protocol v2

The upstream request format migrated from the v1 URI query convention to form-style query encoding. The v2 contract is authoritative for current requests. Preserve pair order and literal plus escaping; only space representation changes from `%20` to `+`.
3 changes: 3 additions & 0 deletions tests/fixtures/query_migration_triage/CONTRACT_v1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Query serialization contract, v1

Effective through 2025-06-30. Query pairs preserve caller order. For each value, encode UTF-8 bytes and percent-escape spaces as `%20`; a literal plus is `%2B`. This was the legacy RFC 3986-style representation.
3 changes: 3 additions & 0 deletions tests/fixtures/query_migration_triage/CONTRACT_v2.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Query serialization contract, v2

Effective 2025-07-01 and superseding v1 for current application requests. Preserve query pair order and values. Encode query pairs with `application/x-www-form-urlencoded` semantics: a space is `+`, while a literal plus remains `%2B`. The v1 golden remains a historical record and is not the v2 expected output.
12 changes: 12 additions & 0 deletions tests/fixtures/query_migration_triage/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Task: repair migrated query serialization

The query serializer fails a new compatibility test after a protocol migration. Determine whether the implementation or the test expectation is out of date by inspecting the versioned local contract, historical fixture, current test, and implementation. Preserve the public function signature, input ordering, and value semantics. Do not edit or weaken tests, contract documents, changelog, or historical fixture; only `query_encoder.py` may change.

Run these as separate commands and retain the observed exit from each:

1. `python -m unittest discover -s tests -p 'test_query_encoder.py' -v` (before edits)
2. The same focused command again after any source edit
3. `python -m unittest discover -s tests -v`
4. `git diff --check`

Inspect the historical fixture and current contract before deciding which representation is authoritative. Do not claim the initial failure passed because post-edit checks pass.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"contract_version": "v1",
"pairs": [["q", "blue sky"], ["tag", "a+b"]],
"query": "q=blue%20sky&tag=a%2Bb"
}
9 changes: 9 additions & 0 deletions tests/fixtures/query_migration_triage/query_encoder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
"""Serialize ordered query pairs for the request client."""
from __future__ import annotations

from urllib.parse import quote, urlencode


def serialize_query(pairs: list[tuple[str, str]]) -> str:
"""Return query pairs in order, preserving the v1 quoting implementation."""
return urlencode(pairs, quote_via=quote)
21 changes: 21 additions & 0 deletions tests/fixtures/query_migration_triage/tests/test_query_encoder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import unittest

from query_encoder import serialize_query


class QueryEncoderTests(unittest.TestCase):
def test_v2_space_encoding(self):
self.assertEqual(serialize_query([("q", "blue sky")]), "q=blue+sky")

def test_literal_plus_remains_escaped(self):
self.assertEqual(serialize_query([("tag", "a+b")]), "tag=a%2Bb")

def test_pair_order_and_empty_values_are_preserved(self):
self.assertEqual(
serialize_query([("b", "2"), ("a", "")]),
"b=2&a=",
)


if __name__ == "__main__":
unittest.main()
107 changes: 107 additions & 0 deletions tests/test_query_migration_fixture.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
"""Integration guards for the staged synthetic query migration case."""
from __future__ import annotations

import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests/fixtures/query_migration_triage"
PROFILE_PATH = ROOT / "tests/fixtures/query_migration_triage.case.json"
ORACLE = ROOT / "scripts/pilot_oracles/query_migration.py"
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "scripts"))
import pilot_contract_triage_pair as pair # noqa: E402


def _run_oracle(fixture: Path) -> subprocess.CompletedProcess[str]:
env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
return subprocess.run(
[sys.executable, str(ORACLE), "--fixture-dir", str(fixture)],
cwd=ROOT, env=env, text=True, capture_output=True, check=False,
)


class QueryMigrationFixtureProfileTests(unittest.TestCase):
def test_profile_loads_and_keeps_diagnostic_candidates_separate_from_causal_order(self) -> None:
profile = pair.load_case_profile(PROFILE_PATH)
self.assertEqual(profile.case_id, "query_migration_triage")
self.assertEqual(profile.fixture_source, FIXTURE)
self.assertEqual(profile.source_file, "query_encoder.py")
self.assertEqual(profile.focused_test_file, "tests/test_query_encoder.py")
self.assertEqual(profile.oracle_script, ORACLE)
self.assertEqual(
set(profile.triage_hypotheses),
{"assertion_behavior_regression", "assertion_expectation_drift"},
)
self.assertEqual(
set(profile.triage_accepted_ids),
{*profile.triage_hypotheses, "confirm_behavior_contract"},
)
# Current runner sends only causal hypotheses; confirm remains accepted
# metadata but is not claimed as a requested diagnostic until the argv
# schema is split from causal ranking.
argv = pair._triage_argv(1, profile)
self.assertIn("assertion_behavior_regression", argv)
self.assertIn("assertion_expectation_drift", argv)
self.assertNotIn("confirm_behavior_contract", argv)
self.assertTrue(profile.rank_hypotheses)
self.assertEqual(profile.triage_accepted_statuses, ("no-remote-choice",))

def test_frozen_seed_has_the_expected_focused_failure(self) -> None:
env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
result = subprocess.run(
[sys.executable, "-m", "unittest", "discover", "-s", "tests",
"-p", "test_query_encoder.py", "-v"],
cwd=FIXTURE, env=env, text=True, capture_output=True, check=False,
)
self.assertEqual(result.returncode, 1)
self.assertIn("FAIL: test_v2_space_encoding", result.stderr)
self.assertIn("q=blue%20sky", result.stderr)
self.assertIn("Ran 3 tests", result.stderr)

def test_oracle_accepts_source_repair_and_git_metadata_but_rejects_other_changes(self) -> None:
with tempfile.TemporaryDirectory(prefix="query-migration-test-") as temp:
root = Path(temp)
repaired = root / "repaired"
shutil.copytree(FIXTURE, repaired)
source = repaired / "query_encoder.py"
text = source.read_text(encoding="utf-8")
self.assertIn("return urlencode(pairs, quote_via=quote)", text)
source.write_text(
text.replace("from urllib.parse import quote, urlencode",
"from urllib.parse import urlencode")
.replace("return urlencode(pairs, quote_via=quote)",
"return urlencode(pairs)"),
encoding="utf-8",
)
(repaired / ".git").mkdir()
(repaired / ".git/HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8")
accepted = _run_oracle(repaired)
self.assertEqual(accepted.returncode, 0, accepted.stderr)
self.assertIn("focused/full tests passed", accepted.stdout)
self.assertIn("diff check passed", accepted.stdout)

tampered = root / "tampered"
shutil.copytree(repaired, tampered)
readme = tampered / "README.md"
readme.write_text(readme.read_text(encoding="utf-8") + "\nchanged\n",
encoding="utf-8")
rejected = _run_oracle(tampered)
self.assertNotEqual(rejected.returncode, 0)
self.assertIn("immutable_fixture_file_changed:README.md", rejected.stderr)

extra = root / "extra"
shutil.copytree(repaired, extra)
(extra / "unexpected.txt").write_text("not part of the fixture\n", encoding="utf-8")
rejected_extra = _run_oracle(extra)
self.assertNotEqual(rejected_extra.returncode, 0)
self.assertIn("fixture_file_set_changed", rejected_extra.stderr)


if __name__ == "__main__":
unittest.main()