diff --git a/PILOT.md b/PILOT.md index ab419b3..0c3259a 100644 --- a/PILOT.md +++ b/PILOT.md @@ -1786,3 +1786,9 @@ Next: preserve this outcome unchanged, add bounded private event retention for f ## VCR398 — Optional private event retention for future trials The runner now exposes --retain-private-events to retain bounded raw CLI events in mode-0600 local files before task-specific parsing. Default execution retains its current behavior. Only capture status and byte count belong in receipts; raw prompts, source and outputs must never be copied into public documentation, logs or decision-service requests. The feature must reject oversize archives and existing/symlink targets, preserve timeout/parser-failure evidence, and leave historical VCR396 unchanged. Verification: 39 focused/regression tests pass; the complete local suite passes 818 tests in 37.686 seconds. Green hosted CI remains the merge gate. + +## VCR399 — Tenant-cache local-resolution control + +Added a synthetic source-only repair fixture with immutable contracts, history, a legacy golden and a consumer path. Three offline guards verify the seeded focused/full failure, independent acceptance of the tenant-aware repair, and rejection of source or evidence tampering. Diagnostic action IDs and causal hypotheses remain distinct. + +The current contract resolves the apparent legacy conflict on ordinary inspection. This fixture is a local-resolution/abstention control, not evidence of a genuinely unresolved remote choice, native delivery or advisor benefit. No paid comparison was launched. Oracle SHA-256: `212bd0339054ed8af8017adfca84f3154ae019b11138638317a3865accd99e78`. diff --git a/TASKS.md b/TASKS.md index d004668..eac13bb 100644 --- a/TASKS.md +++ b/TASKS.md @@ -1190,3 +1190,9 @@ Next: preserve this outcome unchanged, add bounded private event retention for f ## VCR398 — Optional private event retention for future trials The runner now exposes --retain-private-events to retain bounded raw CLI events in mode-0600 local files before task-specific parsing. Default execution retains its current behavior. Only capture status and byte count belong in receipts; raw prompts, source and outputs must never be copied into public documentation, logs or decision-service requests. The feature must reject oversize archives and existing/symlink targets, preserve timeout/parser-failure evidence, and leave historical VCR396 unchanged. Verification: 39 focused/regression tests pass; the complete local suite passes 818 tests in 37.686 seconds. Green hosted CI remains the merge gate. + +## VCR399 — Tenant-cache local-resolution control + +Added a synthetic source-only repair fixture with immutable contracts, history, a legacy golden and a consumer path. Three offline guards verify the seeded focused/full failure, independent acceptance of the tenant-aware repair, and rejection of source or evidence tampering. Diagnostic action IDs and causal hypotheses remain distinct. + +The current contract resolves the apparent legacy conflict on ordinary inspection. This fixture is a local-resolution/abstention control, not evidence of a genuinely unresolved remote choice, native delivery or advisor benefit. No paid comparison was launched. Oracle SHA-256: `212bd0339054ed8af8017adfca84f3154ae019b11138638317a3865accd99e78`. diff --git a/scripts/pilot_oracles/ambiguous_cache.py b/scripts/pilot_oracles/ambiguous_cache.py new file mode 100644 index 0000000..8abf76a --- /dev/null +++ b/scripts/pilot_oracles/ambiguous_cache.py @@ -0,0 +1,112 @@ +"""Independent oracle and immutable-evidence verifier for the profile-cache fixture.""" +from __future__ import annotations + +import argparse +import hashlib +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +IMMUTABLE_SHA256 = { + "CHANGELOG.md": "3e60ac0912115b4ddc52a52e1bc934f7116c0e6f773f373bf1a7215ebc5c2076", + "CONTRACT_v1.md": "78a39fb809a7a6ff54f206b26994656ca53367fe0ce738bf08aee28267be1f20", + "CONTRACT_v2.md": "4d0a421a2dbdc634d387bb70ad04b60e4ed2cca08a4cb295b8acce6880f0e970", + "README.md": "77e8c7640879a71ef903d99926edac49a58c8863dc6b0fce1c52a0e13d9de14d", + "legacy_profile_golden.json": "450be32462b8c33599052fcb2a3f77a94a103019a2db241fdfe7c63334c248d1", + "profile_service.py": "1e470b789e634d084c55e5c0fc97e5eab746290ca5c4e734981c786d17cd9d77", + "tests/test_profile_service.py": "f3c2652b81053cf0de5265749222b77dbc52e8a3330ad13e0a3378f6159e4c34", +} +BASELINE_SOURCE = '''"""Tiny in-memory cache used by the fictional profile service.""" + + +def profile_cache_key(tenant_id: str, profile_id: str) -> object: + """Return the cache key for a profile lookup.""" + return profile_id.casefold() + + +def cached_profile(cache: dict, tenant_id: str, profile_id: str, loader): + key = profile_cache_key(tenant_id, profile_id) + if key not in cache: + cache[key] = loader(tenant_id, profile_id) + return cache[key] +''' + + +def run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> None: + result = subprocess.run(command, cwd=cwd, env=env, text=True, capture_output=True) + if result.returncode != 0: + raise SystemExit( + f"independent_check_failed:{command[1:3]}:exit={result.returncode}:" + f"{result.stderr[-1200:]}" + ) + + +parser = argparse.ArgumentParser() +parser.add_argument("--fixture-dir", required=True, type=Path) +args = parser.parse_args() +fixture = args.fixture_dir.resolve(strict=True) + +# Only profile_cache.py may change; task, tests, and consumer remain frozen. +actual_paths = { + path.relative_to(fixture).as_posix() + for path in fixture.rglob("*") + if path.is_file() + and ".git" not in path.relative_to(fixture).parts + and "__pycache__" not in path.relative_to(fixture).parts +} +expected_paths = set(IMMUTABLE_SHA256) | {"profile_cache.py"} +if actual_paths != expected_paths: + raise SystemExit("fixture_file_set_changed") +for relative, expected in IMMUTABLE_SHA256.items(): + actual = hashlib.sha256((fixture / relative).read_bytes()).hexdigest() + if actual != expected: + raise SystemExit(f"immutable_fixture_file_changed:{relative}") + +sys.path.insert(0, str(fixture)) +from profile_cache import cached_profile # noqa: E402 + +# Independent behavior oracle: a normalized cache hit within one tenant, then +# an independent cache entry for the same profile identifier in another tenant. +calls: list[tuple[str, str]] = [] + + +def load(tenant_id: str, profile_id: str) -> tuple[str, str]: + calls.append((tenant_id, profile_id)) + return tenant_id, profile_id + + +cache: dict[object, tuple[str, str]] = {} +north_first = cached_profile(cache, "north", "USER-7", load) +north_hit = cached_profile(cache, "north", "user-7", load) +south_first = cached_profile(cache, "south", "USER-7", load) +if north_first != ("north", "USER-7") or north_hit is not north_first: + raise SystemExit("oracle_mismatch:same_tenant_cache_hit") +if south_first != ("south", "USER-7"): + raise SystemExit("oracle_mismatch:tenant_cache_isolation") +if calls != [("north", "USER-7"), ("south", "USER-7")]: + raise SystemExit("oracle_mismatch:loader_call_count") + +# Independently run the fixture's exact focused and full test commands. +env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") +run([sys.executable, "-m", "unittest", "discover", "-s", "tests", + "-p", "test_profile_service.py", "-v"], cwd=fixture, env=env) +run([sys.executable, "-m", "unittest", "discover", "-s", "tests", "-v"], + cwd=fixture, env=env) + +# Candidate copies have no Git history; check whitespace against a temporary +# baseline containing only the editable source. +with tempfile.TemporaryDirectory(prefix="profile-cache-diff-") as temp: + work = Path(temp) + candidate = work / "profile_cache.py" + candidate.write_text(BASELINE_SOURCE, encoding="utf-8") + run(["git", "init", "-q"], cwd=work) + run(["git", "config", "user.name", "fixture-oracle"], cwd=work) + run(["git", "config", "user.email", "fixture-oracle@invalid"], cwd=work) + run(["git", "add", "profile_cache.py"], cwd=work) + run(["git", "commit", "-q", "-m", "baseline"], cwd=work) + candidate.write_bytes((fixture / "profile_cache.py").read_bytes()) + run(["git", "diff", "--check"], cwd=work) + +print("oracle_passed:same-tenant reuse, cross-tenant isolation; focused/full tests and diff check passed") diff --git a/tests/fixtures/ambiguous_cache_triage.case.json b/tests/fixtures/ambiguous_cache_triage.case.json new file mode 100644 index 0000000..66dd2c4 --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage.case.json @@ -0,0 +1,83 @@ +{ + "case_id": "ambiguous_cache_triage", + "evidence_files": { + "changelog": "CHANGELOG.md", + "contract_current": "CONTRACT_v2.md", + "contract_legacy": "CONTRACT_v1.md", + "focused_test": "tests/test_profile_service.py", + "implementation": "profile_cache.py", + "legacy_golden": "legacy_profile_golden.json", + "service_consumer": "profile_service.py" + }, + "evidence_markers": { + "changelog": [ + "Contract v2 is authoritative for current requests" + ], + "contract_current": [ + "case-folded profile identifier", + "must never be returned for another tenant" + ], + "contract_legacy": [ + "shared across the service process" + ], + "focused_test": [ + "test_identical_profile_ids_are_isolated_between_tenants" + ], + "implementation": [ + "return profile_id.casefold()" + ], + "legacy_golden": [ + "\"expected_legacy_cache_entries\": 1" + ], + "service_consumer": [ + "return cached_profile(cache, tenant_id, profile_id, provider)" + ] + }, + "failure_markers": [ + "test_identical_profile_ids_are_isolated_between_tenants", + "AssertionError", + "Ran 3 tests" + ], + "fixture_source": "tests/fixtures/ambiguous_cache_triage", + "focused_command": [ + "python", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_profile_service.py", + "-v" + ], + "focused_test_file": "tests/test_profile_service.py", + "oracle_script": "scripts/pilot_oracles/ambiguous_cache.py", + "oracle_sha256": "212bd0339054ed8af8017adfca84f3154ae019b11138638317a3865accd99e78", + "outcome_mode": "repair", + "schema_version": 1, + "source_file": "profile_cache.py", + "task_prompt_file": "README.md", + "triage": { + "accepted_ids": [ + "assertion_behavior_regression", + "assertion_expectation_drift", + "confirm_behavior_contract" + ], + "accepted_statuses": [ + "no-remote-choice" + ], + "hypotheses": [ + "assertion_behavior_regression", + "assertion_expectation_drift" + ], + "kinds": [ + "assertion" + ], + "observations": { + "assertion": [ + "legacy_fixture_conflict" + ] + }, + "rank_hypotheses": true + } +} diff --git a/tests/fixtures/ambiguous_cache_triage/CHANGELOG.md b/tests/fixtures/ambiguous_cache_triage/CHANGELOG.md new file mode 100644 index 0000000..d7d7345 --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/CHANGELOG.md @@ -0,0 +1,3 @@ +# Change log + +2026-04-01: Profile lookup moved to tenant-scoped accounts. Contract v2 is authoritative for current requests; v1 remains historical context only. diff --git a/tests/fixtures/ambiguous_cache_triage/CONTRACT_v1.md b/tests/fixtures/ambiguous_cache_triage/CONTRACT_v1.md new file mode 100644 index 0000000..75e72f1 --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/CONTRACT_v1.md @@ -0,0 +1,3 @@ +# Profile cache contract v1 + +Effective through 2026-03-31. Profile identifiers are case-insensitive. Cache entries are shared across the service process by normalized profile identifier, regardless of tenant. diff --git a/tests/fixtures/ambiguous_cache_triage/CONTRACT_v2.md b/tests/fixtures/ambiguous_cache_triage/CONTRACT_v2.md new file mode 100644 index 0000000..b07f37a --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/CONTRACT_v2.md @@ -0,0 +1,3 @@ +# Profile cache contract v2 + +Effective 2026-04-01 and superseding v1. Profile identifiers are case-insensitive within a tenant. Cache identity is the pair of tenant identifier and case-folded profile identifier. A cached response for one tenant must never be returned for another tenant. diff --git a/tests/fixtures/ambiguous_cache_triage/README.md b/tests/fixtures/ambiguous_cache_triage/README.md new file mode 100644 index 0000000..e639968 --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/README.md @@ -0,0 +1,5 @@ +# Repair the profile cache + +A focused test fails when two tenants request the same profile identifier. Determine whether the implementation or the expected behavior is stale by inspecting the current contract, its history, the golden example, the consumer, and the tests. + +Make a source-only repair in `profile_cache.py`. Keep the function signatures and same-tenant cache behavior. Do not alter the tests, contracts, changelog, golden example, or `profile_service.py`. Do not invent requirements. Run the configured focused tests, then the full fixture tests, and check the diff. Record the initial failure separately from post-edit checks. diff --git a/tests/fixtures/ambiguous_cache_triage/legacy_profile_golden.json b/tests/fixtures/ambiguous_cache_triage/legacy_profile_golden.json new file mode 100644 index 0000000..825c2bc --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/legacy_profile_golden.json @@ -0,0 +1,7 @@ +{ + "contract_version": "v1", + "profile_id": "USER-7", + "shared_cache_key": "user-7", + "tenants": ["north", "south"], + "expected_legacy_cache_entries": 1 +} diff --git a/tests/fixtures/ambiguous_cache_triage/profile_cache.py b/tests/fixtures/ambiguous_cache_triage/profile_cache.py new file mode 100644 index 0000000..adf3538 --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/profile_cache.py @@ -0,0 +1,13 @@ +"""Tiny in-memory cache used by the fictional profile service.""" + + +def profile_cache_key(tenant_id: str, profile_id: str) -> object: + """Return the cache key for a profile lookup.""" + return profile_id.casefold() + + +def cached_profile(cache: dict, tenant_id: str, profile_id: str, loader): + key = profile_cache_key(tenant_id, profile_id) + if key not in cache: + cache[key] = loader(tenant_id, profile_id) + return cache[key] diff --git a/tests/fixtures/ambiguous_cache_triage/profile_service.py b/tests/fixtures/ambiguous_cache_triage/profile_service.py new file mode 100644 index 0000000..d7f8a5b --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/profile_service.py @@ -0,0 +1,6 @@ +"""Consumer path for retrieving tenant-scoped profiles.""" +from profile_cache import cached_profile + + +def load_profile(cache, tenant_id, profile_id, provider): + return cached_profile(cache, tenant_id, profile_id, provider) diff --git a/tests/fixtures/ambiguous_cache_triage/tests/test_profile_service.py b/tests/fixtures/ambiguous_cache_triage/tests/test_profile_service.py new file mode 100644 index 0000000..b7acfa4 --- /dev/null +++ b/tests/fixtures/ambiguous_cache_triage/tests/test_profile_service.py @@ -0,0 +1,35 @@ +"""Behavior tests for the profile cache and its service consumer.""" +import unittest + +from profile_cache import cached_profile +from profile_service import load_profile + + +def _provider(tenant_id, profile_id): + return {"tenant": tenant_id, "profile": profile_id} + + +class ProfileCacheTests(unittest.TestCase): + def test_same_tenant_reuses_case_insensitive_profile_key(self): + cache = {} + first = cached_profile(cache, "north", "USER-7", _provider) + second = cached_profile(cache, "north", "user-7", _provider) + self.assertIs(first, second) + self.assertEqual(first["tenant"], "north") + + def test_identical_profile_ids_are_isolated_between_tenants(self): + cache = {} + north = cached_profile(cache, "north", "USER-7", _provider) + south = cached_profile(cache, "south", "USER-7", _provider) + self.assertEqual(north["tenant"], "north") + self.assertEqual(south["tenant"], "south") + + def test_service_consumer_does_not_cross_tenant_cache_entries(self): + cache = {} + north = load_profile(cache, "north", "USER-7", _provider) + south = load_profile(cache, "south", "USER-7", _provider) + self.assertEqual((north["tenant"], south["tenant"]), ("north", "south")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ambiguous_cache_fixture.py b/tests/test_ambiguous_cache_fixture.py new file mode 100644 index 0000000..1fc2d90 --- /dev/null +++ b/tests/test_ambiguous_cache_fixture.py @@ -0,0 +1,122 @@ +"""Offline guards for the synthetic profile-cache triage fixture.""" +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "tests/fixtures/ambiguous_cache_triage" +PROFILE_PATH = ROOT / "tests/fixtures/ambiguous_cache_triage.case.json" +ORACLE = ROOT / "scripts/pilot_oracles/ambiguous_cache.py" +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) +import pilot_contract_triage_pair as pair # noqa: E402 + + +def _run(command: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + return subprocess.run( + command, + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=15, + check=False, + ) + + +def _run_oracle(fixture: Path) -> subprocess.CompletedProcess[str]: + return _run([sys.executable, str(ORACLE), "--fixture-dir", str(fixture)], cwd=ROOT) + + +class AmbiguousCacheFixtureTests(unittest.TestCase): + def test_case_profile_loads_with_two_causal_candidates_and_separate_diagnostic(self): + profile = pair.load_case_profile(PROFILE_PATH) + self.assertEqual(profile.case_id, "ambiguous_cache_triage") + self.assertEqual(profile.fixture_source, FIXTURE) + self.assertEqual(profile.source_file, "profile_cache.py") + self.assertEqual(profile.focused_test_file, "tests/test_profile_service.py") + self.assertEqual(profile.oracle_script, ORACLE) + self.assertEqual( + set(profile.triage_hypotheses), + {"assertion_behavior_regression", "assertion_expectation_drift"}, + ) + self.assertIn("confirm_behavior_contract", profile.triage_accepted_ids) + self.assertEqual( + profile.triage_observations, + {"assertion": ("legacy_fixture_conflict",)}, + ) + + def test_seed_fails_focused_and_full_with_the_expected_behavioral_failure(self): + profile = pair.load_case_profile(PROFILE_PATH) + focused = _run(list(profile.focused_command), cwd=FIXTURE) + self.assertEqual(focused.returncode, 1, focused.stdout) + self.assertIn("FAIL: test_identical_profile_ids_are_isolated_between_tenants", focused.stdout) + self.assertIn("AssertionError", focused.stdout) + self.assertIn("Ran 3 tests", focused.stdout) + + full = _run( + [sys.executable, "-m", "unittest", "discover", "-s", "tests", "-v"], + cwd=FIXTURE, + ) + self.assertEqual(full.returncode, 1, full.stdout) + self.assertIn("FAIL: test_identical_profile_ids_are_isolated_between_tenants", full.stdout) + self.assertIn("Ran 3 tests", full.stdout) + + def test_independent_oracle_accepts_repair_and_rejects_tampered_source_or_evidence(self): + with tempfile.TemporaryDirectory(prefix="ambiguous-cache-test-") as temp: + root = Path(temp) + repaired = root / "repaired" + shutil.copytree(FIXTURE, repaired) + source = repaired / "profile_cache.py" + seeded = source.read_text(encoding="utf-8") + self.assertIn("return profile_id.casefold()", seeded) + source.write_text( + seeded.replace( + "return profile_id.casefold()", + "return (tenant_id, profile_id.casefold())", + 1, + ), + encoding="utf-8", + ) + accepted = _run_oracle(repaired) + self.assertEqual(accepted.returncode, 0, accepted.stdout) + self.assertIn("oracle_passed", accepted.stdout) + + tampered = root / "tampered" + shutil.copytree(repaired, tampered) + changed = tampered / "profile_cache.py" + text = changed.read_text(encoding="utf-8") + changed.write_text( + text.replace( + "return (tenant_id, profile_id.casefold())", + "return profile_id.casefold()", + 1, + ), + encoding="utf-8", + ) + rejected = _run_oracle(tampered) + self.assertNotEqual(rejected.returncode, 0) + self.assertIn("oracle_mismatch:tenant_cache_isolation", rejected.stdout) + + evidence_tamper = root / "evidence-tamper" + shutil.copytree(repaired, evidence_tamper) + changelog = evidence_tamper / "CHANGELOG.md" + changelog.write_text( + changelog.read_text(encoding="utf-8") + "\nEdited evidence.\n", + encoding="utf-8", + ) + rejected_evidence = _run_oracle(evidence_tamper) + self.assertNotEqual(rejected_evidence.returncode, 0) + self.assertIn("immutable_fixture_file_changed:CHANGELOG.md", rejected_evidence.stdout) + + +if __name__ == "__main__": + unittest.main()