diff --git a/PILOT.md b/PILOT.md index e8b35dd..990be21 100644 --- a/PILOT.md +++ b/PILOT.md @@ -1823,3 +1823,10 @@ One prospective pair ran from merged `ef2f127e053e5f00d4b2fc66705d7421a5fdca5e`, Both source-only repairs passed the immutable oracle, and tests/protected evidence were unchanged. Both task outcomes remain incomplete because mandatory agent command observations were missing. Treatment acknowledgment was invalid, no bridge request occurred, and measured Jev transport calls were zero. The overall receipt is incomplete with failed task acceptance and delivery. Agent exit zero and independent oracle pass do not replace the missing gates. No speed, ranking, quality or native delivery benefit is established. The retained private events show test invocations that differ from the frozen required argv: focused tests used module invocation rather than the configured discovery command, and the baseline full suite omitted the configured verbosity flag. Simple Bash wrappers were already supported; their presence does not establish the cause of missing observations. Combined command exits cannot prove individual checks. Future trials must expose exact mandatory commands symmetrically to both agents; wrapper hardening is a separate prospective improvement. These findings do not explain every missing gate or acknowledgment, and do not rescore this frozen result. Raw events remain private. + + +## VCR404 — Symmetric mandatory-command visibility + +Future repair-profile trials expose exact standalone validation commands to both baseline and treatment before treatment-specific advice instructions. The initial focused check precedes edits; post-edit focused, full-suite and diff checks remain mandatory. Optional initial-failure triage stays optional. Default and non-repair prompts remain unchanged. Historical trials are not rescored or rerun. + +Two new symmetry/compatibility tests, nine profile tests and fifteen triage tests pass. Graph refreshed (3439 nodes, 6944 edges, 239 communities). The concurrent shell-parser patch still has a separate diagnostic test failure and is excluded from this commit. Hosted CI on this isolated change is required before merge. No coding benefit or native-host delivery is claimed. diff --git a/TASKS.md b/TASKS.md index dcf39f3..0cd9d9a 100644 --- a/TASKS.md +++ b/TASKS.md @@ -1227,3 +1227,10 @@ One prospective pair ran from merged `ef2f127e053e5f00d4b2fc66705d7421a5fdca5e`, Both source-only repairs passed the immutable oracle, and tests/protected evidence were unchanged. Both task outcomes remain incomplete because mandatory agent command observations were missing. Treatment acknowledgment was invalid, no bridge request occurred, and measured Jev transport calls were zero. The overall receipt is incomplete with failed task acceptance and delivery. Agent exit zero and independent oracle pass do not replace the missing gates. No speed, ranking, quality or native delivery benefit is established. The retained private events show test invocations that differ from the frozen required argv: focused tests used module invocation rather than the configured discovery command, and the baseline full suite omitted the configured verbosity flag. Simple Bash wrappers were already supported; their presence does not establish the cause of missing observations. Combined command exits cannot prove individual checks. Future trials must expose exact mandatory commands symmetrically to both agents; wrapper hardening is a separate prospective improvement. These findings do not explain every missing gate or acknowledgment, and do not rescore this frozen result. Raw events remain private. + + +## VCR404 — Symmetric mandatory-command visibility + +Future repair-profile trials expose exact standalone validation commands to both baseline and treatment before treatment-specific advice instructions. The initial focused check precedes edits; post-edit focused, full-suite and diff checks remain mandatory. Optional initial-failure triage stays optional. Default and non-repair prompts remain unchanged. Historical trials are not rescored or rerun. + +Two new symmetry/compatibility tests, nine profile tests and fifteen triage tests pass. Graph refreshed (3439 nodes, 6944 edges, 239 communities). The concurrent shell-parser patch still has a separate diagnostic test failure and is excluded from this commit. Hosted CI on this isolated change is required before merge. No coding benefit or native-host delivery is claimed. diff --git a/scripts/pilot_contract_triage_pair.py b/scripts/pilot_contract_triage_pair.py index 7af8d39..c635979 100644 --- a/scripts/pilot_contract_triage_pair.py +++ b/scripts/pilot_contract_triage_pair.py @@ -348,6 +348,28 @@ def load_case_profile(path: Path) -> CaseProfile: rank_hypotheses, outcome, oracle_script, expected_hash, ) +def _case_base_prompt(profile: CaseProfile | None) -> str: + """Build the shared profiled prompt with exact mandatory validation commands.""" + if profile is None: + return BASE_PROMPT + prompt = profile.task_prompt + if profile.outcome_mode != "repair": + return prompt + + focused_command = shlex.join(profile.focused_command) + full_command = shlex.join(shlex.split(REQUIRED_COMMAND)) + diff_command = shlex.join(("git", "diff", "--check")) + return ( + prompt + + "\n\nMandatory validation for both arms. Run every command as a separate " + "standalone command, without changing arguments or substituting an equivalent. " + "Before editing source, run the initial focused test and record its exit code: " + + f"`{focused_command}`. " + "After edits, run these commands separately in this order: focused test " + + f"`{focused_command}`, full test suite `{full_command}`, and diff check " + + f"`{diff_command}`. Preserve each observed exit code in your report." + ) + def _profile_bridge_spec( profile: CaseProfile, *, include_observations: bool = True, @@ -371,6 +393,7 @@ def _case_treatment_prompt( triage_stage: str = "evidence-reviewed", ) -> str: include_observations = triage_stage != "initial-failure" + base_prompt = _case_base_prompt(profile) evidence_paths = ", ".join(profile.evidence_files.values()) triage_command = shlex.join(_triage_argv( 1, profile, include_observations=include_observations, @@ -413,13 +436,13 @@ def _case_treatment_prompt( ) if advice_policy == "nonbinding": return ( - profile.task_prompt + "\n\nBefore using a tool, make your first assistant message contain this " + base_prompt + "\n\nBefore using a tool, make your first assistant message contain this " "exact line: " + WORKFLOW_ACK_LINE + " If you request triage, after its result and before using another tool, " + "make your next assistant message contain this exact line: " + PROFILE_TRIAGE_RESULT_ACK_LINE + guidance ) - return profile.task_prompt + guidance + return base_prompt + guidance def _matches_useful_failure(output: Any, profile: CaseProfile | None = None) -> bool: @@ -1428,7 +1451,7 @@ def run_pair( answers: dict[str, str | None] = {} for true_arm in order: label = labels[true_arm] - prompt = (BASE_PROMPT if case_profile is None else case_profile.task_prompt) + prompt = (BASE_PROMPT if case_profile is None else _case_base_prompt(case_profile)) if true_arm == "treatment": prompt = ( NONBINDING_TREATMENT_PROMPT if case_profile is None and advice_policy == "nonbinding" diff --git a/tests/test_pilot_mandatory_command_visibility.py b/tests/test_pilot_mandatory_command_visibility.py new file mode 100644 index 0000000..0c659c7 --- /dev/null +++ b/tests/test_pilot_mandatory_command_visibility.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import shlex +import sys +import unittest +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "pilot_contract_triage_pair.py" +sys.path.insert(0, str(ROOT / "scripts")) +SPEC = importlib.util.spec_from_file_location("pilot_contract_triage_pair_commands", SCRIPT) +runner = importlib.util.module_from_spec(SPEC) +assert SPEC and SPEC.loader +sys.modules[SPEC.name] = runner +SPEC.loader.exec_module(runner) + + +class MandatoryCommandVisibilityTests(unittest.TestCase): + def setUp(self): + self.profile = SimpleNamespace( + task_prompt="Repair service.py to match the contract.", + outcome_mode="repair", + focused_command=( + "python", "-m", "unittest", "discover", "-s", "tests", + "-p", "test_service.py", "-v", + ), + source_file="src/service.py", + focused_test_file="tests/test_service.py", + evidence_files={"contract": "contract.md"}, + triage_kinds=("assertion",), + triage_hypotheses=( + "assertion_behavior_regression", + "assertion_expectation_drift", + ), + triage_accepted_ids=(), + rank_hypotheses=False, + triage_observations={}, + ) + + def test_exact_standalone_commands_are_shared_before_treatment_extras(self): + focused = shlex.join(self.profile.focused_command) + full = shlex.join(shlex.split(runner.REQUIRED_COMMAND)) + diff = shlex.join(("git", "diff", "--check")) + + baseline_prompt = runner._case_base_prompt(self.profile) + treatment_prompt = runner._case_treatment_prompt( + self.profile, "legacy-required-step", "initial-failure", + ) + + self.assertTrue(treatment_prompt.startswith(baseline_prompt)) + for command in (focused, full, diff): + self.assertIn(f"`{command}`", baseline_prompt) + self.assertIn(f"`{command}`", treatment_prompt) + self.assertIn("Before editing source, run the initial focused test", baseline_prompt) + self.assertIn("After edits, run these commands separately in this order", baseline_prompt) + self.assertIn("standalone command", baseline_prompt) + self.assertIn("may request the exact enum-only triage command", treatment_prompt) + self.assertIn("The request is optional", treatment_prompt) + + def test_legacy_and_nonrepair_prompt_behavior_is_unchanged(self): + self.assertEqual(runner._case_base_prompt(None), runner.BASE_PROMPT) + contract_profile = SimpleNamespace( + task_prompt="Inspect and preserve the original failure.", + outcome_mode="contract_triage", + ) + self.assertEqual(runner._case_base_prompt(contract_profile), contract_profile.task_prompt) + + +if __name__ == "__main__": + unittest.main()