From 5d387f02d84ccbc5f16518ecb6202863690696d0 Mon Sep 17 00:00:00 2001 From: Raphael Shu Date: Sat, 17 Oct 2026 10:01:57 -0700 Subject: [PATCH] Document benchmark metrics and publish checkpoint PSR reconstruction --- README.md | 3 +- docs/benchmark/metrics.md | 200 ++++++++++++++++++++++++++++++++++ docs/benchmark/scoring.md | 17 ++- scripts/score_psr.py | 208 ++++++++++++++++++++++++++++++++++++ tests/benchmark/test_psr.py | 118 ++++++++++++++++++++ 5 files changed, 540 insertions(+), 6 deletions(-) create mode 100644 docs/benchmark/metrics.md create mode 100644 scripts/score_psr.py create mode 100644 tests/benchmark/test_psr.py diff --git a/README.md b/README.md index 8f00e5a92..8d7da2a8d 100755 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ versioned tasks, and evaluation utilities so you can follow an experiment from | **Benchmark a model** | Use an OpenAI-compatible endpoint with the [reference runner](docs/benchmark/quickstart.md). | | **Benchmark my agent harness** | Bring your planner, memory, or framework through a [custom adapter](docs/benchmark/custom-agents.md). | | **Understand a run** | Inspect actions and observations with [trajectory visualizations](docs/benchmark/visualization.md). | +| **Understand the metrics** | Read the [SR, PSR, and CCE definitions and scoring examples](docs/benchmark/metrics.md). | | **Contribute or review results** | Read the [contributor guide](CONTRIBUTING.md) and [submission requirements](docs/benchmark/scoring.md). | ## Inside the benchmark @@ -152,7 +153,7 @@ harness through the game API. | :--- | :--- | | **Success rate · SR** | Offline, task-specific verification with coverage, errors, and per-task outcomes. Full-suite SR is reported only when coverage is complete and there are no scoring errors. | | **Causal Collaboration Effectiveness · CCE** | Research analysis using an LLM judge. Record the judge, configuration, and protocol alongside results. | -| **Partial success rate · PSR** | A validated implementation reproducing the paper's metric is not yet available in this checkout. | +| **Partial success rate · PSR** | [Checkpoint reconstruction and worked example](docs/benchmark/metrics.md#partial-success-rate-psr); not cross-validated against the historical implementation. | The checkout contains 200 augmented task files; the paper reports experiments on 100 augmented variants. Record the exact task manifest when comparing results. diff --git a/docs/benchmark/metrics.md b/docs/benchmark/metrics.md new file mode 100644 index 000000000..3b20dbd42 --- /dev/null +++ b/docs/benchmark/metrics.md @@ -0,0 +1,200 @@ +# Evaluation metrics + +This document specifies the checkpoint PSR reconstruction used for the September +2026 additional-model experiments, alongside SR, CCE, Rounds, and Chats. +The implementation is [scripts/score_psr.py](../../scripts/score_psr.py), with +offline [regression tests](../../tests/benchmark/test_psr.py). + +**Reproducibility boundary:** this is method `checkpoint_fraction_v1`. It +reconstructs the paper's verbal checkpoint definition, but has not been +cross-validated against the unavailable historical PSR program. Label its output +**reconstructed PSR**, rather than claiming it reproduces every published PSR +value. It uses fractional credit for numeric quantities; it is not the proportion +of checkpoints whose targets have been fully met. + +## Partial success rate (PSR) + +For task *i* with *Kᵢ* verifier checkpoints, assign each checkpoint a credit: + +| Checkpoint | Credit | Example | +| --- | --- | --- | +| Quantity with target *r* and observed amount *a* | `min(a / r, 1)` | `Logs: 3/5` → 0.6 | +| Boolean condition | 1 for `True`, 0 for `False` | `All alive: True` → 1 | +| Upper bound on rounds | 1 if `actual <= limit`, otherwise 0 | `Rounds: 30/25` → 0 | + +The per-task score is: + +```text +PSR_i = 1 if the task verifier reports success + (credit_i1 + ... + credit_iK) / K_i otherwise + +PSR_percent = 100 × (PSR_1 + ... + PSR_N) / N +``` + +Each checkpoint has equal weight within its task, and each task has equal weight +within a reported split. We do not pool all checkpoints across tasks: a task with +more checkpoints does not receive more weight. A successful task receives 1 even +if its diagnostic message omits checkpoints or describes intermediate states. + +For example, for an unsuccessful task with this verifier message: + +```text +Logs: 3/5, Kills: 0/3, All alive: True +``` + +the checkpoint credits are 0.6, 0, and 1. Its PSR is `(0.6 + 0 + 1) / 3`, or +**53.33%**, while its binary success score is 0. If another task succeeds, the +two-task mean PSR is **76.67%** and SR is **50%**. + +The script reads comma-separated `label: value` messages from the verifier. +Numeric targets normally appear as `actual/target`. For legacy messages that +omit a denominator, the explicit `BARE_TARGETS` table in the script supplies +targets taken from the corresponding verifier predicates. Unknown formats, +unknown bare targets, and nonpositive targets raise errors; they are not replaced +with zero. If a verifier changes its message or criteria, review this mapping +and version the scoring protocol before comparing results. + +## Task success rate (SR) + +```text +SR_percent = 100 × number of successful tasks / number of tasks +``` + +For new runs, [benchmarks/score.py](../../benchmarks/score.py) calls the checked-in +task-specific verifiers. See the [scoring guide](scoring.md) for commands. + +The historical additional-model collector in `scripts/score_psr.py` deliberately +uses [task_verifier.py](../../task_verifier.py): version 0 for Main, version 1 for +`_v1`, and version 2 for `_v2`. This preserves that experiment's verifier choice. +It does **not** silently switch historical results to the newer adjacent-task +verifier path in `benchmarks/score.py`. Record the verifier and code revision; +these paths are not certified equivalent. For augmented results, do not trust a +saved main-task success flag without verifying the variant. + +## Causal Collaboration Effectiveness (CCE) + +[compute_cce_v2.py](../../analysis/scripts/compute_cce_v2.py) first asks an LLM +judge which actions directly achieved the objective. It then traces backward by +round to identify actions that enabled contributing actions in later rounds. +The union of those actions is the contributing set *C*. + +```text +CCE_i = |C_i| / |T_i| for a successful task with recorded actions + 0 for a failed task or a task with no recorded actions + +CCE = (CCE_1 + ... + CCE_N) / N +``` + +`T` contains the nonempty action strings extracted from the trajectory. The +implementation rounds each per-task CCE to four decimal places before saving +`ce`; the experiment summary averages these saved values. The reported mean +includes failed tasks with zero CCE. It is neither a successful-task-only mean +nor a ratio formed by pooling action counts across tasks. + +CCE requires a configured judge API for successful tasks; PSR does not use an +LLM. See the [CCE commands](scoring.md#causal-collaboration-effectiveness-cce). +Keep the judge model/version, prompts, endpoint, and per-task output with the +results. The additional-model experiments used Vertex Gemini 2.5 Flash; the +paper's original table used GPT-4.1. Scores from different judges must be labeled +and are not directly interchangeable. A judge/API failure is a pending score, +not a model failure or a zero CCE. + +## Rounds and Chats + +These are arithmetic task means of the trajectory's `metrics.total_rounds` and +`metrics.total_chats`, respectively, including valid failed tasks. They are not +per-agent means. Report Main Rounds/Chats when reproducing the table layout used +for the additional-model experiments. + +## Run the PSR script + +Python 3.10+ is sufficient; this script uses only the standard library and the +repository verifier. Run commands from the repository root. + +### From checkpoint inputs + +The input JSON maps model names to task rows. Every row has `stage` (`main` or +`augmented`), the full task filename stem, boolean `success`, verifier `message`, +saved `CCE` in [0, 1], and nonnegative `rounds` and `chats`. For example: + +```json +{ + "example-model": [ + { + "stage": "main", + "task": "task_01_magic_staff", + "success": false, + "message": "Logs: 3/5, Kills: 0/3, All alive: True", + "CCE": 0.0, + "rounds": 30, + "chats": 8 + } + ] +} +``` + +This is a synthetic parser example, not task 01's actual verifier output. Save +it as `runs/checkpoint-inputs.json`, then run: + +```bash +python scripts/score_psr.py runs/checkpoint-inputs.json runs/psr-results.json +``` + +The command writes the inputs back to the input path and writes the full report +to the output path. Use separate paths and copies if you want to preserve an +earlier input snapshot. The result includes per-checkpoint credits, per-task PSR, +the method ID, `historical_implementation_reproduced: false`, and split summaries. + +### From the audited additional-model result layout + +The collector supports any model directory name and both original and copied +result trees: + +```text +RESULTS_ROOT/MODEL/main/TASK/completed.json +RESULTS_ROOT/MODEL/main/TASK/cce.json +RESULTS_ROOT/MODEL/main/TASK/ATTEMPT/task_NN_trajectory.json +RESULTS_ROOT/MODEL/augmented/TASK_v1/... (or TASK_v2) +``` + +```bash +python scripts/score_psr.py runs/checkpoint-inputs.json runs/psr-results.json \ + --collect-root . \ + --results-root /path/to/additional-model-results +``` + +`--collect-root` selects the repository's verifier and task catalog. Without +`--results-root`, collection retains the historical default +`agents/logs/additional_20260925` under that repository. The collector prefers the +sibling attempt trajectory in a copied folder over the original absolute path +stored in a completion marker. It re-verifies outcomes, checks task identity and +marker metrics, and requires agreement with saved CCE success. Cached rows are +invalidated when the completion marker, CCE record, verifier, or trajectory changes. + +This collector requires `completed.json` and `cce.json`, which were produced by +the additional-model experiment harness. It does not discover arbitrary raw +runner outputs or generate CCE scores. A missing CCE file leaves that task out +of the collected rows, so its PSR/SR summaries are also partial until scoring +finishes. The collector is not a replacement for initialization and trace audits. + +### Coverage and reporting + +- Main has 100 tasks; Augmented v1 and v2 have 100 each; their union has 200. +- The script checks task identities against the repository YAML catalog and + rejects duplicate rows, unexpected tasks, and invalid numeric metrics. +- Every summary includes `scored`, `expected`, `missing_tasks`, and `final`. + When `final` is false, numeric means describe only the scored subset. Do not + present them as complete benchmark scores or impute missing tasks as zero. +- Audit complete team initialization, initial observations, inference failures, + and one selected valid attempt per task before collection. Preserve interrupted + attempts; infrastructure failures remain pending rather than model failures. +- Score trials separately and state the trial aggregation policy. Do not select + the best attempt or pool repeated trials as if they were distinct tasks. +- Report Main, v1, and v2 separately. The additional-model v1 split is not + established as equivalent to the original paper's historical augmented subset. + +Run the offline regression tests with: + +```bash +python -m unittest discover -s tests/benchmark -p 'test_psr.py' -v +``` diff --git a/docs/benchmark/scoring.md b/docs/benchmark/scoring.md index 858a2da59..db01f4664 100644 --- a/docs/benchmark/scoring.md +++ b/docs/benchmark/scoring.md @@ -40,11 +40,18 @@ than assuming their saved success flags remain comparable. ## Partial success rate (PSR) -A validated, portable implementation reproducing the paper's PSR is not present -in this upstream checkout. The separately maintained local experiment copy has -a reconstruction, but it has not been cross-validated against the historical -implementation. Do not label a replacement calculation as the published metric. -Report PSR as unavailable until its implementation and protocol are established. +The [metrics guide](metrics.md#partial-success-rate-psr) specifies the checkpoint +reconstruction used for the additional-model experiments, with a worked example +and commands for [scripts/score_psr.py](../../scripts/score_psr.py). Quantities +receive fractional credit capped at 1; booleans and round limits receive 0/1; +checkpoints are averaged within each task and tasks are averaged equally. +Successful tasks receive 1. + +Report this as **reconstructed PSR (`checkpoint_fraction_v1`)**. It has not been +cross-validated against the unavailable historical implementation and does not +certify reproduction of the paper's PSR. The additional-model collector retains +its historical `task_verifier.py` path; it does not silently replace the newer +task-specific verifier protocol used by `benchmarks/score.py` above. ## Causal Collaboration Effectiveness (CCE) diff --git a/scripts/score_psr.py b/scripts/score_psr.py new file mode 100644 index 000000000..f048f50fe --- /dev/null +++ b/scripts/score_psr.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Reconstruct checkpoint PSR from agentworld_paper/colm/main.tex:286. + +Numeric quantity checkpoints receive min(actual / required, 1); booleans +receive 0/1. Round limits are boolean upper-bound checks, not quantities to +accumulate. Checkpoints have equal weight within a task, tasks equal weight +within a set, and successful tasks receive 1. This explicit reconstruction +has not been cross-validated against the unavailable historical PSR program. +""" +import argparse +import datetime +import hashlib +import json +import math +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parents[1] +METHOD = 'checkpoint_fraction_v1' +DEFINITION = ('Mean checkpoint completion per task, then mean across tasks; ' + 'quantities clipped actual/required, booleans 0/1, round-limit ' + 'checks actual<=limit, successful tasks forced to 1.') + +# Denominators omitted from messages, read from task_verifier.py predicates. +BARE_TARGETS = { + ('07', 'Pickaxe count'): 1, + ('31', 'Sticks'): 10, ('31', 'Logs'): 5, + ('52', 'Weapons crafted'): 2, + ('57', 'Gems'): 5, ('57', 'Jewelry'): 2, + ('60', 'Ores'): 15, ('60', 'Bars'): 5, + ('65', 'Potions/consumables'): 5, +} + + +def checkpoint_score(task, message, success): + if success: + return {'PSR': 1.0, 'successful_task_override': True, 'checkpoints': []} + number = re.match(r'task_(\d+)_', task).group(1) + checkpoints = [] + for part in message.split(','): + label, value = (s.strip() for s in part.split(':', 1)) + if value in ('True', 'False'): + credit = float(value == 'True') + details = {'kind': 'boolean', 'actual': value == 'True'} + else: + match = re.fullmatch(r'(\d+(?:\.\d+)?)(?:/(\d+(?:\.\d+)?))?', value) + if not match: + raise ValueError(f'Unrecognized checkpoint: {task}: {part}') + actual = float(match.group(1)) + target = float(match.group(2)) if match.group(2) else BARE_TARGETS[(number, label)] + if target <= 0: + raise ValueError(f'Invalid target: {task}: {part}') + if label == 'Rounds': + credit = float(actual <= target) + kind = 'upper_bound' + else: + credit = min(actual / target, 1.0) + kind = 'quantity' + details = {'kind': kind, 'actual': actual, 'target': target} + checkpoints.append({'label': label, **details, 'credit': credit}) + if not checkpoints: + raise ValueError(f'No checkpoints: {task}') + return {'PSR': sum(c['credit'] for c in checkpoints) / len(checkpoints), + 'successful_task_override': False, 'checkpoints': checkpoints} + + +def expected_task_sets(root=ROOT): + """Use task identities, not just counts, to establish completeness.""" + return {stage: {p.stem for p in (root / 'data_v0.1_multi' / dataset).glob('*.yaml')} + for stage, dataset in [('main', 'v1.3_benchmark'), + ('augmented', 'v1.3_augmented')]} + + +def calculate(inputs, expected_tasks=None): + expected_tasks = expected_task_sets() if expected_tasks is None else expected_tasks + if not all(expected_tasks.get(stage) for stage in ('main', 'augmented')): + raise ValueError('Expected benchmark YAML task sets are missing') + results = {} + for model, rows in inputs.items(): + seen = set() + for row in rows: + identity = (row['stage'], row['task']) + if identity in seen: + raise ValueError(f'Duplicate task: {model}/{identity}') + seen.add(identity) + if row['task'] not in expected_tasks.get(row['stage'], set()): + raise ValueError(f'Unexpected task: {model}/{identity}') + if type(row['success']) is not bool: + raise ValueError(f'Success must be boolean: {model}/{identity}') + for key in ('CCE', 'rounds', 'chats'): + value = row[key] + if type(value) not in (int, float) or not math.isfinite(value) or value < 0: + raise ValueError(f'Invalid {key}: {model}/{identity}') + if row['CCE'] > 1 or (not row['success'] and row['CCE'] != 0): + raise ValueError(f'Invalid CCE for task outcome: {model}/{identity}') + scored = [{**row, **checkpoint_score(row['task'], row['message'], row['success'])} + for row in rows] + summaries = {} + for subset in ['main', 'augmented_v1', 'augmented_v2', 'augmented']: + selected = [r for r in scored if + (r['stage'] == 'main' if subset == 'main' else + r['stage'] == 'augmented' and + (subset == 'augmented' or r['task'].endswith('_' + subset[-2:]))) ] + n = len(selected) + expected_ids = expected_tasks['main' if subset == 'main' else 'augmented'] + if subset in ('augmented_v1', 'augmented_v2'): + expected_ids = {t for t in expected_ids if t.endswith('_' + subset[-2:])} + expected = len(expected_ids) + summaries[subset] = { + 'scored': n, 'expected': expected, + 'final': {r['task'] for r in selected} == expected_ids, + 'missing_tasks': sorted(expected_ids - {r['task'] for r in selected}), + **({key: sum(row[source] for row in selected) / n * scale + for key, source, scale in [('SR_percent', 'success', 100), + ('PSR_percent', 'PSR', 100), ('CCE', 'CCE', 1), + ('rounds', 'rounds', 1), ('chats', 'chats', 1)]} if n else {})} + results[model] = {'summary': summaries, 'tasks': scored} + return {'time_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'method': METHOD, 'definition': DEFINITION, + 'source': 'agentworld_paper/colm/main.tex:286', + 'historical_implementation_reproduced': False, 'models': results} + + +def collect(root, previous, results_root=None): + """Collect completed/CCE-scored attempts; initialization must be audited first.""" + sys.path.insert(0, str(root)) + from task_verifier import verify_task + verifier_hash = hashlib.sha256((root / 'task_verifier.py').read_bytes()).hexdigest() + inputs = {} + results_root = results_root or root / 'agents/logs/additional_20260925' + folders = sorted(p for p in results_root.iterdir() + if p.is_dir() and any((p / s).is_dir() for s in ('main', 'augmented'))) + if not folders: + raise ValueError(f'No model result folders found in {results_root}') + for model_folder in folders: + model = model_folder.name + cached = {(row['stage'], row['task']): row for row in previous.get(model, [])} + rows = [] + for stage in ['main', 'augmented']: + folder = model_folder / stage + for marker in sorted(folder.glob('*/completed.json')): + cce_file = marker.parent / 'cce.json' + if not cce_file.exists(): + continue + completed_text, cce_text = marker.read_text(), cce_file.read_text() + completed, cce = json.loads(completed_text), json.loads(cce_text) + saved_path = Path(completed['trajectory']) + # Prefer the sibling attempt after moving/copying result folders. + local_path = marker.parent / saved_path.parent.name / saved_path.name + trajectory_path = local_path if local_path.is_file() else saved_path + trajectory_text = trajectory_path.read_text() + trajectory = json.loads(trajectory_text) + if (completed['task'] != marker.parent.name or + Path(trajectory['task_source']).stem != completed['task'] or + Path(cce['task_source']).stem != completed['task']): + raise ValueError(f'Task identity mismatch: {marker}') + if completed['metrics'] != trajectory['metrics']: + raise ValueError(f'Marker/trajectory metrics mismatch: {marker}') + if (trajectory_path.parent / 'infra_error.txt').exists(): + raise ValueError(f'Infrastructure failure marked complete: {marker}') + fingerprint = hashlib.sha256((completed_text + cce_text + verifier_hash + + trajectory_text).encode()).hexdigest() + old = cached.get((stage, completed['task'])) + if old and old.get('_fingerprint') == fingerprint: + rows.append(old) + continue + version = 1 if completed['task'].endswith('_v1') else 2 if completed['task'].endswith('_v2') else 0 + passed, message = verify_task(trajectory, version=version) + if bool(passed) != cce['success']: + raise ValueError(f"Verifier/CCE success mismatch: {model}/{completed['task']}") + rows.append({'stage': stage, 'task': completed['task'], + 'success': bool(passed), 'message': message, + 'CCE': cce['ce'], 'rounds': completed['metrics']['total_rounds'], + 'chats': completed['metrics']['total_chats'], + '_fingerprint': fingerprint}) + inputs[model] = rows + return inputs + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('inputs', type=Path) + parser.add_argument('output', type=Path) + parser.add_argument('--collect-root', type=Path, + help='Repository containing task_verifier.py and benchmark YAMLs') + parser.add_argument('--results-root', type=Path, + help='Collect MODEL/{main,augmented}/TASK/completed.json from this directory') + args = parser.parse_args() + if args.inputs.resolve() == args.output.resolve(): + parser.error('Input and output JSON must use different paths') + if not args.inputs.exists() and not (args.collect_root or args.results_root): + parser.error('Input JSON does not exist; provide --results-root to collect it') + inputs = json.loads(args.inputs.read_text()) if args.inputs.exists() else {} + root = (args.collect_root or ROOT).resolve() + if args.collect_root or args.results_root: + inputs = collect(root, inputs, args.results_root) + results = calculate(inputs, expected_task_sets(root)) + for path, value in [(args.inputs, inputs), (args.output, results)]: + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix('.tmp') + temp.write_text(json.dumps(value, indent=2) + '\n') + temp.replace(path) + print(json.dumps({m: r['summary'] for m, r in results['models'].items()}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/tests/benchmark/test_psr.py b/tests/benchmark/test_psr.py new file mode 100644 index 000000000..df5670436 --- /dev/null +++ b/tests/benchmark/test_psr.py @@ -0,0 +1,118 @@ +"""Offline checks for checkpoint PSR, coverage, and relocated experiment traces.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +from scripts.score_psr import calculate, checkpoint_score, collect, expected_task_sets + + +class PSRTests(unittest.TestCase): + def test_fractional_example_and_success_override(self): + message = 'Logs: 3/5, Kills: 0/3, All alive: True' + result = checkpoint_score('task_01_magic_staff', message, False) + self.assertAlmostEqual(result['PSR'], 8 / 15) + self.assertEqual([c['credit'] for c in result['checkpoints']], [0.6, 0, 1]) + self.assertEqual(checkpoint_score('task_01_magic_staff', '', True)['PSR'], 1) + + def test_caps_booleans_round_limits_and_legacy_targets(self): + for message, expected in [('Logs: 8/5', 1), ('All alive: False', 0), + ('Rounds: 25/25', 1), ('Rounds: 26/25', 0), + ('Rounds: 10/25', 1), ('Pickaxe count: 0', 0), + ('Pickaxe count: 1', 1)]: + with self.subTest(message=message): + self.assertEqual(checkpoint_score('task_07_pickaxe_crafting', message, False)['PSR'], expected) + + def test_unknown_checkpoints_are_not_zero(self): + for message in ['Logs: unknown', 'Logs: 1/0', 'Unknown: 3', '']: + with self.subTest(message=message), self.assertRaises((ValueError, KeyError)): + checkpoint_score('task_01_magic_staff', message, False) + + def row(self, task, message, success=False, stage='main', **kwargs): + return dict(stage=stage, task=task, message=message, success=success, + CCE=kwargs.get('CCE', 0), rounds=10, chats=2) + + def test_tasks_weighted_equally_and_partial_coverage(self): + tasks = sorted(expected_task_sets()['main']) + rows = [self.row(tasks[0], 'Logs: 3/5, Kills: 0/3, All alive: True'), + self.row(tasks[1], '', success=True)] + summary = calculate({'example': rows})['models']['example']['summary']['main'] + self.assertAlmostEqual(summary['PSR_percent'], 100 * 23 / 30) + self.assertEqual(summary['SR_percent'], 50) + self.assertFalse(summary['final']) + self.assertEqual(len(summary['missing_tasks']), 98) + + def test_complete_split_coverage(self): + rows = [self.row(task, '', True, stage) for stage, tasks in expected_task_sets().items() + for task in sorted(tasks)] + summary = calculate({'example': rows})['models']['example']['summary'] + for split, expected in [('main', 100), ('augmented_v1', 100), + ('augmented_v2', 100), ('augmented', 200)]: + self.assertTrue(summary[split]['final']) + self.assertEqual(summary[split]['scored'], expected) + self.assertEqual(summary[split]['PSR_percent'], 100) + + def test_duplicate_unknown_and_invalid_values_rejected(self): + task = sorted(expected_task_sets()['main'])[0] + row = self.row(task, 'Logs: 0/5') + with self.assertRaisesRegex(ValueError, 'Duplicate task'): + calculate({'example': [row, row]}) + with self.assertRaisesRegex(ValueError, 'Unexpected task'): + calculate({'example': [dict(row, task='task_999_unknown')]}) + for changes in [{'success': 'false'}, {'CCE': float('nan')}, {'CCE': 1.1}, + {'CCE': 0.5}, {'rounds': -1}, {'chats': '2'}]: + with self.subTest(changes=changes), self.assertRaises(ValueError): + calculate({'example': [dict(row, **changes)]}) + + def test_relocated_collection_and_cache_invalidation(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / 'task_verifier.py').write_text('# hash fixture') + task = 'task_01_magic_staff_v2' + folder = root / 'results/custom-model/augmented' / task + attempt = folder / 'attempt_1' + attempt.mkdir(parents=True) + trajectory = {'task_source': task + '.yaml', 'rounds': [], + 'metrics': {'total_rounds': 10, 'total_chats': 2}} + trajectory_file = attempt / 'task_01_trajectory.json' + trajectory_file.write_text(json.dumps(trajectory)) + (folder / 'completed.json').write_text(json.dumps({'task': task, + 'trajectory': '/old-host/unavailable/attempt_1/task_01_trajectory.json', + 'metrics': trajectory['metrics']})) + (folder / 'cce.json').write_text(json.dumps({'task_source': task + '.yaml', + 'success': False, 'ce': 0})) + with patch('task_verifier.verify_task', return_value=(False, 'Logs: 0/5')) as verify: + first = collect(root, {}, root / 'results') + verify.assert_called_once_with(trajectory, version=2) + verify.reset_mock() + self.assertEqual(collect(root, first, root / 'results'), first) + verify.assert_not_called() + trajectory['rounds'] = [{'round': 1}] + trajectory_file.write_text(json.dumps(trajectory)) + collect(root, first, root / 'results') + verify.assert_called_once() + (attempt / 'infra_error.txt').write_text('inference failed') + with self.assertRaisesRegex(ValueError, 'Infrastructure failure'): + collect(root, first, root / 'results') + + def test_documented_cli_example(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / 'input.json' + output = Path(tmp) / 'nested/output.json' + source.write_text(json.dumps({'example': [self.row('task_01_magic_staff', + 'Logs: 3/5, Kills: 0/3, All alive: True')]})) + proc = subprocess.run([sys.executable, str(ROOT / 'scripts/score_psr.py'), + str(source), str(output)], capture_output=True, text=True) + self.assertEqual(proc.returncode, 0, proc.stderr) + result = json.loads(output.read_text()) + self.assertFalse(result['historical_implementation_reproduced']) + self.assertAlmostEqual(result['models']['example']['tasks'][0]['PSR'], 8 / 15) + + +if __name__ == '__main__': + unittest.main()