From 177f196ee4900ba34bd7bdd3dd85ded1f9160ce1 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 26 Oct 2026 09:13:48 -0400 Subject: [PATCH] fix(harbor): respect trial reward validity --- .../braintrust/integrations/harbor/plugin.py | 13 +++++- .../integrations/harbor/test_harbor.py | 46 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/py/src/braintrust/integrations/harbor/plugin.py b/py/src/braintrust/integrations/harbor/plugin.py index afca942a4..aebff6052 100644 --- a/py/src/braintrust/integrations/harbor/plugin.py +++ b/py/src/braintrust/integrations/harbor/plugin.py @@ -734,6 +734,12 @@ def _root_metadata(self, result: Any, plan: TrialPlan, machine: TrialMachine) -> "warnings": list(machine.warnings), } } + # Harbor 0.25+ computes these from the trial and step error policies. + # Older releases must retain their existing reward conversion behavior. + for key in ("reward_valid", "failure_cause"): + value = getattr(result, key, _UNSET) + if value is not _UNSET: + metadata["harbor"][key] = value if error and traceback_value: metadata["harbor"]["exception_traceback"] = traceback_value return metadata @@ -780,7 +786,12 @@ def _sync_final_result(self, result: Any) -> None: error, _ = _exception(result, self.config.include_tracebacks) metadata = self._root_metadata(result, plan, machine) rewards = _rewards(result) - conversion = classify_rewards(rewards, self.config) + reward_valid = metadata["harbor"].get("reward_valid") + conversion = classify_rewards(rewards if reward_valid is not False else {}, self.config) + if rewards and reward_valid is False: + metadata["harbor"]["warnings"].append( + "rewards omitted because Harbor marked them invalid (reward_valid=False)" + ) metadata["harbor"]["warnings"].extend(conversion.warnings) if not rewards and error is None: metadata["harbor"]["warnings"].append("trial has no reward and is unevaluated") diff --git a/py/src/braintrust/integrations/harbor/test_harbor.py b/py/src/braintrust/integrations/harbor/test_harbor.py index c34e73424..8bf6f979f 100644 --- a/py/src/braintrust/integrations/harbor/test_harbor.py +++ b/py/src/braintrust/integrations/harbor/test_harbor.py @@ -71,7 +71,7 @@ from harbor.models.task.id import LocalTaskId from harbor.models.trajectories.trajectory import Trajectory from harbor.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, TrialConfig -from harbor.models.trial.result import AgentInfo, StepResult, TimingInfo, TrialResult, VerifierResult +from harbor.models.trial.result import AgentInfo, ExceptionInfo, StepResult, TimingInfo, TrialResult, VerifierResult _ABSOLUTE_PATH_RE = re.compile(r"(?:/(?:Users|private|home)/[^\"\\\\\s]+|[A-Za-z]:\\\\[^\"\\\\\s]+)") @@ -774,6 +774,44 @@ def _scored_verifier_trial(tmp_path): return result +@pytest.mark.parametrize("exception_type", ["DownloadVerifierDirError", "AgentTimeoutError"]) +def test_final_sync_respects_reward_validity(tmp_path, exception_type, with_memory_logger, with_simulate_login): + result = _trial_result(tmp_path, "trial-1", "task-a") + result.verifier_result = VerifierResult(rewards={"reward": 0.25, "quality": 0.7}) + result.exception_info = ExceptionInfo( + exception_type=exception_type, + exception_message="trial failed", + exception_traceback="traceback", + occurred_at=datetime.now(timezone.utc), + ) + experiment = init_test_exp("harbor-reward-validity", "harbor") + plugin = HarborPlugin(attachments="none", trajectory_mode="native") + _install_runtime(plugin, result, tmp_path, experiment) + + spans = _sync_spans(plugin, result, experiment, with_memory_logger) + + root = find_span_by_name(spans, "eval") + metadata = root["metadata"]["harbor"] + assert metadata["raw_rewards"] == {"reward": 0.25, "quality": 0.7} + has_reward_validity = hasattr(TrialResult, "reward_valid") + invalid = has_reward_validity and exception_type == "DownloadVerifierDirError" + scorers = find_spans_by_type(spans, "score") + if invalid: + assert scorers == [] + assert "quality" not in root["metrics"] + assert "rewards omitted because Harbor marked them invalid (reward_valid=False)" in metadata["warnings"] + else: + assert scorers[0]["scores"] == {"reward": 0.25} + assert root["metrics"]["quality"] == 0.7 + assert metadata["warnings"] == [] + if has_reward_validity: + assert metadata["reward_valid"] == (not invalid) + assert metadata["failure_cause"] == ("sandbox_failed" if invalid else "agent_exhausted_budget") + else: + assert "reward_valid" not in metadata + assert "failure_cause" not in metadata + + @pytest.mark.parametrize( ("attachments", "expected_summary"), [ @@ -837,6 +875,12 @@ def test_verification_span_keeps_verifier_evidence_without_scores(tmp_path, with spans = _sync_spans(plugin, result, experiment, with_memory_logger) assert not find_spans_by_type(spans, "score") + metadata = find_span_by_name(spans, "eval")["metadata"]["harbor"] + assert "trial has no reward and is unevaluated" in metadata["warnings"] + assert not any("reward_valid=False" in warning for warning in metadata["warnings"]) + if hasattr(TrialResult, "reward_valid"): + assert metadata["reward_valid"] is False + assert metadata["failure_cause"] is None verification = find_span_by_name(spans, "verification") assert verification["output"]["verifier_output"]["type"] == "braintrust_attachment"