diff --git a/.github/workflows/bcal-evaluation.yml b/.github/workflows/bcal-evaluation.yml index dcdb95865..f4c103ce3 100644 --- a/.github/workflows/bcal-evaluation.yml +++ b/.github/workflows/bcal-evaluation.yml @@ -30,6 +30,7 @@ concurrency: env: EVALUATION_RESULTS_DIR: evaluation_results + BCCONTAINERHELPER_VERSION: 6.1.18 jobs: get-entries: @@ -38,9 +39,44 @@ jobs: test-run: ${{ inputs.test-run || false }} category: nl2al - evaluate-with-bcal: + prepare-bccontainerhelper: runs-on: windows-latest needs: get-entries + if: needs.get-entries.outputs.entries != '[]' + steps: + - name: Cache pinned BcContainerHelper + id: cache-bccontainerhelper + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~\Documents\PowerShell\Modules\BcContainerHelper + key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }} + + - name: Download pinned BcContainerHelper + if: steps.cache-bccontainerhelper.outputs.cache-hit != 'true' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $moduleRoot = Join-Path $HOME 'Documents\PowerShell\Modules' + New-Item -ItemType Directory -Path $moduleRoot -Force | Out-Null + Save-Module ` + -Name BcContainerHelper ` + -RequiredVersion $env:BCCONTAINERHELPER_VERSION ` + -Repository PSGallery ` + -Path $moduleRoot ` + -Force ` + -ErrorAction Stop + + - name: Verify pinned BcContainerHelper + shell: pwsh + run: | + Import-Module BcContainerHelper -RequiredVersion $env:BCCONTAINERHELPER_VERSION -Force -DisableNameChecking + if ((Get-Module BcContainerHelper).Version -ne [Version]$env:BCCONTAINERHELPER_VERSION) { + throw "Expected BcContainerHelper $env:BCCONTAINERHELPER_VERSION." + } + + evaluate-with-bcal: + runs-on: windows-latest + needs: [get-entries, prepare-bccontainerhelper] outputs: results-dir: ${{ env.EVALUATION_RESULTS_DIR }} if: needs.get-entries.outputs.entries != '[]' @@ -61,6 +97,17 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Python with UV + id: setup-python + continue-on-error: true + uses: $/.github/actions/setup-python-uv + + - name: Wait before retrying Python setup + if: steps.setup-python.outcome == 'failure' + shell: pwsh + run: Start-Sleep -Seconds 20 + + - name: Retry Setup Python with UV + if: steps.setup-python.outcome == 'failure' uses: $/.github/actions/setup-python-uv - name: Azure Login (OIDC) for ADO feed @@ -135,6 +182,13 @@ jobs: id: bcversion run: uv run bcbench dataset version "${{ matrix.entry }}" --category nl2al --github-output version + - name: Restore pinned BcContainerHelper + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~\Documents\PowerShell\Modules\BcContainerHelper + key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }} + fail-on-cache-miss: true + # Cache BcContainerHelper's artifact folder (its hardcoded default path) keyed by this entry's BC version - name: Cache BC sandbox artifacts id: bccache @@ -147,7 +201,6 @@ jobs: if: steps.bccache.outputs.cache-hit != 'true' shell: pwsh run: | - Install-Module -Name BcContainerHelper -Force -AllowClobber -AllowPrerelease .\scripts\Download-BCSymbols.ps1 -Category nl2al -InstanceId "${{ matrix.entry }}" - name: Azure Login (OIDC) for LLM API certificate Key Vault @@ -192,21 +245,26 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: - name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} + name: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.entry }} path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: - needs: evaluate-with-bcal + needs: [get-entries, evaluate-with-bcal] + if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.get-entries.outputs.entry-count != '0' }} uses: $/.github/workflows/summarize-results.yml permissions: contents: write id-token: write with: - results-dir: ${{ needs.evaluate-with-bcal.outputs.results-dir }} + results-dir: evaluation_results model: ${{ inputs.external-model || 'gpt-55-chat' }} agent: BCal-external-command mock: ${{ inputs.test-run || false }} category: nl2al skip-leaderboard: true + expected-total: ${{ fromJSON(needs.get-entries.outputs.entry-count) }} + expected-entries: ${{ needs.get-entries.outputs.entries }} + artifact-pattern: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-* + allow-unscored-results: true secrets: inherit diff --git a/.github/workflows/get-entries.yml b/.github/workflows/get-entries.yml index b49e08d86..7f73d6907 100644 --- a/.github/workflows/get-entries.yml +++ b/.github/workflows/get-entries.yml @@ -24,6 +24,9 @@ on: entries: description: JSON array of dataset entries value: ${{ jobs.get-entries.outputs.entries }} + entry-count: + description: Number of dataset entries + value: ${{ jobs.get-entries.outputs.entry-count }} runner: description: GitHub Actions runner label to evaluate this category on value: ${{ jobs.get-entries.outputs.runner }} @@ -39,6 +42,7 @@ jobs: runs-on: ubuntu-latest outputs: entries: ${{ steps.get-entries.outputs.entries }} + entry-count: ${{ steps.get-entries.outputs.entry-count }} runner: ${{ steps.runtime-config.outputs.runner }} requires-container: ${{ steps.runtime-config.outputs.requires-container }} requires-repo: ${{ steps.runtime-config.outputs.requires-repo }} @@ -58,7 +62,7 @@ jobs: - name: Get entries for matrix id: get-entries run: | - cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries" + cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries --count-output entry-count" if [[ "${{ inputs.modified-only }}" == "true" ]]; then cmd="$cmd --modified-only" diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml index 4ecd1f25a..35ec2f781 100644 --- a/.github/workflows/summarize-results.yml +++ b/.github/workflows/summarize-results.yml @@ -34,10 +34,31 @@ on: required: false type: string default: "" + expected-total: + description: "Expected unique matrix entry count; negative disables completeness enforcement" + required: false + type: number + default: -1 + expected-entries: + description: "JSON array of expected matrix entry IDs; missing IDs become infrastructure failure results" + required: false + type: string + default: "[]" + artifact-pattern: + description: "Artifact glob to download" + required: false + type: string + default: "*" + allow-unscored-results: + description: "Allow complete result sets containing explicit unscored infrastructure errors" + required: false + type: boolean + default: false env: BCEVAL_RESULT_FILE: bceval_results.jsonl SUMMARY_OUTPUT_FILE: evaluation_summary.json + COMPLETENESS_OUTPUT_FILE: evaluation_completeness.json jobs: summarize-results: @@ -58,22 +79,32 @@ jobs: - name: Download all evaluation results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: ${{ inputs.expected-total >= 0 }} with: path: ${{ inputs.results-dir }}/ + pattern: ${{ inputs.artifact-pattern }} merge-multiple: true - name: Summarize evaluation results - run: uv run bcbench result summarize --category "${{ inputs.category }}" --result-dir "${{ inputs.results-dir }}" --bceval-output "${{ env.BCEVAL_RESULT_FILE }}" --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}" --git-ref "${{ inputs.git-ref || github.ref_name }}" - - - name: Upload evaluation summary to artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: evaluation-summary - path: ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }} - if-no-files-found: error - retention-days: ${{ inputs.mock && 1 || 30 }} + id: summarize + env: + EXPECTED_ENTRIES_JSON: ${{ inputs.expected-entries }} + run: >- + uv run bcbench result summarize + --category "${{ inputs.category }}" + --result-dir "${{ inputs.results-dir }}" + --bceval-output "${{ env.BCEVAL_RESULT_FILE }}" + --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}" + --completeness-output "${{ env.COMPLETENESS_OUTPUT_FILE }}" + --expected-total "${{ inputs.expected-total }}" + --expected-entries-json "${EXPECTED_ENTRIES_JSON}" + --model "${{ inputs.model }}" + --agent "${{ inputs.agent }}" + --git-ref "${{ inputs.git-ref || github.ref_name }}" + --github-output - name: Azure Login with OIDC for bceval package + if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true' uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} @@ -81,10 +112,12 @@ jobs: allow-no-subscriptions: true - name: Resolve bceval evaluator config for category + if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true' id: bceval run: uv run bcbench category bceval-config --category "${{ inputs.category }}" - - name: Upload result using bceval + - name: Score and upload complete result using bceval + if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true' env: BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} @@ -96,11 +129,14 @@ jobs: uv tool install bc-eval==0.6.1 --python 3.12 --index "https://anything:${ADO_TOKEN}@dynamicssmb2.pkgs.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_packaging/BC_PythonFeed/pypi/simple/" - # Upload summary using bc-eval MODEL_TAG="${{ inputs.model }}" MODEL_TAG="${MODEL_TAG//./-}" BRANCH_TAG="${{ inputs.git-ref || github.ref_name }}" BRANCH_TAG="${BRANCH_TAG//./-}" + CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}") + if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]; then + CORE_SCORE_ARGS=() + fi bceval metrics calculate \ --feature-name "BC-Bench" \ --eval-suite-name "${{ inputs.category }}" \ @@ -110,13 +146,25 @@ jobs: --input-file "${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }}" \ --evaluator-definitions "${{ github.workspace }}/evaluator/scores.py" \ --evaluators "${{ steps.bceval.outputs.evaluators }}" \ - --core-score "${{ steps.bceval.outputs.core_score }}" \ + "${CORE_SCORE_ARGS[@]}" \ --metric-definitions "${{ github.workspace }}/evaluator/metrics.py" \ --metrics "bc_bench_metrics" \ ${{ !inputs.mock && '--storage braintrust --storage kusto' || '' }} + - name: Upload evaluation summary to artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evaluation-summary-${{ github.run_attempt }} + path: | + ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }} + ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }} + ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }} + if-no-files-found: error + retention-days: ${{ inputs.mock && 1 || 30 }} + - name: Update leaderboard in a new branch - if: ${{ !inputs.mock && !inputs.skip-leaderboard }} + if: ${{ !inputs.mock && !inputs.skip-leaderboard && steps.summarize.outputs.complete == 'true' }} run: | git fetch origin main @@ -150,3 +198,13 @@ jobs: git commit -m "Update leaderboard: ${{ inputs.agent }} (${{ inputs.model }}) - Run ${{ github.run_id }}" git push --set-upstream origin $branchName + + - name: Require complete matrix + if: ${{ always() && inputs.expected-total >= 0 }} + run: | + completeness_file="${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }}" + if [[ ! -f "$completeness_file" ]]; then + echo "::error::Completeness metadata was not produced." + exit 1 + fi + uv run bcbench result require-complete "$completeness_file" diff --git a/evaluator/scores.py b/evaluator/scores.py index 569b54800..10839a647 100644 --- a/evaluator/scores.py +++ b/evaluator/scores.py @@ -1,7 +1,92 @@ from __future__ import annotations +import importlib from typing import Any +_TIMEOUT_REASON = "Agent timed out before producing output" +_TEST_PASSED_SCORE = "test_passed" +_MISSING_TEST_PASSED_SCORE = "missing-test-passed-score" + + +def _timeout_scores(scorer: object, expected: object, metadata: dict[str, Any]) -> list[object]: + if not isinstance(expected, dict): + raise TypeError("expected must be a dict containing an 'assertions' key") + assertions = expected.get("assertions") + if not isinstance(assertions, list): + raise TypeError("expected['assertions'] must be a list") + + judged = [] + for index, assertion in enumerate(assertions): + if not isinstance(assertion, dict): + raise TypeError(f"expected['assertions'][{index}] must be a dict") + if "text" not in assertion: + raise ValueError(f"expected['assertions'][{index}] must contain a 'text' key") + judged.append({**assertion, "pass": False, "reasoning": _TIMEOUT_REASON}) + + score_metadata = {"assertions": judged, "error": _TIMEOUT_REASON} + metadata["assertionResults"] = judged + score = importlib.import_module("autoevals").Score + rate = getattr(scorer, "_rate", None) + if not callable(rate): + raise TypeError("LM Checklist scorer does not provide a callable '_rate'") + return [ + score(name="pass_rate", score=rate(judged), metadata=score_metadata), + score(name="critical_pass_rate", score=rate(judged, "critical"), metadata=score_metadata), + score(name="expected_pass_rate", score=rate(judged, "expected"), metadata=score_metadata), + score(name="aspirational_pass_rate", score=rate(judged, "aspirational"), metadata=score_metadata), + score(name="test_passed", score=0.0, metadata=score_metadata), + ] + + +def _infrastructure_error_scores(metadata: dict[str, Any]) -> list[object]: + error = str(metadata.get("Error") or metadata.get("error_message") or "Infrastructure error") + score_metadata = {"error": error} + score = importlib.import_module("autoevals").Score + return [ + score(name="pass_rate", score=None, metadata=score_metadata), + score(name="critical_pass_rate", score=None, metadata=score_metadata), + score(name="expected_pass_rate", score=None, metadata=score_metadata), + score(name="aspirational_pass_rate", score=None, metadata=score_metadata), + score(name="test_passed", score=None, metadata=score_metadata), + ] + + +def _set_test_passed_core_score(result: object, metadata: dict[str, Any]) -> object: + if not isinstance(result, list): + raise TypeError("LM Checklist scorer must return a list of scores") + test_passed_scores = [score for score in result if getattr(score, "name", None) == _TEST_PASSED_SCORE] + if len(test_passed_scores) != 1 or getattr(test_passed_scores[0], "score", None) is None: + raise ValueError("Evaluated rows must produce exactly one non-null 'test_passed' score") + try: + metadata["core_score"] = float(test_passed_scores[0].score) + except (TypeError, ValueError) as exc: + raise ValueError("The 'test_passed' score must be numeric") from exc + return result + + +class LmChecklist: + def __new__(cls) -> object: + module = importlib.import_module("bc_eval.scorers.autoeval.lmchecklist") + scorer = module.LmChecklist() + run_eval_sync = scorer._run_eval_sync + + def failure_aware_run_eval_sync(output: object, expected: object = None, **kwargs: object) -> object: + metadata = kwargs.get("metadata") + if isinstance(metadata, dict): + metadata["core_score_name"] = _TEST_PASSED_SCORE + if metadata.get("infrastructure_error") is True: + metadata.pop("core_score", None) + return _infrastructure_error_scores(metadata) + if metadata.get("timeout") is True: + metadata["core_score"] = 0.0 + return _timeout_scores(scorer, expected, metadata) + metadata["core_score"] = _MISSING_TEST_PASSED_SCORE + return _set_test_passed_core_score(run_eval_sync(output, expected, **kwargs), metadata) + return run_eval_sync(output, expected, **kwargs) + + scorer._run_eval_sync = failure_aware_run_eval_sync + return scorer + class ResolutionRate: def __call__(self, *, metadata: dict[str, Any], **kwargs: object) -> bool: diff --git a/src/bcbench/commands/dataset.py b/src/bcbench/commands/dataset.py index 3f3092749..a3e904d9f 100644 --- a/src/bcbench/commands/dataset.py +++ b/src/bcbench/commands/dataset.py @@ -21,6 +21,7 @@ def list_entries( category: EvaluationCategoryOption = EvaluationCategory.BUG_FIX, github_output: Annotated[str | None, typer.Option(help="Write JSON output to GITHUB_OUTPUT with this key name")] = None, + count_output: Annotated[str | None, typer.Option(help="Write the entry count to GITHUB_OUTPUT with this key name")] = None, modified_only: Annotated[bool, typer.Option(help="Only list entries that have been modified in git diff")] = False, test_run: Annotated[bool, typer.Option(help="Indicate this is a test run (with 2 entries)")] = False, ) -> None: @@ -58,8 +59,12 @@ def list_entries( for entry_id in entry_ids: print(f" - {entry_id}") - if github_output: - write_step_outputs({github_output: json.dumps(entry_ids)}) + step_outputs = { + **({github_output: json.dumps(entry_ids)} if github_output else {}), + **({count_output: str(len(entry_ids))} if count_output else {}), + } + if step_outputs: + write_step_outputs(step_outputs) @dataset_app.command("view") diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index a95110104..28a917848 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -27,6 +27,7 @@ from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, NL2ALEntry from bcbench.evaluate import AgentRunner, EvaluationPipeline +from bcbench.evaluate.base import EvaluationOutcome from bcbench.evaluate.codereview_judge_calibration import run_calibration from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory, EvaluationContext, ExperimentConfiguration @@ -264,7 +265,7 @@ def evaluate_bcal( category=category, ) - category.pipeline.execute( + outcome = category.pipeline.execute( context, lambda ctx: run_bcal_agent( entry=cast(NL2ALEntry, ctx.entry), @@ -272,6 +273,8 @@ def evaluate_bcal( backend_config=backend_config, ), ) + if outcome is EvaluationOutcome.AGENT_TIMEOUT: + raise typer.Exit(code=1) logger.info("Evaluation complete!") logger.info(f"Results saved to: {run_dir}") diff --git a/src/bcbench/commands/result.py b/src/bcbench/commands/result.py index 71e646c7a..055474ff3 100644 --- a/src/bcbench/commands/result.py +++ b/src/bcbench/commands/result.py @@ -3,18 +3,22 @@ import re from collections import defaultdict from pathlib import Path -from typing import Annotated +from typing import Annotated, cast import typer from bcbench.cli_options import EvaluationCategoryOption, OutputDir, RunId from bcbench.config import get_config +from bcbench.dataset import BaseDatasetEntry +from bcbench.github_actions import write_step_outputs from bcbench.results import ( BaseEvaluationResult, + EvaluationCompleteness, EvaluationResultSummary, Leaderboard, LeaderboardAggregate, create_console_summary, + create_github_completeness_summary, create_github_job_summary, write_bceval_results, ) @@ -23,10 +27,85 @@ _config = get_config() +_MISSING_RESULT_ERROR = "GitHub Actions matrix shard produced no evaluation result artifact. The shard failed before bcbench could persist a result; inspect the matrix job logs." result_app = typer.Typer(help="Process and display evaluation results") +def _parse_expected_instance_ids(expected_entries_json: str | None, expected_total: int) -> list[str] | None: + if expected_entries_json is None or expected_entries_json.strip() in {"", "[]"}: + return None + + try: + payload = json.loads(expected_entries_json) + except json.JSONDecodeError as exc: + raise typer.BadParameter(f"Expected entries must be valid JSON: {exc.msg}") from exc + + if not isinstance(payload, list) or not all(isinstance(instance_id, str) and instance_id for instance_id in payload): + raise typer.BadParameter("Expected entries must be a JSON array of non-empty strings.") + + expected_instance_ids = cast(list[str], payload) + if len(set(expected_instance_ids)) != len(expected_instance_ids): + raise typer.BadParameter("Expected entries must not contain duplicate instance IDs.") + if expected_total >= 0 and expected_total != len(expected_instance_ids): + raise typer.BadParameter(f"Expected total {expected_total} does not match {len(expected_instance_ids)} expected entry IDs.") + return expected_instance_ids + + +def _create_missing_infrastructure_results( + category: EvaluationCategoryOption, + missing_instance_ids: list[str], + existing_results: list[BaseEvaluationResult], + model: str | None, + agent: str | None, +) -> list[BaseEvaluationResult]: + dataset_entries: list[BaseDatasetEntry] = category.entry_class.load(category.dataset_path) + entries_by_id = {entry.instance_id: entry for entry in dataset_entries} + unknown_ids = sorted(set(missing_instance_ids) - entries_by_id.keys()) + if unknown_ids: + logger.error("Expected entries are absent from the %s dataset: %s", category.value, ", ".join(unknown_ids)) + raise typer.Exit(code=1) + + template = existing_results[0] if existing_results else None + resolved_model = template.model if template else model + resolved_agent = template.agent_name if template else agent + if not resolved_model or not resolved_agent: + logger.error("Model and agent are required to synthesize infrastructure failures when no shard produced a result.") + raise typer.Exit(code=1) + + synthetic_results: list[BaseEvaluationResult] = [] + for instance_id in missing_instance_ids: + entry = entries_by_id[instance_id] + payload: dict[str, object] = { + "instance_id": instance_id, + "project": entry.extract_project_name(), + "model": resolved_model.replace(".", "-"), + "agent_name": resolved_agent, + "agent_version": template.agent_version if template else None, + "category": category, + "infrastructure_error": True, + "error_provider": "github-actions", + "error_status_code": None, + "error_message": _MISSING_RESULT_ERROR, + "experiment": template.experiment if template else None, + } + if (judge_model := category.judge_model) is not None: + payload["judge_model"] = judge_model + synthetic_results.append(category.result_class.model_validate(payload)) + + return synthetic_results + + +def _order_results_by_expected_entries(results: list[BaseEvaluationResult], expected_instance_ids: list[str]) -> list[BaseEvaluationResult]: + grouped: defaultdict[str, list[BaseEvaluationResult]] = defaultdict(list) + for result in results: + grouped[result.instance_id].append(result) + + ordered = [result for instance_id in expected_instance_ids for result in grouped.pop(instance_id, [])] + unexpected = [result for result in results if result.instance_id in grouped] + return [*ordered, *unexpected] + + @result_app.command("summarize") def result_summarize( run_id: RunId, @@ -36,6 +115,12 @@ def result_summarize( summary_output: Annotated[str, typer.Option(help="Output filename for summary JSON")] = "evaluation_summary.json", bceval_output: Annotated[str, typer.Option(help="Output filename for bceval results")] = "bceval_results.jsonl", git_ref: Annotated[str | None, typer.Option("--git-ref", help="Git ref (branch/tag) the run was dispatched from; recorded in bceval metadata as git_branch")] = None, + expected_total: Annotated[int, typer.Option(help="Expected number of unique matrix entries; negative disables completeness tracking")] = -1, + expected_entries_json: Annotated[str | None, typer.Option(help="JSON array of expected matrix entry IDs; missing IDs become infrastructure failure results")] = None, + model: Annotated[str | None, typer.Option(help="Model for synthesized infrastructure failure results when no real result is available")] = None, + agent: Annotated[str | None, typer.Option(help="Agent for synthesized infrastructure failure results when no real result is available")] = None, + completeness_output: Annotated[str, typer.Option(help="Output filename for matrix completeness metadata")] = "evaluation_completeness.json", + github_output: Annotated[bool, typer.Option(help="Write produced/expected/complete step outputs to GITHUB_OUTPUT")] = False, ) -> None: """ Summarize evaluation results from a completed run. @@ -43,13 +128,18 @@ def result_summarize( Aggregates individual instance results, displays job summaries and generates bceval output format. """ run_dir: Path = result_dir / run_id + expected_instance_ids = _parse_expected_instance_ids(expected_entries_json, expected_total) + effective_expected_total = len(expected_instance_ids) if expected_instance_ids is not None else expected_total + track_completeness = effective_expected_total >= 0 if not run_dir.exists(): - logger.error(f"Results directory not found: {run_dir}") - raise typer.Exit(code=1) + if not track_completeness: + logger.error(f"Results directory not found: {run_dir}") + raise typer.Exit(code=1) + run_dir.mkdir(parents=True) result_files = list(run_dir.rglob(result_pattern)) - if not result_files: + if not result_files and not track_completeness: logger.error(f"No result files matching '{result_pattern}' found in {run_dir}") raise typer.Exit(code=1) @@ -57,7 +147,7 @@ def result_summarize( instance_pattern_regex = re.compile(_config.file_patterns.instance_pattern) result_files = [f for f in result_files if instance_pattern_regex.match(f.stem)] - if not result_files: + if not result_files and not track_completeness: logger.error(f"No instance-specific result files found in {run_dir}") raise typer.Exit(code=1) @@ -67,13 +157,74 @@ def result_summarize( with results_path.open() as f: results.extend(BaseEvaluationResult.from_json(json.loads(line)) for line in f if line.strip()) - if not results: + if expected_instance_ids is not None: + artifact_completeness = EvaluationCompleteness.from_instance_ids( + effective_expected_total, + (result.instance_id for result in results), + expected_instance_ids=expected_instance_ids, + ) + if artifact_completeness.missing_instance_ids: + synthetic_results = _create_missing_infrastructure_results( + category, + artifact_completeness.missing_instance_ids, + results, + model, + agent, + ) + results.extend(synthetic_results) + logger.warning("Synthesized %d infrastructure failure result(s) for missing matrix artifacts.", len(synthetic_results)) + results = _order_results_by_expected_entries(results, expected_instance_ids) + + if not results and not track_completeness: logger.error("No results found in the result files") raise typer.Exit(code=1) - write_bceval_results(results, run_dir, run_id, bceval_output, category, git_ref=git_ref) + completeness = ( + EvaluationCompleteness.from_instance_ids( + effective_expected_total, + (result.instance_id for result in results), + expected_instance_ids=expected_instance_ids, + infrastructure_failure_instance_ids=(result.instance_id for result in results if result.infrastructure_error), + ) + if track_completeness + else None + ) + if completeness: + completeness.save(run_dir / completeness_output) + logger.info( + "Evaluation completeness: expected=%d produced=%d missing=%d infrastructure_failures=%d unexpected=%d duplicates=%d", + completeness.expected_entry_count, + completeness.produced_entry_count, + completeness.missing_entry_count, + completeness.infrastructure_failure_count, + completeness.unexpected_entry_count, + completeness.duplicate_result_count, + ) + if _config.env.github_actions and not results: + create_github_completeness_summary(completeness) + + if github_output: + output_completeness = completeness or EvaluationCompleteness.from_instance_ids(len({result.instance_id for result in results}), (result.instance_id for result in results)) + write_step_outputs( + { + "expected": str(output_completeness.expected_entry_count), + "produced": str(output_completeness.produced_entry_count), + "missing": str(output_completeness.missing_entry_count), + "infrastructure-failures": str(output_completeness.infrastructure_failure_count), + "complete": str(output_completeness.complete).lower(), + "has-results": str(bool(results)).lower(), + } + ) + + if not results: + logger.error("No matrix shard produced an evaluation result.") + return + + write_bceval_results(results, run_dir, run_id, bceval_output, category, git_ref=git_ref, completeness=completeness) summary = EvaluationResultSummary.from_results(results, run_id=run_id) + if completeness: + summary = summary.model_copy(update={"completeness": completeness}) if _config.env.github_actions: create_github_job_summary(results, summary) @@ -83,6 +234,30 @@ def result_summarize( summary.save(run_dir, summary_output) +@result_app.command("require-complete") +def result_require_complete( + completeness_file: Annotated[Path, typer.Argument(exists=True, file_okay=True, dir_okay=False)], +) -> None: + completeness = EvaluationCompleteness.load(completeness_file) + if completeness.complete: + logger.info( + "Evaluation is complete: %d/%d entries produced.", + completeness.produced_entry_count, + completeness.expected_entry_count, + ) + return + + logger.error( + "Evaluation is incomplete: expected=%d produced=%d missing=%d unexpected=%d duplicates=%d", + completeness.expected_entry_count, + completeness.produced_entry_count, + completeness.missing_entry_count, + completeness.unexpected_entry_count, + completeness.duplicate_result_count, + ) + raise typer.Exit(code=1) + + def _rebuild_aggregates(runs: list[EvaluationResultSummary]) -> list[LeaderboardAggregate]: grouped: defaultdict[tuple[str | None, ...], list[EvaluationResultSummary]] = defaultdict(list) for run in runs: diff --git a/src/bcbench/evaluate/__init__.py b/src/bcbench/evaluate/__init__.py index a0e7f1fd4..fb7d1145d 100644 --- a/src/bcbench/evaluate/__init__.py +++ b/src/bcbench/evaluate/__init__.py @@ -1,6 +1,6 @@ """Evaluation module for running pipelines and creating results.""" -from bcbench.evaluate.base import AgentRunner, EvaluationPipeline +from bcbench.evaluate.base import AgentRunner, EvaluationOutcome, EvaluationPipeline from bcbench.evaluate.bugfix import BugFixPipeline from bcbench.evaluate.codereview import CodeReviewPipeline from bcbench.evaluate.dataquery import DataQueryPipeline @@ -15,6 +15,7 @@ "BugFixPipeline", "CodeReviewPipeline", "DataQueryPipeline", + "EvaluationOutcome", "EvaluationPipeline", "ExtRequestAdvisorPipeline", "ExtRequestImplementPipeline", diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index fdee6867e..11c9714ce 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -2,6 +2,7 @@ import logging from abc import ABC, abstractmethod +from enum import StrEnum from pathlib import Path from typing import Protocol @@ -14,7 +15,12 @@ logger = logging.getLogger(__name__) _config = get_config() -__all__ = ["AgentRunner", "EvaluationPipeline"] +__all__ = ["AgentRunner", "EvaluationOutcome", "EvaluationPipeline"] + + +class EvaluationOutcome(StrEnum): + COMPLETED = "completed" + AGENT_TIMEOUT = "agent_timeout" class AgentRunner[E: BaseDatasetEntry](Protocol): @@ -79,7 +85,7 @@ def execute( self, context: EvaluationContext[E], agent_runner: AgentRunner[E], - ) -> None: + ) -> EvaluationOutcome: """Template method orchestrating the evaluation flow. Executes setup, runs agent, evaluates results, and saves outcomes. @@ -99,12 +105,13 @@ def execute( result = context.category.result_class.create_agent_timeout_failure(context) self.save_result(context, result) logger.info("Agent timed out during execution, counting as failure.") - return + return EvaluationOutcome.AGENT_TIMEOUT finally: logger.info(f"Agent metrics: {context.metrics}") logger.info(f"Experiment configuration: {context.experiment}") self.evaluate(context) + return EvaluationOutcome.COMPLETED def save_result(self, context: EvaluationContext[E], result: BaseEvaluationResult) -> None: """Save result directly using result object. diff --git a/src/bcbench/results/__init__.py b/src/bcbench/results/__init__.py index 1d8112d0d..6971ae82a 100644 --- a/src/bcbench/results/__init__.py +++ b/src/bcbench/results/__init__.py @@ -1,7 +1,8 @@ from bcbench.results.base import ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.results.bceval_export import write_bceval_results from bcbench.results.codereview import CodeReviewResult, CodeReviewResultSummary -from bcbench.results.display import create_console_summary, create_github_job_summary +from bcbench.results.completeness import EvaluationCompleteness +from bcbench.results.display import create_console_summary, create_github_completeness_summary, create_github_job_summary from bcbench.results.leaderboard import ( CodeReviewLeaderboardAggregate, ExecutionBasedLeaderboardAggregate, @@ -20,6 +21,7 @@ "CodeReviewLeaderboardAggregate", "CodeReviewResult", "CodeReviewResultSummary", + "EvaluationCompleteness", "EvaluationResultSummary", "ExecutionBasedEvaluationResult", "ExecutionBasedEvaluationResultSummary", @@ -29,6 +31,7 @@ "Leaderboard", "LeaderboardAggregate", "create_console_summary", + "create_github_completeness_summary", "create_github_job_summary", "write_bceval_results", ] diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 809486724..909f0216c 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -24,6 +24,9 @@ class BaseEvaluationResult(BaseModel): agent_version: str | None = None timeout: bool = False + infrastructure_error: bool = False + error_provider: str | None = None + error_status_code: int | None = None output: str = "" error_message: str | None = None @@ -56,6 +59,23 @@ def _base_fields[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> dic def create_agent_timeout_failure[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> Self: return cls(**cls._base_fields(context), timeout=True, error_message="Agent timed out") + @classmethod + def create_agent_infrastructure_failure[E: BaseDatasetEntry]( + cls, + context: EvaluationContext[E], + *, + error_message: str, + provider: str, + status_code: int | None, + ) -> Self: + return cls( + **cls._base_fields(context), + infrastructure_error=True, + error_provider=provider, + error_status_code=status_code, + error_message=error_message, + ) + def save(self, output_dir: Path, result_file: str) -> None: output_file = output_dir / result_file output_dir.mkdir(parents=True, exist_ok=True) @@ -71,6 +91,8 @@ def status_label(self) -> str: """Short human-readable label for the result status shown in tables (e.g. 'Completed', 'Timeout').""" if self.timeout: return "Timeout" + if self.infrastructure_error: + return "Infrastructure Error" if self.error_message: return "Error" return "Completed" @@ -131,6 +153,8 @@ def create_result[E: BaseDatasetEntry](cls, context: EvaluationContext[E], outpu def status_label(self) -> str: if self.timeout: return "Timeout" + if self.infrastructure_error: + return "Infrastructure Error" return "Success" if self.resolved else "Failed" @property @@ -151,7 +175,7 @@ def restore_missing_timeout_judge_model(cls, payload: object) -> object: return payload payload = cast(dict[str, object], payload) - if payload.get("timeout") is not True or "judge_model" in payload: + if not (payload.get("timeout") is True or payload.get("infrastructure_error") is True) or "judge_model" in payload: return payload category = EvaluationCategory(payload["category"]) @@ -195,6 +219,8 @@ def create_empty_output[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) def status_label(self) -> str: if self.timeout: return "Timeout" + if self.infrastructure_error: + return "Infrastructure Error" if self.error_message: return "Error" return "Unscored" diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index a5e76f820..091e5e241 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -9,6 +9,7 @@ from bcbench.dataset import BaseDatasetEntry from bcbench.results.base import BaseEvaluationResult +from bcbench.results.completeness import EvaluationCompleteness from bcbench.results.summary import get_benchmark_version from bcbench.types import EvaluationCategory, ExpectedOutput, ExperimentConfiguration @@ -37,6 +38,7 @@ def write_bceval_results( output_filename: str, category: EvaluationCategory, git_ref: str | None = None, + completeness: EvaluationCompleteness | None = None, ) -> None: """Write results into a JSONL file for bceval consumption.""" entry_cls = category.entry_class @@ -70,9 +72,20 @@ def write_bceval_results( "run_id": run_id, "project": result.project, "error_message": result.error_message, + "timeout": result.timeout, "tool_usage": (result.metrics.tool_usage if result.metrics and result.metrics.tool_usage else None) or 0, **_experiment_metadata(result.experiment, git_ref, benchmark_version), + **(completeness.to_metadata() if completeness else {}), } + if result.infrastructure_error: + metadata.update( + { + "infrastructure_error": True, + "error_provider": result.error_provider, + "error_status_code": result.error_status_code, + "Error": result.error_message, + } + ) bceval_result = { "id": result.instance_id, diff --git a/src/bcbench/results/completeness.py b/src/bcbench/results/completeness.py new file mode 100644 index 000000000..6f8741dc3 --- /dev/null +++ b/src/bcbench/results/completeness.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Iterable +from pathlib import Path + +from pydantic import BaseModel, Field + + +class EvaluationCompleteness(BaseModel): + expected_entry_count: int = Field(ge=0) + produced_entry_count: int = Field(ge=0) + missing_entry_count: int = Field(ge=0) + unexpected_entry_count: int = Field(ge=0) + duplicate_result_count: int = Field(ge=0) + infrastructure_failure_count: int = Field(ge=0) + missing_instance_ids: list[str] = Field(default_factory=list) + unexpected_instance_ids: list[str] = Field(default_factory=list) + duplicate_instance_ids: list[str] = Field(default_factory=list) + infrastructure_failure_instance_ids: list[str] = Field(default_factory=list) + complete: bool + + @classmethod + def from_instance_ids( + cls, + expected_entry_count: int, + instance_ids: Iterable[str], + *, + expected_instance_ids: Iterable[str] | None = None, + infrastructure_failure_instance_ids: Iterable[str] = (), + ) -> EvaluationCompleteness: + ids = list(instance_ids) + counts = Counter(ids) + produced_ids = set(counts) + produced_entry_count = len(produced_ids) + duplicate_instance_ids = sorted(instance_id for instance_id, count in counts.items() if count > 1) + duplicate_result_count = sum(count - 1 for count in counts.values()) + + if expected_instance_ids is None: + missing_instance_ids: list[str] = [] + unexpected_instance_ids: list[str] = [] + missing_entry_count = max(expected_entry_count - produced_entry_count, 0) + unexpected_entry_count = max(produced_entry_count - expected_entry_count, 0) + else: + expected_ids = list(expected_instance_ids) + expected_set = set(expected_ids) + if len(expected_set) != len(expected_ids): + raise ValueError("Expected instance IDs must be unique.") + if len(expected_ids) != expected_entry_count: + raise ValueError(f"Expected entry count {expected_entry_count} does not match {len(expected_ids)} expected instance IDs.") + + missing_instance_ids = sorted(expected_set - produced_ids) + unexpected_instance_ids = sorted(produced_ids - expected_set) + missing_entry_count = len(missing_instance_ids) + unexpected_entry_count = len(unexpected_instance_ids) + + infrastructure_ids = sorted(set(infrastructure_failure_instance_ids)) + complete = missing_entry_count == 0 and unexpected_entry_count == 0 and duplicate_result_count == 0 + return cls( + expected_entry_count=expected_entry_count, + produced_entry_count=produced_entry_count, + missing_entry_count=missing_entry_count, + unexpected_entry_count=unexpected_entry_count, + duplicate_result_count=duplicate_result_count, + infrastructure_failure_count=len(infrastructure_ids), + missing_instance_ids=missing_instance_ids, + unexpected_instance_ids=unexpected_instance_ids, + duplicate_instance_ids=duplicate_instance_ids, + infrastructure_failure_instance_ids=infrastructure_ids, + complete=complete, + ) + + def to_metadata(self) -> dict[str, int | bool]: + return { + "expected_entry_count": self.expected_entry_count, + "produced_entry_count": self.produced_entry_count, + "missing_entry_count": self.missing_entry_count, + "unexpected_entry_count": self.unexpected_entry_count, + "duplicate_result_count": self.duplicate_result_count, + "infrastructure_failure_count": self.infrastructure_failure_count, + "evaluation_complete": self.complete, + } + + def save(self, output_file: Path) -> None: + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(self.model_dump_json(indent=2), encoding="utf-8") + + @classmethod + def load(cls, input_file: Path) -> EvaluationCompleteness: + return cls.model_validate(json.loads(input_file.read_text(encoding="utf-8"))) + + def render_markdown(self) -> str: + status = ":white_check_mark: Complete" if self.complete else ":x: Incomplete" + return "\n".join( + [ + "## Evaluation completeness", + "", + "| Expected entries | Produced entries | Missing entries | Infrastructure failures | Unexpected entries | Duplicate results | Status |", + "|---:|---:|---:|---:|---:|---:|:---|", + ( + f"| {self.expected_entry_count} | {self.produced_entry_count} | {self.missing_entry_count} | " + f"{self.infrastructure_failure_count} | {self.unexpected_entry_count} | {self.duplicate_result_count} | {status} |" + ), + ] + ) diff --git a/src/bcbench/results/display.py b/src/bcbench/results/display.py index f7cf3b3c6..3acd07c03 100644 --- a/src/bcbench/results/display.py +++ b/src/bcbench/results/display.py @@ -7,6 +7,7 @@ from bcbench.config import get_config from bcbench.results.base import BaseEvaluationResult +from bcbench.results.completeness import EvaluationCompleteness from bcbench.results.summary import EvaluationResultSummary, calculate_average_tool_usage logger = logging.getLogger(__name__) @@ -15,7 +16,7 @@ def _status_style(status_label: str) -> tuple[str, str]: """Return (rich_color, github_emoji) for a status label.""" - if status_label in ("Timeout", "Error", "Failed"): + if status_label in ("Timeout", "Infrastructure Error", "Error", "Failed"): return "red", ":x:" if status_label == "Unscored": return "yellow", ":grey_question:" @@ -24,6 +25,8 @@ def _status_style(status_label: str) -> tuple[str, str]: def create_console_summary(results: Sequence[BaseEvaluationResult], summary: EvaluationResultSummary) -> None: console.print("\n[bold cyan]Evaluation Results Summary[/bold cyan]") + if summary.completeness: + console.print(f"Completeness: [bold]{summary.completeness.produced_entry_count}/{summary.completeness.expected_entry_count}[/bold] ({summary.completeness.missing_entry_count} missing/failed)") console.print(f"Total Processed: [bold]{len(results)}[/bold], using [bold]{results[0].agent_name}({results[0].model})[/bold]") console.print(f"Agent Version: [bold]{results[0].agent_version or 'Unrecorded'}[/bold]") console.print(f"Category: [bold]{results[0].category.value}[/bold]") @@ -80,6 +83,7 @@ def _get_short_error_message(error_message: str | None) -> str: def create_github_job_summary(results: Sequence[BaseEvaluationResult], summary: EvaluationResultSummary) -> None: metrics_section: str = summary.render_github_metrics_markdown().strip() + completeness_section = summary.completeness.render_markdown() if summary.completeness else "" # Calculate average tool usage tool_usage_section: str = "" @@ -104,7 +108,10 @@ def create_github_job_summary(results: Sequence[BaseEvaluationResult], summary: f"- Plugins: {', '.join(results[0].experiment.plugins) if results[0].experiment and results[0].experiment.plugins else 'None'}", ] ) - sections: list[str] = [header_section, *(section for section in [metrics_section, tool_usage_section] if section), "## Detailed Results\n\n"] + sections: list[str] = [ + *(section for section in [completeness_section, header_section, metrics_section, tool_usage_section] if section), + "## Detailed Results\n\n", + ] markdown_summary: str = "\n\n".join(sections) # Dynamic columns from display_row() @@ -132,6 +139,10 @@ def create_github_job_summary(results: Sequence[BaseEvaluationResult], summary: _write_github_step_summary(markdown_summary) +def create_github_completeness_summary(completeness: EvaluationCompleteness) -> None: + _write_github_step_summary(completeness.render_markdown()) + + def _write_github_step_summary(content: str) -> None: config = get_config() if config.env.github_step_summary: diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index 0aa202779..2c0b3e2a6 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, Field from bcbench.results.base import BaseEvaluationResult +from bcbench.results.completeness import EvaluationCompleteness from bcbench.types import EvaluationCategory, ExperimentConfiguration if TYPE_CHECKING: @@ -58,6 +59,7 @@ class EvaluationResultSummary(BaseModel, ABC): github_run_id: str | None = None experiment: ExperimentConfiguration | None = None + completeness: EvaluationCompleteness | None = None benchmark_version: str @@ -131,6 +133,8 @@ def to_dict(self) -> dict[str, Any]: data["average_llm_duration"] = round(data["average_llm_duration"], 1) if data["average_llm_duration"] is not None else None if data["average_tool_usage"] is None: del data["average_tool_usage"] + if data["completeness"] is None: + del data["completeness"] return data def save(self, output_dir: Path, summary_file: str) -> None: diff --git a/tests/test_bceval_scores.py b/tests/test_bceval_scores.py new file mode 100644 index 000000000..dc01921e9 --- /dev/null +++ b/tests/test_bceval_scores.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import sys +import types +from typing import cast + +import pytest + +from evaluator import scores + + +class _Score: + def __init__(self, *, name, score, metadata): + self.name = name + self.score = score + self.metadata = metadata + + +def _install_lm_checklist_dependencies(monkeypatch): + class _BuiltInLmChecklist: + def __init__(self): + self.calls = [] + + def _run_eval_sync(self, output, expected, **kwargs): + self.calls.append((output, expected, kwargs)) + return [_Score(name="test_passed", score=1.0, metadata={})] + + @staticmethod + def _rate(assertions, level=None): + selected = [assertion for assertion in assertions if level is None or assertion.get("level") == level] + if not selected: + return 0.0 + return sum(1 for assertion in selected if assertion["pass"]) / len(selected) + + bc_eval_module = types.ModuleType("bc_eval.scorers.autoeval.lmchecklist") + bc_eval_module.LmChecklist = _BuiltInLmChecklist # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, bc_eval_module.__name__, bc_eval_module) + + autoevals_module = types.ModuleType("autoevals") + autoevals_module.Score = _Score # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, autoevals_module.__name__, autoevals_module) + return _BuiltInLmChecklist + + +def test_lm_checklist_delegates_normal_results_and_selects_row_core_score(monkeypatch): + built_in = _install_lm_checklist_dependencies(monkeypatch) + scorer = cast(built_in, scores.LmChecklist()) + metadata = {} + + result = scorer._run_eval_sync("answer", {"assertions": []}, metadata=metadata) + + assert {score.name: score.score for score in result} == {"test_passed": 1.0} + assert metadata["core_score_name"] == "test_passed" + assert metadata["core_score"] == 1.0 + assert len(scorer.calls) == 1 + + +@pytest.mark.parametrize( + "delegated_scores", + [ + [], + [_Score(name="pass_rate", score=1.0, metadata={})], + [_Score(name="test_passed", score=None, metadata={})], + ], +) +def test_lm_checklist_rejects_normal_results_without_core_score(monkeypatch, delegated_scores): + built_in = _install_lm_checklist_dependencies(monkeypatch) + monkeypatch.setattr( + built_in, + "_run_eval_sync", + lambda self, output, expected, **kwargs: delegated_scores, + ) + scorer = cast(built_in, scores.LmChecklist()) + + metadata = {} + with pytest.raises(ValueError, match="exactly one non-null 'test_passed'"): + scorer._run_eval_sync("answer", {"assertions": []}, metadata=metadata) + + assert metadata["core_score_name"] == "test_passed" + with pytest.raises(ValueError, match="could not convert string to float"): + float(metadata["core_score"]) + + +def test_lm_checklist_leaves_hard_failure_guard_when_judge_raises(monkeypatch): + built_in = _install_lm_checklist_dependencies(monkeypatch) + + def raise_judge_error(self, output, expected, **kwargs): + raise RuntimeError("Judge failed") + + monkeypatch.setattr(built_in, "_run_eval_sync", raise_judge_error) + scorer = cast(built_in, scores.LmChecklist()) + metadata = {} + + with pytest.raises(RuntimeError, match="Judge failed"): + scorer._run_eval_sync("answer", {"assertions": []}, metadata=metadata) + + assert metadata["core_score_name"] == "test_passed" + with pytest.raises(ValueError, match="could not convert string to float"): + float(metadata["core_score"]) + + +def test_lm_checklist_scores_timeout_without_calling_judge(monkeypatch): + built_in = _install_lm_checklist_dependencies(monkeypatch) + metadata = {"timeout": True} + assertions = [ + {"text": "Critical behavior.", "level": "critical"}, + {"text": "Expected behavior.", "level": "expected"}, + ] + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync("", {"assertions": assertions}, metadata=metadata) + + assert {score.name: score.score for score in result} == { + "pass_rate": 0.0, + "critical_pass_rate": 0.0, + "expected_pass_rate": 0.0, + "aspirational_pass_rate": 0.0, + "test_passed": 0.0, + } + assert scorer.calls == [] + assert metadata["core_score_name"] == "test_passed" + assert metadata["core_score"] == 0.0 + assert metadata["assertionResults"] == [{**assertion, "pass": False, "reasoning": "Agent timed out before producing output"} for assertion in assertions] + + +def test_lm_checklist_skips_infrastructure_error_without_calling_judge(monkeypatch): + built_in = _install_lm_checklist_dependencies(monkeypatch) + error_message = "GitHub Actions dependency setup failed before evaluation." + metadata = { + "infrastructure_error": True, + "Error": error_message, + } + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync( + "", + {"assertions": [{"text": "Critical behavior.", "level": "critical"}]}, + metadata=metadata, + ) + + assert {score.name: score.score for score in result} == { + "pass_rate": None, + "critical_pass_rate": None, + "expected_pass_rate": None, + "aspirational_pass_rate": None, + "test_passed": None, + } + assert all(score.metadata == {"error": error_message} for score in result) + assert metadata["core_score_name"] == "test_passed" + assert "core_score" not in metadata + assert scorer.calls == [] + + +def test_lm_checklist_accepts_publishable_complete_mixed_batch(monkeypatch): + built_in = _install_lm_checklist_dependencies(monkeypatch) + scorer = cast(built_in, scores.LmChecklist()) + metadata = [{} for _ in range(109)] + + rows = [scorer._run_eval_sync("answer", {"assertions": []}, metadata=row_metadata) for row_metadata in metadata] + infrastructure_metadata = {"infrastructure_error": True, "Error": "Dependency setup failed."} + rows.append( + scorer._run_eval_sync( + "", + {"assertions": [{"text": "Critical behavior.", "level": "critical"}]}, + metadata=infrastructure_metadata, + ) + ) + + assert len(rows) == 110 + assert all(row_metadata["core_score_name"] == "test_passed" for row_metadata in metadata) + assert all(row_metadata["core_score"] == 1.0 for row_metadata in metadata) + assert infrastructure_metadata["core_score_name"] == "test_passed" + assert "core_score" not in infrastructure_metadata + assert all(next(score for score in row if score.name == "test_passed").score == 1.0 for row in rows[:109]) + assert next(score for score in rows[-1] if score.name == "test_passed").score is None diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index b97d2c856..1214881d8 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -15,7 +15,9 @@ from bcbench.cli_options import resolve_agent_runtime, resolve_evaluation_runtime from bcbench.commands import evaluate as evaluate_commands from bcbench.commands import run as run_commands -from bcbench.dataset.dataset_entry import _BugFixTestGenBase +from bcbench.dataset.dataset_entry import NL2ALEntry, _BugFixTestGenBase +from bcbench.evaluate.base import EvaluationOutcome +from bcbench.results.base import JudgeBasedEvaluationResult from bcbench.types import AgentMetrics, EvaluationCategory from tests.conftest import ( create_bugfix_result, @@ -125,6 +127,7 @@ def load(_dataset_path, entry_id: str): class Pipeline: def execute(self, context, _agent_runner): captured["context"] = context + return EvaluationOutcome.COMPLETED with ( patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=tmp_path / "nl2al.jsonl"), @@ -143,6 +146,37 @@ def execute(self, context, _agent_runner): assert captured["context"].model == expected_model +def test_evaluate_bcal_exits_nonzero_after_persisted_timeout(tmp_path): + entry = create_nl2al_entry() + + class EntryClass: + @staticmethod + def load(_dataset_path, entry_id: str): + assert entry_id == entry.instance_id + return [entry] + + class Pipeline: + def execute(self, _context, _agent_runner): + return EvaluationOutcome.AGENT_TIMEOUT + + with ( + patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=tmp_path / "nl2al.jsonl"), + patch.object(EvaluationCategory, "entry_class", new_callable=PropertyMock, return_value=EntryClass), + patch.object(EvaluationCategory, "pipeline", new_callable=PropertyMock, return_value=Pipeline()), + pytest.raises(typer.Exit) as exc_info, + ): + evaluate_commands.evaluate_bcal( + entry_id=entry.instance_id, + repo_path=tmp_path / "repo", + output_dir=tmp_path / "results", + run_id="bcal-run", + llm_command="python bridge.py", + llm_model="gpt-55-chat", + ) + + assert exc_info.value.exit_code == 1 + + def _option_names(command: click.Command) -> set[str]: return {option for parameter in command.params if isinstance(parameter, click.Option) for option in (*parameter.opts, *parameter.secondary_opts)} @@ -432,6 +466,149 @@ def test_result_summarize_with_custom_pattern(sample_results_directory, problem_ assert result.exit_code == 0 +@pytest.mark.integration +def test_result_summarize_records_partial_matrix_completeness(sample_results_directory, problem_statement_dir, tmp_path, monkeypatch): + base_path, run_id, dataset_path = sample_results_directory + github_output = tmp_path / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(github_output)) + + with ( + patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), + patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=dataset_path), + ): + result = runner.invoke( + app, + [ + "result", + "summarize", + "--category", + "bug-fix", + "--run-id", + run_id, + "--result-dir", + str(base_path), + "--expected-total", + "4", + "--github-output", + ], + ) + + assert result.exit_code == 0, result.exception + run_dir = base_path / run_id + completeness = json.loads((run_dir / "evaluation_completeness.json").read_text()) + assert completeness["expected_entry_count"] == 4 + assert completeness["produced_entry_count"] == 3 + assert completeness["missing_entry_count"] == 1 + assert completeness["complete"] is False + + summary = json.loads((run_dir / "evaluation_summary.json").read_text()) + assert summary["completeness"] == completeness + exported = [json.loads(line) for line in (run_dir / "bceval_results.jsonl").read_text().splitlines()] + assert {row["metadata"]["evaluation_complete"] for row in exported} == {False} + assert "complete=false" in github_output.read_text() + + +@pytest.mark.integration +def test_result_summarize_zero_results_still_writes_completeness(tmp_path): + run_id = "empty-matrix" + result = runner.invoke( + app, + [ + "result", + "summarize", + "--category", + "nl2al", + "--run-id", + run_id, + "--result-dir", + str(tmp_path), + "--expected-total", + "110", + ], + ) + + assert result.exit_code == 0 + completeness = json.loads((tmp_path / run_id / "evaluation_completeness.json").read_text()) + assert completeness["produced_entry_count"] == 0 + assert completeness["missing_entry_count"] == 110 + assert completeness["complete"] is False + assert not (tmp_path / run_id / "evaluation_summary.json").exists() + + +@pytest.mark.integration +def test_result_summarize_synthesizes_pre_evaluation_failure_for_full_110_entry_report(tmp_path, monkeypatch): + run_id = "bcal-run" + run_dir = tmp_path / run_id + run_dir.mkdir() + entries = [create_nl2al_entry(instance_id=f"nl2al__matrix-entry-{index}") for index in range(1, 111)] + judge_model = EvaluationCategory.NL2AL.judge_model + assert judge_model is not None + + for entry in entries[:-1]: + result = JudgeBasedEvaluationResult( + instance_id=entry.instance_id, + project=entry.extract_project_name(), + model="gpt-55-chat", + agent_name="BCal", + category=EvaluationCategory.NL2AL, + output="generated AL", + judge_model=judge_model, + ) + result.save(run_dir, f"{entry.instance_id}.jsonl") + + github_output = tmp_path / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(github_output)) + expected_instance_ids = [entry.instance_id for entry in entries] + with patch.object(NL2ALEntry, "load", return_value=entries): + result = runner.invoke( + app, + [ + "result", + "summarize", + "--category", + "nl2al", + "--run-id", + run_id, + "--result-dir", + str(tmp_path), + "--expected-total", + "110", + "--expected-entries-json", + json.dumps(expected_instance_ids), + "--model", + "gpt-55-chat", + "--agent", + "BCal", + "--github-output", + ], + ) + + assert result.exit_code == 0, result.exception + failed_instance_id = entries[-1].instance_id + completeness = json.loads((run_dir / "evaluation_completeness.json").read_text(encoding="utf-8")) + assert completeness["expected_entry_count"] == 110 + assert completeness["produced_entry_count"] == 110 + assert completeness["missing_entry_count"] == 0 + assert completeness["infrastructure_failure_count"] == 1 + assert completeness["infrastructure_failure_instance_ids"] == [failed_instance_id] + assert completeness["complete"] is True + + summary = json.loads((run_dir / "evaluation_summary.json").read_text(encoding="utf-8")) + assert summary["total"] == 110 + assert summary["completeness"] == completeness + + exported = [json.loads(line) for line in (run_dir / "bceval_results.jsonl").read_text(encoding="utf-8").splitlines()] + assert len(exported) == 110 + failed_row = next(row for row in exported if row["id"] == failed_instance_id) + assert failed_row["metadata"]["infrastructure_error"] is True + assert failed_row["metadata"]["error_provider"] == "github-actions" + assert "before bcbench could persist a result" in failed_row["metadata"]["Error"] + assert "test_passed" not in failed_row["metadata"] + assert "produced=110" in github_output.read_text(encoding="utf-8") + assert "infrastructure-failures=1" in github_output.read_text(encoding="utf-8") + assert "complete=true" in github_output.read_text(encoding="utf-8") + + @pytest.mark.integration def test_dataset_list_displays_all_entries(sample_dataset_file_for_cli): with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): @@ -450,6 +627,18 @@ def test_dataset_list_displays_all_entries(sample_dataset_file_for_cli): assert "Found 3 entry(ies)" in result.stdout +@pytest.mark.integration +def test_dataset_list_writes_entry_count_output(sample_dataset_file_for_cli, tmp_path, monkeypatch): + github_output = tmp_path / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(github_output)) + + with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): + result = runner.invoke(app, ["dataset", "list", "--count-output", "entry-count"]) + + assert result.exit_code == 0 + assert github_output.read_text() == "entry-count=3\n" + + @pytest.mark.integration def test_dataset_bc_version_returns_entry_version(sample_dataset_file_for_cli): with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): diff --git a/tests/test_completeness.py b/tests/test_completeness.py new file mode 100644 index 000000000..f4f0d56ba --- /dev/null +++ b/tests/test_completeness.py @@ -0,0 +1,40 @@ +from bcbench.results.completeness import EvaluationCompleteness + + +def test_completeness_counts_unique_missing_and_duplicate_results(): + completeness = EvaluationCompleteness.from_instance_ids(4, ["a", "b", "b"]) + + assert completeness.expected_entry_count == 4 + assert completeness.produced_entry_count == 2 + assert completeness.missing_entry_count == 2 + assert completeness.duplicate_result_count == 1 + assert completeness.complete is False + + +def test_completeness_metadata_is_explicit_for_external_storage(): + completeness = EvaluationCompleteness.from_instance_ids(2, ["a", "b"], infrastructure_failure_instance_ids=["b"]) + + assert completeness.to_metadata() == { + "expected_entry_count": 2, + "produced_entry_count": 2, + "missing_entry_count": 0, + "unexpected_entry_count": 0, + "duplicate_result_count": 0, + "infrastructure_failure_count": 1, + "evaluation_complete": True, + } + + +def test_completeness_compares_exact_expected_ids(): + completeness = EvaluationCompleteness.from_instance_ids( + 2, + ["a", "a", "c"], + expected_instance_ids=["a", "b"], + ) + + assert completeness.produced_entry_count == 2 + assert completeness.missing_instance_ids == ["b"] + assert completeness.unexpected_instance_ids == ["c"] + assert completeness.duplicate_instance_ids == ["a"] + assert completeness.duplicate_result_count == 1 + assert completeness.complete is False diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 00cd01890..e4abbb7c2 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -10,7 +10,7 @@ from bcbench.commands.evaluate import MockEvaluationPipeline from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, BugFixEntry, NL2ALEntry -from bcbench.evaluate.base import AgentRunner, EvaluationPipeline +from bcbench.evaluate.base import AgentRunner, EvaluationOutcome, EvaluationPipeline from bcbench.exceptions import AgentTimeoutError from bcbench.results.base import BaseEvaluationResult, JudgeBasedEvaluationResult from bcbench.types import AgentMetrics, EvaluationCategory, EvaluationContext, ExperimentConfiguration @@ -64,8 +64,9 @@ def test_runs_setup_then_agent_then_evaluate(self, tmp_path): ctx = create_evaluation_context(tmp_path) pipeline = _StubPipeline[BugFixEntry]() - pipeline.execute(ctx, _noop_runner) + outcome = pipeline.execute(ctx, _noop_runner) + assert outcome is EvaluationOutcome.COMPLETED assert pipeline.setup_called assert pipeline.run_agent_called assert pipeline.evaluate_called @@ -78,8 +79,9 @@ def test_persists_timeout_result_and_skips_evaluate(self, tmp_path): timeout_config = ExperimentConfiguration(custom_instructions=True) pipeline = _StubPipeline[BugFixEntry](raise_in_run_agent=AgentTimeoutError("test timeout", metrics=timeout_metrics, config=timeout_config)) - pipeline.execute(ctx, _noop_runner) + outcome = pipeline.execute(ctx, _noop_runner) + assert outcome is EvaluationOutcome.AGENT_TIMEOUT assert pipeline.evaluate_called is False result = _read_only_result(ctx) assert result.timeout is True @@ -92,8 +94,9 @@ def test_persists_category_specific_timeout_result(self, tmp_path): ctx = create_evaluation_context(tmp_path, entry=entry, category=EvaluationCategory.NL2AL) pipeline = _StubPipeline[NL2ALEntry](raise_in_run_agent=AgentTimeoutError("test timeout")) - pipeline.execute(ctx, lambda _: (None, None)) + outcome = pipeline.execute(ctx, lambda _: (None, None)) + assert outcome is EvaluationOutcome.AGENT_TIMEOUT result_file = ctx.result_dir / f"{entry.instance_id}{get_config().file_patterns.result_pattern}" payload = json.loads(result_file.read_text(encoding="utf-8")) assert payload["judge_model"] == EvaluationCategory.NL2AL.judge_model diff --git a/tests/test_result_hierarchy.py b/tests/test_result_hierarchy.py index 556888b7e..1a28ea921 100644 --- a/tests/test_result_hierarchy.py +++ b/tests/test_result_hierarchy.py @@ -87,6 +87,10 @@ def test_execution_based_failed(self): result = create_bugfix_result(resolved=False, build=True, error_message="Tests failed") assert result.status_label == "Failed" + def test_infrastructure_error(self): + result = create_bugfix_result(resolved=False, build=False).model_copy(update={"infrastructure_error": True}) + assert result.status_label == "Infrastructure Error" + # --------------------------------------------------------------------------- # category_metrics diff --git a/tests/test_result_writer.py b/tests/test_result_writer.py index 5dc8a0b0d..9ef457815 100644 --- a/tests/test_result_writer.py +++ b/tests/test_result_writer.py @@ -4,13 +4,75 @@ import pytest from bcbench.dataset.codereview import CodeReviewEntry -from bcbench.dataset.dataset_entry import BugFixEntry, _BugFixTestGenBase +from bcbench.dataset.dataset_entry import BugFixEntry, NL2ALEntry, _BugFixTestGenBase +from bcbench.results.base import JudgeBasedEvaluationResult from bcbench.results.bceval_export import write_bceval_results from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory, ExperimentConfiguration, PRReviewMetrics -from tests.conftest import VALID_INSTANCE_ID, create_bugfix_result, create_codereview_entry, create_codereview_result +from tests.conftest import VALID_INSTANCE_ID, create_bugfix_result, create_codereview_entry, create_codereview_result, create_nl2al_entry class TestWriteBcevalResults: + def test_exports_timeout_metadata(self, tmp_path): + entry = create_nl2al_entry() + judge_model = EvaluationCategory.NL2AL.judge_model + assert judge_model is not None + result = JudgeBasedEvaluationResult( + instance_id=entry.instance_id, + project="BaseApp", + model="gpt-55-chat", + agent_name=AgentHarness.BCAL, + category=EvaluationCategory.NL2AL, + timeout=True, + error_message="Agent timed out", + judge_model=judge_model, + ) + + with patch.object(NL2ALEntry, "load", return_value=[entry]): + write_bceval_results( + results=[result], + out_dir=tmp_path, + run_id="run", + output_filename="results.jsonl", + category=EvaluationCategory.NL2AL, + ) + + metadata = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metadata"] + assert metadata["timeout"] is True + assert metadata["error_message"] == "Agent timed out" + + def test_exports_infrastructure_error_without_model_score(self, tmp_path): + entry = create_nl2al_entry() + judge_model = EvaluationCategory.NL2AL.judge_model + assert judge_model is not None + error_message = "GitHub Actions dependency setup failed before evaluation." + result = JudgeBasedEvaluationResult( + instance_id=entry.instance_id, + project="BaseApp", + model="gpt-55-chat", + agent_name=AgentHarness.BCAL, + category=EvaluationCategory.NL2AL, + infrastructure_error=True, + error_provider="github-actions", + error_message=error_message, + judge_model=judge_model, + ) + + with patch.object(NL2ALEntry, "load", return_value=[entry]): + write_bceval_results( + results=[result], + out_dir=tmp_path, + run_id="run", + output_filename="results.jsonl", + category=EvaluationCategory.NL2AL, + ) + + metadata = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metadata"] + assert metadata["infrastructure_error"] is True + assert metadata["error_provider"] == "github-actions" + assert metadata["error_status_code"] is None + assert metadata["Error"] == error_message + assert "test_passed" not in metadata + @pytest.mark.parametrize( "metrics", [ @@ -70,6 +132,8 @@ def test_writes_bceval_results_with_all_fields(self, tmp_path, sample_dataset_fi assert data["metadata"]["agent_version"] == "1.2.3" # bug-fix is not judge-scored, so no judge model is exported assert "judge_model" not in data["metadata"] + assert "infrastructure_error" not in data["metadata"] + assert "Error" not in data["metadata"] assert data["metadata"]["prompt_tokens"] == 5000 assert data["metadata"]["completion_tokens"] == 1200 assert data["metadata"]["latency"] == 120.5 diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index 3f58a14a0..4bc514480 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -203,3 +203,67 @@ def test_agent_workflows_select_al_tool_dotnet_version_for_bc_version() -> None: for workflow_name in ("claude-evaluation.yml", "copilot-evaluation.yml"): workflow = _workflow(workflow_name) assert '--framework "net${{ steps.setup-env.outputs.al_tool_dotnet_version }}"' in workflow + + +def test_bcal_summary_runs_after_matrix_failures_and_enforces_completeness() -> None: + workflow = yaml.safe_load(_workflow("bcal-evaluation.yml")) + evaluation_steps = workflow["jobs"]["evaluate-with-bcal"]["steps"] + summarize = workflow["jobs"]["summarize-results"] + upload = next(step for step in evaluation_steps if step["name"] == "Upload evaluation results") + + assert upload["if"] == "always()" + assert summarize["needs"] == ["get-entries", "evaluate-with-bcal"] + assert "always()" in summarize["if"] + assert summarize["with"]["expected-total"] == "${{ fromJSON(needs.get-entries.outputs.entry-count) }}" + assert summarize["with"]["expected-entries"] == "${{ needs.get-entries.outputs.entries }}" + assert summarize["with"]["results-dir"] == "evaluation_results" + assert summarize["with"]["artifact-pattern"] == "evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-*" + assert summarize["with"]["allow-unscored-results"] is True + + +def test_bcal_retries_python_setup_once_without_hiding_final_failure() -> None: + workflow = yaml.safe_load(_workflow("bcal-evaluation.yml")) + steps = workflow["jobs"]["evaluate-with-bcal"]["steps"] + setup = next(step for step in steps if step["name"] == "Setup Python with UV") + wait = next(step for step in steps if step["name"] == "Wait before retrying Python setup") + retry = next(step for step in steps if step["name"] == "Retry Setup Python with UV") + + assert setup["id"] == "setup-python" + assert setup["continue-on-error"] is True + assert wait["if"] == "steps.setup-python.outcome == 'failure'" + assert "Start-Sleep -Seconds 20" in wait["run"] + assert retry["if"] == "steps.setup-python.outcome == 'failure'" + assert retry["uses"] == setup["uses"] + assert "continue-on-error" not in retry + + +def test_bcal_downloads_one_pinned_cached_bccontainerhelper() -> None: + workflow_text = _workflow("bcal-evaluation.yml") + workflow = yaml.safe_load(workflow_text) + prepare_steps = workflow["jobs"]["prepare-bccontainerhelper"]["steps"] + download_step = next(step for step in prepare_steps if step["name"] == "Download pinned BcContainerHelper") + download_script = download_step["run"] + + assert "BCCONTAINERHELPER_VERSION: 6.1.18" in workflow_text + assert download_script.count("Save-Module") == 1 + assert "RequiredVersion $env:BCCONTAINERHELPER_VERSION" in download_script + assert "fail-on-cache-miss: true" in workflow_text + assert "Install-Module -Name BcContainerHelper" not in workflow_text + + +def test_summarize_workflow_preserves_partial_diagnostics_and_blocks_upload() -> None: + workflow_text = _workflow("summarize-results.yml") + workflow = yaml.safe_load(workflow_text) + + assert "continue-on-error: ${{ inputs.expected-total >= 0 }}" in workflow_text + assert "evaluation_completeness.json" in workflow_text + assert "bceval_results.jsonl" in workflow_text + assert "steps.summarize.outputs.complete == 'true'" in workflow_text + assert "bcbench.bceval_runner" not in workflow_text + assert "if: always()" in workflow_text + assert "bcbench result require-complete" in workflow_text + assert workflow[True]["workflow_call"]["inputs"]["allow-unscored-results"]["default"] is False + assert 'CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}")' in workflow_text + assert 'if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]' in workflow_text + assert '"${CORE_SCORE_ARGS[@]}"' in workflow_text + assert "${{ !inputs.mock && '--storage braintrust --storage kusto' || '' }}" in workflow_text