Harden BCal aggregation against shard setup failures

.github/workflows/bcal-evaluation.yml

@@ -30,6 +30,7 @@ concurrency:
 
 env:
   EVALUATION_RESULTS_DIR: evaluation_results
+  BCCONTAINERHELPER_VERSION: 6.1.18
 
 jobs:
   get-entries:
@@ -38,9 +39,44 @@ jobs:
       test-run: ${{ inputs.test-run || false }}
       category: nl2al
 
-  evaluate-with-bcal:
+  prepare-bccontainerhelper:
     runs-on: windows-latest
     needs: get-entries
+    if: needs.get-entries.outputs.entries != '[]'
+    steps:
+      - name: Cache pinned BcContainerHelper
+        id: cache-bccontainerhelper
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~\Documents\PowerShell\Modules\BcContainerHelper
+          key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }}
+
+      - name: Download pinned BcContainerHelper
+        if: steps.cache-bccontainerhelper.outputs.cache-hit != 'true'
+        shell: pwsh
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $moduleRoot = Join-Path $HOME 'Documents\PowerShell\Modules'
+          New-Item -ItemType Directory -Path $moduleRoot -Force | Out-Null
+          Save-Module `
+            -Name BcContainerHelper `
+            -RequiredVersion $env:BCCONTAINERHELPER_VERSION `
+            -Repository PSGallery `
+            -Path $moduleRoot `
+            -Force `
+            -ErrorAction Stop
+
+      - name: Verify pinned BcContainerHelper
+        shell: pwsh
+        run: |
+          Import-Module BcContainerHelper -RequiredVersion $env:BCCONTAINERHELPER_VERSION -Force -DisableNameChecking
+          if ((Get-Module BcContainerHelper).Version -ne [Version]$env:BCCONTAINERHELPER_VERSION) {
+            throw "Expected BcContainerHelper $env:BCCONTAINERHELPER_VERSION."
+          }
+
+  evaluate-with-bcal:
+    runs-on: windows-latest
+    needs: [get-entries, prepare-bccontainerhelper]
     outputs:
       results-dir: ${{ env.EVALUATION_RESULTS_DIR }}
     if: needs.get-entries.outputs.entries != '[]'
@@ -61,6 +97,17 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Setup Python with UV
+        id: setup-python
+        continue-on-error: true
+        uses: $/.github/actions/setup-python-uv
+
+      - name: Wait before retrying Python setup
+        if: steps.setup-python.outcome == 'failure'
+        shell: pwsh
+        run: Start-Sleep -Seconds 20
+
+      - name: Retry Setup Python with UV
+        if: steps.setup-python.outcome == 'failure'
         uses: $/.github/actions/setup-python-uv
 
       - name: Azure Login (OIDC) for ADO feed
@@ -135,6 +182,13 @@ jobs:
         id: bcversion
         run: uv run bcbench dataset version "${{ matrix.entry }}" --category nl2al --github-output version
 
+      - name: Restore pinned BcContainerHelper
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~\Documents\PowerShell\Modules\BcContainerHelper
+          key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }}
+          fail-on-cache-miss: true
+
       # Cache BcContainerHelper's artifact folder (its hardcoded default path) keyed by this entry's BC version
       - name: Cache BC sandbox artifacts
         id: bccache
@@ -147,7 +201,6 @@ jobs:
         if: steps.bccache.outputs.cache-hit != 'true'
         shell: pwsh
         run: |
-          Install-Module -Name BcContainerHelper -Force -AllowClobber -AllowPrerelease
           .\scripts\Download-BCSymbols.ps1 -Category nl2al -InstanceId "${{ matrix.entry }}"
 
       - name: Azure Login (OIDC) for LLM API certificate Key Vault
@@ -192,21 +245,26 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()
         with:
-          name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }}
+          name: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.entry }}
           path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
           retention-days: ${{ inputs.test-run && 1 || 30 }}
 
   summarize-results:
-    needs: evaluate-with-bcal
+    needs: [get-entries, evaluate-with-bcal]
+    if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.get-entries.outputs.entry-count != '0' }}
     uses: $/.github/workflows/summarize-results.yml
     permissions:
       contents: write
       id-token: write
     with:
-      results-dir: ${{ needs.evaluate-with-bcal.outputs.results-dir }}
+      results-dir: evaluation_results
       model: ${{ inputs.external-model || 'gpt-55-chat' }}
       agent: BCal-external-command
       mock: ${{ inputs.test-run || false }}
       category: nl2al
       skip-leaderboard: true
+      expected-total: ${{ fromJSON(needs.get-entries.outputs.entry-count) }}
+      expected-entries: ${{ needs.get-entries.outputs.entries }}
+      artifact-pattern: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-*
+      allow-unscored-results: true
     secrets: inherit

.github/workflows/get-entries.yml

@@ -24,6 +24,9 @@ on:
       entries:
         description: JSON array of dataset entries
         value: ${{ jobs.get-entries.outputs.entries }}
+      entry-count:
+        description: Number of dataset entries
+        value: ${{ jobs.get-entries.outputs.entry-count }}
       runner:
         description: GitHub Actions runner label to evaluate this category on
         value: ${{ jobs.get-entries.outputs.runner }}
@@ -39,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       entries: ${{ steps.get-entries.outputs.entries }}
+      entry-count: ${{ steps.get-entries.outputs.entry-count }}
       runner: ${{ steps.runtime-config.outputs.runner }}
       requires-container: ${{ steps.runtime-config.outputs.requires-container }}
       requires-repo: ${{ steps.runtime-config.outputs.requires-repo }}
@@ -58,7 +62,7 @@ jobs:
       - name: Get entries for matrix
         id: get-entries
         run: |
-          cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries"
+          cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries --count-output entry-count"
 
           if [[ "${{ inputs.modified-only }}" == "true" ]]; then
             cmd="$cmd --modified-only"

.github/workflows/summarize-results.yml

@@ -34,10 +34,31 @@ on:
         required: false
         type: string
         default: ""
+      expected-total:
+        description: "Expected unique matrix entry count; negative disables completeness enforcement"
+        required: false
+        type: number
+        default: -1
+      expected-entries:
+        description: "JSON array of expected matrix entry IDs; missing IDs become infrastructure failure results"
+        required: false
+        type: string
+        default: "[]"
+      artifact-pattern:
+        description: "Artifact glob to download"
+        required: false
+        type: string
+        default: "*"
+      allow-unscored-results:
+        description: "Allow complete result sets containing explicit unscored infrastructure errors"
+        required: false
+        type: boolean
+        default: false
 
 env:
   BCEVAL_RESULT_FILE: bceval_results.jsonl
   SUMMARY_OUTPUT_FILE: evaluation_summary.json
+  COMPLETENESS_OUTPUT_FILE: evaluation_completeness.json
 
 jobs:
   summarize-results:
@@ -58,33 +79,45 @@ jobs:
 
       - name: Download all evaluation results
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        continue-on-error: ${{ inputs.expected-total >= 0 }}
         with:
           path: ${{ inputs.results-dir }}/
+          pattern: ${{ inputs.artifact-pattern }}
           merge-multiple: true
 
       - name: Summarize evaluation results
-        run: uv run bcbench result summarize --category "${{ inputs.category }}" --result-dir "${{ inputs.results-dir }}" --bceval-output "${{ env.BCEVAL_RESULT_FILE }}" --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}" --git-ref "${{ inputs.git-ref || github.ref_name }}"
-
-      - name: Upload evaluation summary to artifacts
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-        with:
-          name: evaluation-summary
-          path: ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }}
-          if-no-files-found: error
-          retention-days: ${{ inputs.mock && 1 || 30 }}
+        id: summarize
+        env:
+          EXPECTED_ENTRIES_JSON: ${{ inputs.expected-entries }}
+        run: >-
+          uv run bcbench result summarize
+          --category "${{ inputs.category }}"
+          --result-dir "${{ inputs.results-dir }}"
+          --bceval-output "${{ env.BCEVAL_RESULT_FILE }}"
+          --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}"
+          --completeness-output "${{ env.COMPLETENESS_OUTPUT_FILE }}"
+          --expected-total "${{ inputs.expected-total }}"
+          --expected-entries-json "${EXPECTED_ENTRIES_JSON}"
+          --model "${{ inputs.model }}"
+          --agent "${{ inputs.agent }}"
+          --git-ref "${{ inputs.git-ref || github.ref_name }}"
+          --github-output
 
       - name: Azure Login with OIDC for bceval package
+        if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true'
         uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           allow-no-subscriptions: true
 
       - name: Resolve bceval evaluator config for category
+        if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true'
         id: bceval
         run: uv run bcbench category bceval-config --category "${{ inputs.category }}"
 
-      - name: Upload result using bceval
+      - name: Score and upload complete result using bceval
+        if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true'
         env:
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
@@ -96,11 +129,14 @@ jobs:
 
           uv tool install bc-eval==0.6.1 --python 3.12 --index "https://anything:${ADO_TOKEN}@dynamicssmb2.pkgs.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_packaging/BC_PythonFeed/pypi/simple/"
 
-          # Upload summary using bc-eval
           MODEL_TAG="${{ inputs.model }}"
           MODEL_TAG="${MODEL_TAG//./-}"
           BRANCH_TAG="${{ inputs.git-ref || github.ref_name }}"
           BRANCH_TAG="${BRANCH_TAG//./-}"
+          CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}")
+          if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]; then
+            CORE_SCORE_ARGS=()
+          fi
           bceval metrics calculate \
             --feature-name "BC-Bench" \
             --eval-suite-name "${{ inputs.category }}" \
@@ -110,13 +146,25 @@ jobs:
             --input-file "${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }}" \
             --evaluator-definitions "${{ github.workspace }}/evaluator/scores.py" \
             --evaluators "${{ steps.bceval.outputs.evaluators }}" \
-            --core-score "${{ steps.bceval.outputs.core_score }}" \
+            "${CORE_SCORE_ARGS[@]}" \
             --metric-definitions "${{ github.workspace }}/evaluator/metrics.py" \
             --metrics "bc_bench_metrics" \
             ${{ !inputs.mock && '--storage braintrust --storage kusto' || '' }}
 
+      - name: Upload evaluation summary to artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: evaluation-summary-${{ github.run_attempt }}
+          path: |
+            ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }}
+            ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }}
+            ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }}
+          if-no-files-found: error
+          retention-days: ${{ inputs.mock && 1 || 30 }}
+
       - name: Update leaderboard in a new branch
-        if: ${{ !inputs.mock && !inputs.skip-leaderboard }}
+        if: ${{ !inputs.mock && !inputs.skip-leaderboard && steps.summarize.outputs.complete == 'true' }}
         run: |
           git fetch origin main
 
@@ -150,3 +198,13 @@ jobs:
           git commit -m "Update leaderboard: ${{ inputs.agent }} (${{ inputs.model }}) - Run ${{ github.run_id }}"
 
           git push --set-upstream origin $branchName
+
+      - name: Require complete matrix
+        if: ${{ always() && inputs.expected-total >= 0 }}
+        run: |
+          completeness_file="${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }}"
+          if [[ ! -f "$completeness_file" ]]; then
+            echo "::error::Completeness metadata was not produced."
+            exit 1
+          fi
+          uv run bcbench result require-complete "$completeness_file"

evaluator/scores.py

@@ -1,7 +1,92 @@
 from __future__ import annotations
 
+import importlib
 from typing import Any
 
+_TIMEOUT_REASON = "Agent timed out before producing output"
+_TEST_PASSED_SCORE = "test_passed"
+_MISSING_TEST_PASSED_SCORE = "missing-test-passed-score"
+
+
+def _timeout_scores(scorer: object, expected: object, metadata: dict[str, Any]) -> list[object]:
+    if not isinstance(expected, dict):
+        raise TypeError("expected must be a dict containing an 'assertions' key")
+    assertions = expected.get("assertions")
+    if not isinstance(assertions, list):
+        raise TypeError("expected['assertions'] must be a list")
+
+    judged = []
+    for index, assertion in enumerate(assertions):
+        if not isinstance(assertion, dict):
+            raise TypeError(f"expected['assertions'][{index}] must be a dict")
+        if "text" not in assertion:
+            raise ValueError(f"expected['assertions'][{index}] must contain a 'text' key")
+        judged.append({**assertion, "pass": False, "reasoning": _TIMEOUT_REASON})
+
+    score_metadata = {"assertions": judged, "error": _TIMEOUT_REASON}
+    metadata["assertionResults"] = judged
+    score = importlib.import_module("autoevals").Score
+    rate = getattr(scorer, "_rate", None)
+    if not callable(rate):
+        raise TypeError("LM Checklist scorer does not provide a callable '_rate'")
+    return [
+        score(name="pass_rate", score=rate(judged), metadata=score_metadata),
+        score(name="critical_pass_rate", score=rate(judged, "critical"), metadata=score_metadata),
+        score(name="expected_pass_rate", score=rate(judged, "expected"), metadata=score_metadata),
+        score(name="aspirational_pass_rate", score=rate(judged, "aspirational"), metadata=score_metadata),
+        score(name="test_passed", score=0.0, metadata=score_metadata),
+    ]
+
+
+def _infrastructure_error_scores(metadata: dict[str, Any]) -> list[object]:
+    error = str(metadata.get("Error") or metadata.get("error_message") or "Infrastructure error")
+    score_metadata = {"error": error}
+    score = importlib.import_module("autoevals").Score
+    return [
+        score(name="pass_rate", score=None, metadata=score_metadata),
+        score(name="critical_pass_rate", score=None, metadata=score_metadata),
+        score(name="expected_pass_rate", score=None, metadata=score_metadata),
+        score(name="aspirational_pass_rate", score=None, metadata=score_metadata),
+        score(name="test_passed", score=None, metadata=score_metadata),
+    ]
+
+
+def _set_test_passed_core_score(result: object, metadata: dict[str, Any]) -> object:
+    if not isinstance(result, list):
+        raise TypeError("LM Checklist scorer must return a list of scores")
+    test_passed_scores = [score for score in result if getattr(score, "name", None) == _TEST_PASSED_SCORE]
+    if len(test_passed_scores) != 1 or getattr(test_passed_scores[0], "score", None) is None:
+        raise ValueError("Evaluated rows must produce exactly one non-null 'test_passed' score")
+    try:
+        metadata["core_score"] = float(test_passed_scores[0].score)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("The 'test_passed' score must be numeric") from exc
+    return result
+
+
+class LmChecklist:
+    def __new__(cls) -> object:
+        module = importlib.import_module("bc_eval.scorers.autoeval.lmchecklist")
+        scorer = module.LmChecklist()
+        run_eval_sync = scorer._run_eval_sync
+
+        def failure_aware_run_eval_sync(output: object, expected: object = None, **kwargs: object) -> object:
+            metadata = kwargs.get("metadata")
+            if isinstance(metadata, dict):
+                metadata["core_score_name"] = _TEST_PASSED_SCORE
+                if metadata.get("infrastructure_error") is True:
+                    metadata.pop("core_score", None)
+                    return _infrastructure_error_scores(metadata)
+                if metadata.get("timeout") is True:
+                    metadata["core_score"] = 0.0
+                    return _timeout_scores(scorer, expected, metadata)
+                metadata["core_score"] = _MISSING_TEST_PASSED_SCORE
+                return _set_test_passed_core_score(run_eval_sync(output, expected, **kwargs), metadata)
+            return run_eval_sync(output, expected, **kwargs)
+
+        scorer._run_eval_sync = failure_aware_run_eval_sync
+        return scorer
+
 
 class ResolutionRate:
     def __call__(self, *, metadata: dict[str, Any], **kwargs: object) -> bool: