diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index db23ece3410..bad051f4642 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -8,3 +8,10 @@ self-hosted-runner: labels: - linux-amd64-cpu8 - linux-amd64-gpu-l4-latest-1 + +# GitHub supports queued concurrency runs, but the latest actionlint release +# does not yet recognize the concurrency.queue key. +paths: + ".github/workflows/ci-workflow-health.yml": + ignore: + - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/workflows/ci-workflow-health.yml b/.github/workflows/ci-workflow-health.yml new file mode 100644 index 00000000000..0a17564b20f --- /dev/null +++ b/.github/workflows/ci-workflow-health.yml @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +name: "CI: Track workflow health" + +on: + workflow_run: + # These names must exactly match the top-level workflows. Keep this list in + # sync when an unattended CI workflow is added or renamed. + workflows: + - "CI" + - "CI: Coverage" + - "CI: Nightly optional-deps" + - "CI: pixi run test (source build)" + - "Security Suite (Pulse + CodeQL)" + - "Static Analysis: Bandit Scan" + types: + - completed + branches: + - main + +# Serialize updates to one incident while allowing scheduled and main-push +# health to be tracked independently for workflows that support both. Queue all +# events; the script also rejects stale completions because dispatch order is +# not guaranteed. +concurrency: + group: >- + ${{ github.workflow }}-${{ github.event.workflow_run.workflow_id }}-${{ + github.event.workflow_run.event }} + queue: max + +permissions: {} + +jobs: + update-incident: + name: Update CI health issue + if: >- + github.repository == 'NVIDIA/cuda-python' && + ( + github.event.workflow_run.event == 'schedule' || + ( + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == github.event.repository.default_branch + ) + ) && + ( + github.event.workflow_run.conclusion == 'success' || + contains( + fromJSON('["action_required","failure","stale","startup_failure","timed_out"]'), + github.event.workflow_run.conclusion + ) || + ( + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion == 'cancelled' + ) + ) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: read + issues: write + steps: + - name: Open, update, or close the workflow incident + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const run = context.payload.workflow_run; + const scheduled = run.event === "schedule"; + const scopeKey = scheduled ? "schedule" : "push-main"; + const scopeLabel = scheduled ? "scheduled runs" : "on main"; + const trackerMarker = + ``; + const eventMarker = + ``; + const incidentLabel = "ci-workflow-health"; + const title = `[CI failure] ${run.name} ${scopeLabel}`; + const repositoryUrl = + `${process.env.GITHUB_SERVER_URL}/${context.repo.owner}/${context.repo.repo}`; + const runUrl = `${run.html_url}/attempts/${run.run_attempt}`; + const shortSha = run.head_sha.slice(0, 7); + const report = [ + `[${run.name} run #${run.run_number}, attempt ${run.run_attempt}](${runUrl}) ` + + `concluded with **${run.conclusion}**.`, + "", + `- Trigger: \`${run.event}\``, + `- Branch: \`${run.head_branch}\``, + `- Commit: [\`${shortSha}\`](${repositoryUrl}/commit/${run.head_sha})`, + `- Completed: \`${run.updated_at}\``, + ].join("\n"); + + const {data: {workflow_runs: completedRuns}} = + await github.rest.actions.listWorkflowRuns({ + ...context.repo, + workflow_id: run.workflow_id, + branch: run.head_branch, + event: run.event, + status: "completed", + per_page: 100, + }); + const isNewer = (candidate, reference) => + candidate.run_number > reference.run_number || + ( + candidate.run_number === reference.run_number && + candidate.run_attempt > reference.run_attempt + ); + const unhealthyConclusions = new Set([ + "action_required", + "failure", + "stale", + "startup_failure", + "timed_out", + ]); + const actionableRuns = completedRuns.filter((candidate) => + candidate.conclusion === "success" || + unhealthyConclusions.has(candidate.conclusion) || + ( + candidate.event === "schedule" && + candidate.conclusion === "cancelled" + ), + ); + const latestRun = actionableRuns.reduce( + (latest, candidate) => isNewer(candidate, latest) ? candidate : latest, + run, + ); + if (isNewer(latestRun, run)) { + core.info( + `Ignoring stale completion for run #${run.run_number}, attempt ` + + `${run.run_attempt}; run #${latestRun.run_number}, attempt ` + + `${latestRun.run_attempt} has already completed.`, + ); + return; + } + + const openIssues = await github.paginate( + github.rest.issues.listForRepo, + { + ...context.repo, + state: "open", + per_page: 100, + }, + ); + const incidents = openIssues + .filter((issue) => + !issue.pull_request && + typeof issue.body === "string" && + issue.body.includes(trackerMarker), + ) + .sort((left, right) => left.number - right.number); + + if (incidents.length > 1) { + core.warning( + `Found ${incidents.length} open incidents for ${run.name} (${scopeKey}).`, + ); + } + + for (const incident of incidents) { + const hasIncidentLabel = incident.labels.some((label) => + (typeof label === "string" ? label : label.name) === incidentLabel, + ); + if (!hasIncidentLabel) { + await github.rest.issues.addLabels({ + ...context.repo, + issue_number: incident.number, + labels: [incidentLabel], + }); + } + } + + async function eventAlreadyRecorded(issue) { + if (issue.body.includes(eventMarker)) { + return true; + } + const comments = await github.paginate( + github.rest.issues.listComments, + { + ...context.repo, + issue_number: issue.number, + per_page: 100, + }, + ); + return comments.some((comment) => + typeof comment.body === "string" && + comment.body.includes(eventMarker), + ); + } + + if (run.conclusion === "success") { + if (incidents.length === 0) { + core.info(`No open incident for ${run.name} (${scopeKey}).`); + return; + } + + for (const incident of incidents) { + if (!(await eventAlreadyRecorded(incident))) { + await github.rest.issues.createComment({ + ...context.repo, + issue_number: incident.number, + body: [ + eventMarker, + "### Recovered", + "", + report, + "", + "Closing this incident automatically.", + ].join("\n"), + }); + } + await github.rest.issues.update({ + ...context.repo, + issue_number: incident.number, + state: "closed", + state_reason: "completed", + }); + } + return; + } + + if (incidents.length === 0) { + await github.rest.issues.create({ + ...context.repo, + title, + labels: ["bug", "CI/CD", "triage", incidentLabel], + body: [ + trackerMarker, + eventMarker, + "This issue tracks an unhealthy unattended CI workflow.", + "Subsequent failures are recorded in comments; a successful run in the", + "same trigger scope closes the issue automatically.", + "", + "### First unhealthy run", + "", + report, + "", + "_Created automatically by the CI workflow-health monitor._", + ].join("\n"), + }); + return; + } + + const incident = incidents[0]; + if (await eventAlreadyRecorded(incident)) { + core.info( + `Run ${run.id}, attempt ${run.run_attempt} is already recorded in ` + + `issue #${incident.number}.`, + ); + return; + } + await github.rest.issues.createComment({ + ...context.repo, + issue_number: incident.number, + body: [ + eventMarker, + "### Another unhealthy run", + "", + report, + ].join("\n"), + });