From 071b88dcce7a9785ee067535ad2cc7a3ed98c751 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Tue, 20 Oct 2026 13:22:05 +0000 Subject: [PATCH] ALCF CI: report results through a GitLab webhook, re-enable the bridge alcf.yml no longer keeps a runner polling the Aurora pipeline for hours. It starts the pipeline as before and sets the commit status `ALCF / LTS stack (Aurora)` to pending; alcf-report.yml (restored) sets the result when GitLab's pipeline webhook dispatches it. Jobs that fail without running their script (no Aurora runner online, stuck_pending_no_matching_runners; a PBS job the runner lost, no_updates_running; runner errors) say nothing about the change. Such a pipeline is retried once with a fresh pipeline, and reported as an error that names the infrastructure failure if that fails too, instead of as a test failure. The concurrency group moves to the job, so that a labeled same-repo PR no longer cancels its own run (supersedes #722). --- .github/workflows/alcf-report.yml | 195 ++++++++++++++++++++++++++++++ .github/workflows/alcf.yml | 107 ++++++++-------- 2 files changed, 244 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/alcf-report.yml diff --git a/.github/workflows/alcf-report.yml b/.github/workflows/alcf-report.yml new file mode 100644 index 00000000..39c975da --- /dev/null +++ b/.github/workflows/alcf-report.yml @@ -0,0 +1,195 @@ +name: ALCF CI report + +# Reports the result of an ALCF GitLab CI pipeline (see .gitlab-ci.yml) as a commit status +# on GitHub, without a runner having to wait for the pipeline: GitLab calls this workflow +# when a pipeline changes state. alcf.yml only starts the pipeline and sets the status to +# pending. +# +# - The GitLab project (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl) has a webhook +# for pipeline events that posts to this workflow's workflow_dispatch endpoint +# https://api.github.com/repos/JuliaGPU/oneAPI.jl/actions/workflows/alcf-report.yml/dispatches +# with the custom template +# {"ref":"main","inputs":{"pipeline_id":"{{object_attributes.id}}","status":"{{object_attributes.status}}"}} +# and an "Authorization: Bearer " custom header. The token is a fine-grained +# personal access token for this repository with only the "Actions: read and write" +# permission. It lives in the webhook configuration, so CI jobs never see it. When it +# expires GitLab disables the webhook after a few failed deliveries and statuses stay +# pending: renew the token, re-enable the webhook, and re-report by hand (below). +# - The inputs are only a hint. The commit and the result are read back from GitLab, so a +# forged or reordered dispatch cannot report anything but a pipeline's actual state. +# - Jobs can fail without running their script at all: no Aurora runner online for hours +# (stuck_pending_no_matching_runners), the runner losing a batch job that PBS never +# started (no_updates_running), or a runner error. Such a pipeline says nothing about the +# commit, so it is retried once with a fresh pipeline, and reported as an error that +# names the infrastructure if the retry fails the same way. +# - The GitLab instance requires an ALCF login, so the job logs are copied into the run, +# which the commit status links to. +# +# To (re-)report a pipeline by hand, e.g. after a lost webhook delivery: +# gh workflow run alcf-report.yml -f pipeline_id= + +on: + workflow_dispatch: + inputs: + pipeline_id: + description: ALCF GitLab pipeline id + required: true + type: string + status: + description: Pipeline status as sent by the GitLab webhook (leave empty) + required: false + type: string + +run-name: ALCF pipeline ${{ inputs.pipeline_id }} ${{ inputs.status }} + +permissions: + statuses: write + +jobs: + report: + name: Report pipeline result + # The webhook fires on every status change of a pipeline, but only the final ones are + # worth a runner. A run without the status hint is a manual one and always reports. + if: >- + inputs.status == '' || + contains(fromJSON('["success", "failed", "canceled", "skipped"]'), inputs.status) + runs-on: ubuntu-latest + timeout-minutes: 15 + # One at a time per pipeline. Each run reports the state it reads from GitLab, so it + # does not matter which of several queued runs gets to go. + concurrency: + group: alcf-report-${{ inputs.pipeline_id }} + env: + GITLAB_HOST: gitlab-ci.alcf.anl.gov + GITLAB_PROJECT: mschanen/oneAPI-jl + API: https://gitlab-ci.alcf.anl.gov/api/v4/projects/238 + GITLAB_TOKEN: ${{ secrets.ALCF_GITLAB_TOKEN }} + GITLAB_TRIGGER_TOKEN: ${{ secrets.ALCF_GITLAB_TRIGGER_TOKEN }} + GH_TOKEN: ${{ github.token }} + PIPELINE_ID: ${{ inputs.pipeline_id }} + CONTEXT: ALCF / LTS stack (Aurora) + # GitLab job failure reasons that mean the job never got to run its script + INFRA_REASONS: >- + ["stuck_pending_no_matching_runners", "no_updates_running", "runner_system_failure", + "stuck_or_timeout_failure", "scheduler_failure", "runner_unsupported"] + steps: + # This run is attached to the head of main, not to the commit the pipeline tested, + # so it only fails if reporting itself fails. The pipeline's result goes into the + # commit status. + - name: Set commit status + run: | + case "$PIPELINE_ID" in + ''|*[!0-9]*) echo "::error::pipeline_id must be a number"; exit 1 ;; + esac + api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } + + PIPELINE="" + for i in $(seq 1 5); do + PIPELINE=$(api "$API/pipelines/$PIPELINE_ID") + if echo "$PIPELINE" | jq -e '.sha and .status' >/dev/null 2>&1; then break; fi + PIPELINE="" + sleep 15 + done + if [ -z "$PIPELINE" ]; then + echo "::error::could not read GitLab pipeline $PIPELINE_ID" + exit 1 + fi + SHA=$(echo "$PIPELINE" | jq -r '.sha') + REF=$(echo "$PIPELINE" | jq -r '.ref') + STATUS=$(echo "$PIPELINE" | jq -r '.status') + URL="https://${GITLAB_HOST}/${GITLAB_PROJECT}/-/pipelines/$PIPELINE_ID" + RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + echo "pipeline: $URL" + echo "commit: $SHA ($REF)" + echo "status: $STATUS" + + FAILED="" + INFRA=false + TARGET="$RUN_URL" + case "$STATUS" in + success) + STATE=success; DESC="Pipeline $PIPELINE_ID passed" ;; + failed) + JOBS=$(api "$API/pipelines/$PIPELINE_ID/jobs?per_page=100") + FAILED=$(echo "$JOBS" | jq -r '[.[] | select(.status=="failed" and (.allow_failure | not)) + | "\(.name) (\(.failure_reason))"] | join(", ")' 2>/dev/null || true) + INFRA=$(echo "$JOBS" | jq -r --argjson infra "$INFRA_REASONS" \ + '[.[] | select(.status=="failed" and (.allow_failure | not))] + | length > 0 and all(.failure_reason as $r | any($infra[]; . == $r))' 2>/dev/null || echo false) + if [ "$INFRA" = true ]; then + REASONS=$(echo "$JOBS" | jq -r '[.[] | select(.status=="failed") | .failure_reason] | unique | join(", ")') + STATE=error; DESC="ALCF infrastructure failure, not a test failure ($REASONS)" + else + STATE=failure; DESC="Pipeline $PIPELINE_ID failed${FAILED:+: $FAILED}" + fi ;; + canceled) + STATE=error; DESC="Pipeline $PIPELINE_ID canceled" ;; + skipped) + # [skip ci] commits: there is nothing to report, and GitHub skipped alcf.yml too. + echo "pipeline $PIPELINE_ID was skipped; leaving the commit status alone" + exit 0 ;; + *) + STATE=pending; DESC="Pipeline $PIPELINE_ID $STATUS"; TARGET="$URL" ;; + esac + if [ "$STATE" != pending ]; then echo "FINISHED=true" >> "$GITHUB_ENV"; fi + + { + echo "[ALCF GitLab pipeline $PIPELINE_ID]($URL) for $SHA: **$STATUS**" + if [ -n "$FAILED" ]; then echo; echo "Failed jobs: $FAILED"; fi + } >> "$GITHUB_STEP_SUMMARY" + + # A commit can have several pipelines (re-runs). Only the newest one speaks for + # the commit, so that a late report of an old pipeline does not replace it. + NEWEST=$(api "$API/pipelines?sha=$SHA&order_by=id&sort=desc&per_page=1" \ + | jq -r '.[0].id // empty' 2>/dev/null || true) + if [ -n "$NEWEST" ] && [ "$NEWEST" != "$PIPELINE_ID" ]; then + echo "::notice::pipeline $PIPELINE_ID was superseded by $NEWEST for $SHA; leaving the commit status alone" + exit 0 + fi + + # Retry an infrastructure failure once: only for the first pipeline of the commit, + # and only while the commit is still the tip of its ref (the trigger runs the tip). + if [ "$INFRA" = true ] && [ -n "$GITLAB_TRIGGER_TOKEN" ]; then + COUNT=$(api "$API/pipelines?sha=$SHA&per_page=100" | jq -r 'length' 2>/dev/null || echo 0) + TIP=$(api "$API/repository/branches/$(jq -rn --arg r "$REF" '$r | @uri')" \ + | jq -r '.commit.id // empty' 2>/dev/null || true) + if [ "$COUNT" = 1 ] && [ "$TIP" = "$SHA" ]; then + RESP=$(curl -sS --max-time 30 -X POST "$API/trigger/pipeline" \ + --form-string "token=${GITLAB_TRIGGER_TOKEN}" --form-string "ref=$REF" || true) + RETRY=$(echo "$RESP" | jq -r --arg sha "$SHA" 'select(.sha == $sha) | .id // empty' 2>/dev/null || true) + if [ -n "$RETRY" ]; then + STATE=pending + DESC="ALCF infrastructure failure in pipeline $PIPELINE_ID, retrying as $RETRY" + TARGET="https://${GITLAB_HOST}/${GITLAB_PROJECT}/-/pipelines/$RETRY" + else + echo "::warning::could not trigger a retry pipeline for $SHA: $RESP" + fi + else + echo "not retrying: $COUNT pipeline(s) for $SHA, $REF is at ${TIP:-unknown}" + fi + fi + + if [ "$STATE" = failure ] || [ "$STATE" = error ]; then + { + echo + echo "To run it again, re-run the [ALCF CI check of that commit]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$SHA/checks); for a pull request from a fork, re-apply the \`alcf-ci\` label." + } >> "$GITHUB_STEP_SUMMARY" + fi + + gh api -X POST "repos/$GITHUB_REPOSITORY/statuses/$SHA" \ + -f state="$STATE" -f context="$CONTEXT" \ + -f description="${DESC:0:140}" -f target_url="$TARGET" >/dev/null + echo "set '$CONTEXT' to $STATE: $DESC" + + - name: Publish GitLab job logs + if: env.FINISHED == 'true' + run: | + api() { curl -sS --max-time 60 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } + api "$API/pipelines/$PIPELINE_ID/jobs?per_page=100" \ + | jq -r '.[] | "\(.id)|\(.name)|\(.status)"' \ + | while IFS='|' read -r jid jname jstatus; do + echo "::group::${jname} — ${jstatus}" + api "$API/jobs/$jid/trace" + echo "" + echo "::endgroup::" + done diff --git a/.github/workflows/alcf.yml b/.github/workflows/alcf.yml index 43584f5d..7234a1e6 100644 --- a/.github/workflows/alcf.yml +++ b/.github/workflows/alcf.yml @@ -2,11 +2,12 @@ name: ALCF CI # Bridges GitHub to ALCF GitLab CI (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl), # which pull-mirrors this repository and runs the LTS-stack test suite on Aurora (see -# .gitlab-ci.yml). This workflow makes the GitLab pipeline show up as a PR check: +# .gitlab-ci.yml). This workflow only starts the GitLab pipeline and sets the commit status +# `ALCF / LTS stack (Aurora)` to pending; alcf-report.yml, which GitLab calls when the +# pipeline finishes, sets the result. No runner waits for the pipeline. # # - push to main / same-repo PRs: the mirror already carries the branch; force a mirror -# sync (instead of waiting for the ~30 min schedule), then poll the pipeline for the -# head SHA and adopt its result. +# sync (instead of waiting for the ~30 min schedule), which creates the pipeline. # - fork PRs: never run automatically — GitLab CI jobs execute as an ALCF user on # Aurora, so running fork code is opt-in. A maintainer applies the 'alcf-ci' label, # which pushes the PR head to GitLab as branch gh-pr- and triggers a pipeline. @@ -26,29 +27,28 @@ on: pull_request_target: types: [labeled] -concurrency: - group: alcf-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - permissions: contents: read + statuses: write # the pending status + actions: write # re-dispatching alcf-report.yml for an already finished pipeline jobs: aurora: name: LTS stack (ALCF GitLab CI, Aurora) runs-on: ubuntu-latest - # PBS queue wait + 1 h walltime + build stage; generous but below the 6 h runner cap - timeout-minutes: 330 - # Temporarily disabled: ALCF GitLab jobs fail intermittently without producing any - # output, i.e. before they get to run. Drop the leading `false &&` to re-enable. + timeout-minutes: 30 + # On the job rather than the workflow, so that runs whose job is skipped (e.g. a same-repo + # PR being labeled) don't cancel the one actually starting the pipeline. + concurrency: + group: alcf-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true if: >- - false && ( github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || (github.event_name == 'pull_request_target' && github.event.label.name == 'alcf-ci' && - github.event.pull_request.head.repo.full_name != github.repository)) + github.event.pull_request.head.repo.full_name != github.repository) env: GITLAB_HOST: gitlab-ci.alcf.anl.gov GITLAB_PROJECT: mschanen/oneAPI-jl @@ -56,6 +56,8 @@ jobs: GITLAB_TOKEN: ${{ secrets.ALCF_GITLAB_TOKEN }} GITLAB_TRIGGER_TOKEN: ${{ secrets.ALCF_GITLAB_TRIGGER_TOKEN }} HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + CONTEXT: ALCF / LTS stack (Aurora) + GH_TOKEN: ${{ github.token }} steps: - name: Push fork PR head to GitLab (label-gated) if: github.event_name == 'pull_request_target' @@ -89,7 +91,7 @@ jobs: if: github.event_name != 'pull_request_target' run: | # Best-effort: if this fails, the scheduled mirror sync still triggers the - # pipeline, just later; the polling step below tolerates the delay. + # pipeline, just later, and the webhook reports it whenever it finishes. curl -sS -X POST -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$API/mirror/pull" \ || echo "::warning::mirror sync request failed; relying on scheduled sync" @@ -107,7 +109,7 @@ jobs: api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } # Only replace a pipeline that finished unsuccessfully. Anything else (none - # yet, still active, succeeded, lookup failed) is left to the wait step below. + # yet, still active, succeeded, lookup failed) is left to the steps below. STATUS=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \ | jq -r '.[0].status // empty' 2>/dev/null || true) case "$STATUS" in @@ -133,57 +135,46 @@ jobs: fi echo "triggered fresh pipeline $(echo "$RESP" | jq -r .id) for $HEAD_SHA" - - name: Wait for pipeline and adopt its result + - name: Set pending status run: | - # Transient API failures (network, rate limiting) must not fail the check: - # api() degrades to empty output, and the loops treat that as "try again". + # Transient API failures must not fail the check: api() degrades to empty + # output, which is treated as "no pipeline yet". api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } + newest() { api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1"; } - echo "waiting for a GitLab pipeline for $HEAD_SHA" + # The mirror sync is asynchronous. Wait a little for the pipeline so that the + # status can link to it; if it takes longer, the webhook still reports it. ID="" - for i in $(seq 1 80); do # up to 40 min for mirror sync + pipeline creation - ID=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \ - | jq -r '.[0].id // empty' 2>/dev/null || true) + for i in $(seq 1 20); do + ID=$(newest | jq -r '.[0].id // empty' 2>/dev/null || true) if [ -n "$ID" ]; then break; fi sleep 30 done - if [ -z "$ID" ]; then - echo "::error::no GitLab pipeline appeared for $HEAD_SHA within 40 min (mirror not synced, or ref filtered by workflow rules)" - exit 1 + if [ -n "$ID" ]; then + URL="https://${GITLAB_HOST}/${GITLAB_PROJECT}/-/pipelines/$ID" + DESC="Pipeline $ID on Aurora" + echo "pipeline: $URL" + echo "[ALCF GitLab pipeline $ID]($URL)" >> "$GITHUB_STEP_SUMMARY" + if [ "$(api "$API/pipelines/$ID" | jq -r '.status // empty' 2>/dev/null)" = skipped ]; then + echo "pipeline $ID was skipped; not setting a status" + exit 0 + fi + else + URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + DESC="Waiting for the ALCF GitLab mirror" + echo "::warning::no GitLab pipeline for $HEAD_SHA yet; it is reported when it finishes" fi - URL="https://${GITLAB_HOST}/${GITLAB_PROJECT}/-/pipelines/$ID" - echo "pipeline: $URL" - echo "PIPELINE_ID=$ID" >> "$GITHUB_ENV" - echo "[ALCF GitLab pipeline $ID]($URL)" >> "$GITHUB_STEP_SUMMARY" - prev="" - while true; do + gh api -X POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state=pending -f context="$CONTEXT" -f description="$DESC" -f target_url="$URL" >/dev/null + + # A pipeline that finished before the status above was set (an existing branch + # opened as a PR, a re-run, a fast failure) has been reported already, and that + # report was just overwritten. Report it again. + if [ -n "$ID" ]; then STATUS=$(api "$API/pipelines/$ID" | jq -r '.status // empty' 2>/dev/null || true) - if [ -n "$STATUS" ] && [ "$STATUS" != "$prev" ]; then echo "status: $STATUS"; prev="$STATUS"; fi case "$STATUS" in - success) - exit 0 ;; - failed|canceled|skipped) - echo "failed jobs:" - api "$API/pipelines/$ID/jobs?per_page=100" \ - | jq -r '.[] | select(.status=="failed") | " \(.name): \(.web_url)"' 2>/dev/null \ - | tee -a "$GITHUB_STEP_SUMMARY" || true - echo "::error::GitLab pipeline $STATUS: $URL" - exit 1 ;; + success|failed|canceled) + echo "pipeline $ID already $STATUS; reporting it" + gh workflow run alcf-report.yml -R "$GITHUB_REPOSITORY" -f pipeline_id="$ID" ;; esac - sleep 60 - done - - # The GitLab instance requires an ALCF login, so mirror the job logs into this - # run where any contributor can read them. - - name: Publish GitLab job logs - if: always() && env.PIPELINE_ID != '' - run: | - api() { curl -sS --max-time 60 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } - api "$API/pipelines/$PIPELINE_ID/jobs?per_page=100" \ - | jq -r '.[] | "\(.id)|\(.name)|\(.status)"' \ - | while IFS='|' read -r jid jname jstatus; do - echo "::group::${jname} — ${jstatus}" - api "$API/jobs/$jid/trace" - echo "" - echo "::endgroup::" - done + fi