From 2c6ddfc9a456d679ff915b288edcd84748dcd68f Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 29 Oct 2026 21:59:31 +0000 Subject: [PATCH 1/3] Run credentialed CI on copy-pr-bot's pull-request branches Fork pull_request runs get no secrets and no OIDC token, so every job that assumes the CI test role fails at "Configure AWS credentials". copy-pr-bot already pushes each vetted pull request to pull-request/, and pushes there get credentials. Testing, Databricks, Snowflake, GCS and Kumo Installation now also run on those pushes. A new _credentials-gate.yml job looks up the pull request: a same-repo pull request already ran these jobs on pull_request, so its copy skips them, and a fork's copy runs them. The gate also returns the pull request's merge base, which each change filter diffs against, so the copy tests the pull request's changes. Only pushes to main run the full suite. Testing's jobs that need no credentials skip on these pushes, since they already ran on pull_request. Co-Authored-By: Claude --- .github/workflows/_credentials-gate.yml | 52 ++++++++++++++++++++++++ .github/workflows/install.yml | 16 ++++++++ .github/workflows/testing.yml | 31 +++++++++++--- .github/workflows/testing_databricks.yml | 18 +++++++- .github/workflows/testing_gcs.yml | 16 ++++++++ .github/workflows/testing_snowflake.yml | 20 ++++++++- 6 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/_credentials-gate.yml diff --git a/.github/workflows/_credentials-gate.yml b/.github/workflows/_credentials-gate.yml new file mode 100644 index 000000000..0b8f54b34 --- /dev/null +++ b/.github/workflows/_credentials-gate.yml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Credentials gate + +# Decides whether a run gets the jobs that need cloud credentials. Fork pull +# requests get them only after copy-pr-bot pushes a vetted commit to +# pull-request/. Same-repo pull requests already run those jobs on the +# pull_request event, so their copies skip them. +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + run: + description: "'true' when this run should run the jobs that need cloud credentials." + value: ${{ jobs.gate.outputs.run }} + base: + description: "On pull-request/ pushes, the pull request's merge base. Empty otherwise." + value: ${{ jobs.gate.outputs.base }} + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + run: ${{ steps.gate.outputs.run }} + base: ${{ steps.gate.outputs.base }} + steps: + - name: Decide whether to run the credentialed jobs + id: gate + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + run=true + base='' + if [[ "$GITHUB_REF" == refs/heads/pull-request/* ]]; then + pr_url="repos/$GITHUB_REPOSITORY/pulls/${GITHUB_REF_NAME#pull-request/}" + if [[ "$(gh api "$pr_url" --jq .head.repo.full_name)" == "$GITHUB_REPOSITORY" ]]; then + run=false + fi + base_ref="$(gh api "$pr_url" --jq .base.ref)" + base="$(gh api "repos/$GITHUB_REPOSITORY/compare/$base_ref...$GITHUB_SHA" --jq .merge_base_commit.sha)" + fi + echo "run=$run" >> "$GITHUB_OUTPUT" + echo "base=$base" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 11fc6e8bf..99870a967 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**"] pull_request: merge_group: @@ -22,7 +23,15 @@ permissions: id-token: write jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + import: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: gha-kumo-workflow-tests runs-on: ubuntu-24.04 env: @@ -32,6 +41,12 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + # This enables us to skip the import test for ui-only changes. uv.lock # is deliberately excluded: a lock-only dependency bump is exactly what # this job's install + import smoke test needs to validate. @@ -39,6 +54,7 @@ jobs: id: ui-changes uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ui/** diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a7a2b15cd..ffbffc31c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -62,7 +63,7 @@ env: STORE_DURATIONS: ${{ inputs.store_durations == true }} # True for push-to-main runs and for manual dispatch/call with full_test=true # (unless store_durations is also set, which forces minimal mode). - RUN_FULL_TEST: ${{ github.event_name == 'push' || (inputs.full_test == true && inputs.store_durations != true) }} + RUN_FULL_TEST: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (inputs.full_test == true && inputs.store_durations != true) }} # True when pytest must run even if only_changed says no pytest-affecting # files changed. Currently only durations-recording runs set this — they # need the full matrix of tests to actually execute to produce a complete @@ -70,14 +71,22 @@ env: FORCE_RUN_TESTS: ${{ inputs.store_durations == true }} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + diskgraph-wheel: + needs: [credentials] + if: ${{ needs.credentials.outputs.run == 'true' || !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read uses: ./.github/workflows/_diskgraph-wheel.yml pytest: - needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + needs: [diskgraph-wheel, credentials] + if: ${{ !cancelled() && needs.credentials.outputs.run == 'true' }} runs-on: ubuntu-24.04 continue-on-error: true environment: @@ -168,6 +177,12 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Set up Python uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: @@ -188,6 +203,7 @@ jobs: id: non-pytest-changes uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ui/** CHANGELOG.md @@ -343,6 +359,7 @@ jobs: if-no-files-found: ignore diskgraph-rust: + if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -385,6 +402,7 @@ jobs: - run: cargo test --locked --workspace diskgraph-clippy: + if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -423,7 +441,7 @@ jobs: diskgraph-python: needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -504,7 +522,7 @@ jobs: # version. Run the DiskGraph and kumo-core suites at both ends instead. name: pyarrow-range (pyarrow${{ matrix.pyarrow }}) needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -581,7 +599,7 @@ jobs: # The repository's primary interpreter runs every package suite and # consumer check. needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && !startsWith(github.ref, 'refs/heads/pull-request/') }} runs-on: ubuntu-24.04 permissions: contents: read @@ -857,6 +875,7 @@ jobs: retention-days: 1 pql-public: + if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: diff --git a/.github/workflows/testing_databricks.yml b/.github/workflows/testing_databricks.yml index c4a17eb33..20fee0ce4 100644 --- a/.github/workflows/testing_databricks.yml +++ b/.github/workflows/testing_databricks.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -52,11 +53,19 @@ env: GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} STORE_DURATIONS: ${{ inputs.store_durations == true }} - RUN_FULL_TEST: ${{ github.event_name == 'push' || (inputs.full_test == true && inputs.store_durations != true) }} + RUN_FULL_TEST: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (inputs.full_test == true && inputs.store_durations != true) }} FORCE_RUN_TESTS: ${{ inputs.store_durations == true }} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + pytest: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. @@ -82,6 +91,12 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Set up Python uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: @@ -94,6 +109,7 @@ jobs: id: non-pytest-changes uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ui/** CHANGELOG.md diff --git a/.github/workflows/testing_gcs.yml b/.github/workflows/testing_gcs.yml index a49396bae..01cce9e80 100644 --- a/.github/workflows/testing_gcs.yml +++ b/.github/workflows/testing_gcs.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -38,7 +39,15 @@ env: GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + pytest-gcs: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. @@ -70,10 +79,17 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Get GCS related changes id: gcs-files-changed uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | .github/actions/setup/action.yml .github/workflows/testing_gcs.yml diff --git a/.github/workflows/testing_snowflake.yml b/.github/workflows/testing_snowflake.yml index e665ef75e..b717e42b2 100644 --- a/.github/workflows/testing_snowflake.yml +++ b/.github/workflows/testing_snowflake.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -38,7 +39,15 @@ env: GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + pytest: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. @@ -153,10 +162,17 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Get any data backend related changes id: data-backend-files-changed uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ${{ matrix.files_changed }} @@ -192,7 +208,7 @@ jobs: # coverage.py also has additional nice features that may be useful in the # future for better coverage reports. - name: Run minimal test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && github.event_name != 'push' && inputs.full_test != true && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.data-backend-files-changed.outputs.any_changed == 'true' && (github.event_name != 'push' || github.ref != 'refs/heads/main') && inputs.full_test != true && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' timeout-minutes: 120 run: | set -o pipefail @@ -208,7 +224,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Run full test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && (github.event_name == 'push' || inputs.full_test == true) && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.data-backend-files-changed.outputs.any_changed == 'true' && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || inputs.full_test == true) && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' timeout-minutes: 120 run: | set -o pipefail From 62d278c73f4ee7dc28b6241c894efa9453fa2665 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 29 Oct 2026 22:00:02 +0000 Subject: [PATCH 2/3] Skip credentialed jobs on fork pull_request runs A fork's pull_request run has no OIDC token, so its credentialed jobs failed at "Configure AWS credentials". The credentials gate now skips them on that event; they run on copy-pr-bot's pull-request/ push instead. The OpenAPI check, which has no such push path, skips on forks too. Co-Authored-By: Claude --- .github/workflows/_credentials-gate.yml | 14 +++++++++----- .github/workflows/ui-client-code-generate.yml | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_credentials-gate.yml b/.github/workflows/_credentials-gate.yml index 0b8f54b34..afdb2f1ad 100644 --- a/.github/workflows/_credentials-gate.yml +++ b/.github/workflows/_credentials-gate.yml @@ -4,10 +4,11 @@ name: Credentials gate -# Decides whether a run gets the jobs that need cloud credentials. Fork pull -# requests get them only after copy-pr-bot pushes a vetted commit to -# pull-request/. Same-repo pull requests already run those jobs on the -# pull_request event, so their copies skip them. +# Decides whether a run gets the jobs that need cloud credentials. A fork's +# pull_request run gets no secrets or OIDC token, so it skips them; they run +# once copy-pr-bot pushes a vetted commit to pull-request/. Same-repo +# pull requests already run those jobs on pull_request, so their copies skip +# them. on: # yamllint disable-line rule:truthy workflow_call: outputs: @@ -36,11 +37,14 @@ jobs: id: gate env: GH_TOKEN: ${{ github.token }} + PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} run: | set -euo pipefail run=true base='' - if [[ "$GITHUB_REF" == refs/heads/pull-request/* ]]; then + if [[ "$GITHUB_EVENT_NAME" == pull_request && "$PR_HEAD_REPO" != "$GITHUB_REPOSITORY" ]]; then + run=false + elif [[ "$GITHUB_REF" == refs/heads/pull-request/* ]]; then pr_url="repos/$GITHUB_REPOSITORY/pulls/${GITHUB_REF_NAME#pull-request/}" if [[ "$(gh api "$pr_url" --jq .head.repo.full_name)" == "$GITHUB_REPOSITORY" ]]; then run=false diff --git a/.github/workflows/ui-client-code-generate.yml b/.github/workflows/ui-client-code-generate.yml index 7f72736e9..5d65e01b8 100644 --- a/.github/workflows/ui-client-code-generate.yml +++ b/.github/workflows/ui-client-code-generate.yml @@ -44,7 +44,8 @@ jobs: environment: gha-kumo-workflow-tests name: Check OpenAPI Changes runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' + # A fork's pull_request run gets no OIDC token. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository permissions: id-token: write # Required for configure-aws-credentials OIDC. contents: read From 3411cb24427d8793f2daa1ab9576c1820df8a292 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 29 Oct 2026 22:00:14 +0000 Subject: [PATCH 3/3] Skip the coverage upload when no artifact name was set The artifact name is set by a step that runs only if every earlier step passed. After an earlier failure, the upload ran with an empty name and failed with "Input required and not supplied: path", adding a second error to the job. Co-Authored-By: Claude --- .github/workflows/testing.yml | 2 +- .github/workflows/testing_databricks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ffbffc31c..9e437380d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -328,7 +328,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Upload coverage artifact - if: always() + if: always() && env.ARTIFACT_NAME_SAFE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_SAFE }} diff --git a/.github/workflows/testing_databricks.yml b/.github/workflows/testing_databricks.yml index 20fee0ce4..5eb093b43 100644 --- a/.github/workflows/testing_databricks.yml +++ b/.github/workflows/testing_databricks.yml @@ -192,7 +192,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Upload coverage artifact - if: always() + if: always() && env.ARTIFACT_NAME_SAFE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_SAFE }}