diff --git a/.github/workflows/_credentials-gate.yml b/.github/workflows/_credentials-gate.yml new file mode 100644 index 000000000..afdb2f1ad --- /dev/null +++ b/.github/workflows/_credentials-gate.yml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Credentials gate + +# Decides whether a run gets the jobs that need cloud credentials. A fork's +# pull_request run gets no secrets or OIDC token, so it skips them; they run +# once copy-pr-bot pushes a vetted commit to pull-request/. Same-repo +# pull requests already run those jobs on pull_request, so their copies skip +# them. +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + run: + description: "'true' when this run should run the jobs that need cloud credentials." + value: ${{ jobs.gate.outputs.run }} + base: + description: "On pull-request/ pushes, the pull request's merge base. Empty otherwise." + value: ${{ jobs.gate.outputs.base }} + +permissions: + contents: read + +jobs: + gate: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + run: ${{ steps.gate.outputs.run }} + base: ${{ steps.gate.outputs.base }} + steps: + - name: Decide whether to run the credentialed jobs + id: gate + env: + GH_TOKEN: ${{ github.token }} + PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + run: | + set -euo pipefail + run=true + base='' + if [[ "$GITHUB_EVENT_NAME" == pull_request && "$PR_HEAD_REPO" != "$GITHUB_REPOSITORY" ]]; then + run=false + elif [[ "$GITHUB_REF" == refs/heads/pull-request/* ]]; then + pr_url="repos/$GITHUB_REPOSITORY/pulls/${GITHUB_REF_NAME#pull-request/}" + if [[ "$(gh api "$pr_url" --jq .head.repo.full_name)" == "$GITHUB_REPOSITORY" ]]; then + run=false + fi + base_ref="$(gh api "$pr_url" --jq .base.ref)" + base="$(gh api "repos/$GITHUB_REPOSITORY/compare/$base_ref...$GITHUB_SHA" --jq .merge_base_commit.sha)" + fi + echo "run=$run" >> "$GITHUB_OUTPUT" + echo "base=$base" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 11fc6e8bf..99870a967 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**"] pull_request: merge_group: @@ -22,7 +23,15 @@ permissions: id-token: write jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + import: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: gha-kumo-workflow-tests runs-on: ubuntu-24.04 env: @@ -32,6 +41,12 @@ jobs: - name: Checkout repository and submodules uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + # This enables us to skip the import test for ui-only changes. uv.lock # is deliberately excluded: a lock-only dependency bump is exactly what # this job's install + import smoke test needs to validate. @@ -39,6 +54,7 @@ jobs: id: ui-changes uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ui/** diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a7a2b15cd..9e437380d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -62,7 +63,7 @@ env: STORE_DURATIONS: ${{ inputs.store_durations == true }} # True for push-to-main runs and for manual dispatch/call with full_test=true # (unless store_durations is also set, which forces minimal mode). - RUN_FULL_TEST: ${{ github.event_name == 'push' || (inputs.full_test == true && inputs.store_durations != true) }} + RUN_FULL_TEST: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (inputs.full_test == true && inputs.store_durations != true) }} # True when pytest must run even if only_changed says no pytest-affecting # files changed. Currently only durations-recording runs set this — they # need the full matrix of tests to actually execute to produce a complete @@ -70,14 +71,22 @@ env: FORCE_RUN_TESTS: ${{ inputs.store_durations == true }} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + diskgraph-wheel: + needs: [credentials] + if: ${{ needs.credentials.outputs.run == 'true' || !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read uses: ./.github/workflows/_diskgraph-wheel.yml pytest: - needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + needs: [diskgraph-wheel, credentials] + if: ${{ !cancelled() && needs.credentials.outputs.run == 'true' }} runs-on: ubuntu-24.04 continue-on-error: true environment: @@ -168,6 +177,12 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Set up Python uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: @@ -188,6 +203,7 @@ jobs: id: non-pytest-changes uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ui/** CHANGELOG.md @@ -312,7 +328,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Upload coverage artifact - if: always() + if: always() && env.ARTIFACT_NAME_SAFE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_SAFE }} @@ -343,6 +359,7 @@ jobs: if-no-files-found: ignore diskgraph-rust: + if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -385,6 +402,7 @@ jobs: - run: cargo test --locked --workspace diskgraph-clippy: + if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -423,7 +441,7 @@ jobs: diskgraph-python: needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -504,7 +522,7 @@ jobs: # version. Run the DiskGraph and kumo-core suites at both ends instead. name: pyarrow-range (pyarrow${{ matrix.pyarrow }}) needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: @@ -581,7 +599,7 @@ jobs: # The repository's primary interpreter runs every package suite and # consumer check. needs: [diskgraph-wheel] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && !startsWith(github.ref, 'refs/heads/pull-request/') }} runs-on: ubuntu-24.04 permissions: contents: read @@ -857,6 +875,7 @@ jobs: retention-days: 1 pql-public: + if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: contents: read env: diff --git a/.github/workflows/testing_databricks.yml b/.github/workflows/testing_databricks.yml index c4a17eb33..5eb093b43 100644 --- a/.github/workflows/testing_databricks.yml +++ b/.github/workflows/testing_databricks.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -52,11 +53,19 @@ env: GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} STORE_DURATIONS: ${{ inputs.store_durations == true }} - RUN_FULL_TEST: ${{ github.event_name == 'push' || (inputs.full_test == true && inputs.store_durations != true) }} + RUN_FULL_TEST: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (inputs.full_test == true && inputs.store_durations != true) }} FORCE_RUN_TESTS: ${{ inputs.store_durations == true }} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + pytest: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. @@ -82,6 +91,12 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Set up Python uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: @@ -94,6 +109,7 @@ jobs: id: non-pytest-changes uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ui/** CHANGELOG.md @@ -176,7 +192,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Upload coverage artifact - if: always() + if: always() && env.ARTIFACT_NAME_SAFE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_SAFE }} diff --git a/.github/workflows/testing_gcs.yml b/.github/workflows/testing_gcs.yml index a49396bae..01cce9e80 100644 --- a/.github/workflows/testing_gcs.yml +++ b/.github/workflows/testing_gcs.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -38,7 +39,15 @@ env: GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + pytest-gcs: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. @@ -70,10 +79,17 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Get GCS related changes id: gcs-files-changed uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | .github/actions/setup/action.yml .github/workflows/testing_gcs.yml diff --git a/.github/workflows/testing_snowflake.yml b/.github/workflows/testing_snowflake.yml index e665ef75e..b717e42b2 100644 --- a/.github/workflows/testing_snowflake.yml +++ b/.github/workflows/testing_snowflake.yml @@ -8,6 +8,7 @@ on: # yamllint disable-line rule:truthy push: branches: - main + - "pull-request/[0-9]+" paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: @@ -38,7 +39,15 @@ env: GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} jobs: + credentials: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/_credentials-gate.yml + pytest: + needs: [credentials] + if: needs.credentials.outputs.run == 'true' environment: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. @@ -153,10 +162,17 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Fetch the pull request's merge base + if: needs.credentials.outputs.base != '' + env: + BASE: ${{ needs.credentials.outputs.base }} + run: git fetch --no-tags --depth=1 origin "$BASE" + - name: Get any data backend related changes id: data-backend-files-changed uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 with: + base_sha: ${{ needs.credentials.outputs.base }} files: | ${{ matrix.files_changed }} @@ -192,7 +208,7 @@ jobs: # coverage.py also has additional nice features that may be useful in the # future for better coverage reports. - name: Run minimal test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && github.event_name != 'push' && inputs.full_test != true && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.data-backend-files-changed.outputs.any_changed == 'true' && (github.event_name != 'push' || github.ref != 'refs/heads/main') && inputs.full_test != true && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' timeout-minutes: 120 run: | set -o pipefail @@ -208,7 +224,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Run full test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && (github.event_name == 'push' || inputs.full_test == true) && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.data-backend-files-changed.outputs.any_changed == 'true' && ((github.event_name == 'push' && github.ref == 'refs/heads/main') || inputs.full_test == true) && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' timeout-minutes: 120 run: | set -o pipefail diff --git a/.github/workflows/ui-client-code-generate.yml b/.github/workflows/ui-client-code-generate.yml index 7f72736e9..5d65e01b8 100644 --- a/.github/workflows/ui-client-code-generate.yml +++ b/.github/workflows/ui-client-code-generate.yml @@ -44,7 +44,8 @@ jobs: environment: gha-kumo-workflow-tests name: Check OpenAPI Changes runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' + # A fork's pull_request run gets no OIDC token. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository permissions: id-token: write # Required for configure-aws-credentials OIDC. contents: read