diff --git a/.github/scripts/approved-pr.test.cjs b/.github/scripts/approved-pr.test.cjs new file mode 100644 index 000000000..aa8a2c3ea --- /dev/null +++ b/.github/scripts/approved-pr.test.cjs @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { readCopiedPullRequest, setCopiedPullRequestOutputs } = require('./copied-pr.cjs'); +const { reportOpenAPI } = require('./openapi-pr.cjs'); + +function fixture({ changed = true, comments = [], fork = true } = {}) { + const context = { + eventName: 'push', ref: 'refs/heads/pull-request/42', sha: 'a'.repeat(40), + repo: { owner: 'example', repo: 'engine' }, serverUrl: 'https://github.com', runId: 123, + }; + const pr = { + number: 42, state: 'open', + head: { sha: context.sha, repo: { full_name: fork ? 'contributor/engine' : 'example/engine' } }, + base: { sha: 'b'.repeat(40), repo: { full_name: 'example/engine' } }, + }; + const writes = []; + const github = { + rest: { + pulls: { get: async (args) => { assert.equal(args.pull_number, 42); return { data: pr }; } }, + issues: { + listComments: () => {}, + createComment: async (args) => writes.push({ kind: 'create', ...args }), + updateComment: async (args) => writes.push({ kind: 'update', ...args }), + }, + }, + paginate: async () => comments, + }; + return { github, context, changed, pr, writes }; +} + +test('only current copied commits of open PRs targeting this repository are accepted', async () => { + const f = fixture(); + assert.equal(await readCopiedPullRequest(f), f.pr); + for (const [field, value] of [['eventName', 'pull_request'], ['ref', 'refs/heads/main'], ['ref', 'refs/heads/pull-request/42/extra']]) { + await assert.rejects(readCopiedPullRequest({ ...f, context: { ...f.context, [field]: value } }), /copy-pr-bot/); + } + f.pr.head.sha = 'c'.repeat(40); + await assert.rejects(readCopiedPullRequest(f), /stale/); + f.pr.head.sha = f.context.sha; + f.pr.state = 'closed'; + await assert.rejects(readCopiedPullRequest(f), /open pull request/); + f.pr.state = 'open'; + f.pr.base.repo.full_name = 'other/engine'; + await assert.rejects(readCopiedPullRequest(f), /this repository/); +}); + +test('merge-base selection retains earlier PR changes and excludes independent base changes', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'copied-pr-base-')); + const previous = process.cwd(); + const git = (...args) => execFileSync('git', ['-c', 'user.name=CI Test', '-c', 'user.email=ci@example.test', + '-c', 'commit.gpgsign=false', '-c', 'core.hooksPath=/dev/null', ...args], { cwd: dir, encoding: 'utf8' }).trim(); + const commit = (file) => { + fs.writeFileSync(path.join(dir, file), file); + git('add', file); + git('commit', '-qm', file); + return git('rev-parse', 'HEAD'); + }; + try { + git('init', '-qb', 'main'); + const common = commit('base.txt'); + git('checkout', '-qb', 'feature'); + commit('service.py'); + const head = commit('README.md'); + git('checkout', '-q', 'main'); + const base = commit('unrelated-gcp.py'); + git('checkout', '-q', 'feature'); + const f = fixture(); + f.context.sha = f.pr.head.sha = head; + f.pr.base.sha = base; + const outputs = {}; + process.chdir(dir); + await setCopiedPullRequestOutputs({ ...f, core: { setOutput: (key, value) => { outputs[key] = value; } } }); + assert.deepEqual(outputs, { number: '42', head, base: common }); + assert.deepEqual(git('diff', '--name-only', `${outputs.base}..${head}`).split('\n'), ['README.md', 'service.py']); + } finally { + process.chdir(previous); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('fork warning links the analyzed SHA, schema patch, and local generation command', async () => { + const f = fixture(); + await reportOpenAPI(f); + assert.equal(f.writes.length, 1); + assert.equal(f.writes[0].issue_number, 42); + assert.match(f.writes[0].body, //); + assert.match(f.writes[0].body, /make generate-ui-client-code/); + assert.match(f.writes[0].body, /push.*fork/); + assert.match(f.writes[0].body, /actions\/runs\/123/); + assert.match(f.writes[0].body, /View the schema patch in the workflow run/); + assert.ok(f.writes[0].body.includes(f.context.sha)); + assert.doesNotMatch(f.writes[0].body, /trigger-ui-codegen/); +}); + +test('same-repository warning preserves the authorized code-generation command', async () => { + const f = fixture({ fork: false }); + await reportOpenAPI(f); + assert.match(f.writes[0].body, /trigger-ui-codegen/); +}); + +test('an existing bot warning is updated when schemas become current', async () => { + const f = fixture({ changed: false, comments: [ + { id: 1, user: { login: 'contributor' }, body: ' spoof' }, + { id: 2, user: { login: 'github-actions[bot]' }, body: '⚠️ **OpenAPI spec changes detected in this PR**' }, + ] }); + await reportOpenAPI(f); + assert.equal(f.writes[0].kind, 'update'); + assert.equal(f.writes[0].comment_id, 2); + assert.match(f.writes[0].body, /up to date/); +}); + +test('a repeated warning updates the bot comment instead of creating duplicates', async () => { + const f = fixture({ comments: [{ id: 7, user: { login: 'github-actions[bot]' }, body: ' old' }] }); + await reportOpenAPI(f); + assert.equal(f.writes[0].kind, 'update'); + assert.equal(f.writes[0].comment_id, 7); +}); + +test('current schemas do not create an unsolicited success comment', async () => { + const f = fixture({ changed: false }); + await reportOpenAPI(f); + assert.equal(f.writes.length, 0); +}); + +test('a PR update during reporting prevents a stale write', async () => { + const f = fixture(); + f.github.paginate = async () => { + f.pr.head.sha = 'c'.repeat(40); + return []; + }; + await assert.rejects(reportOpenAPI(f), /stale/); + assert.equal(f.writes.length, 0); +}); diff --git a/.github/scripts/copied-pr.cjs b/.github/scripts/copied-pr.cjs new file mode 100644 index 000000000..3099660f7 --- /dev/null +++ b/.github/scripts/copied-pr.cjs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { execFileSync } = require('node:child_process'); + +async function readCopiedPullRequest({ github, context }) { + const match = /^refs\/heads\/pull-request\/([1-9][0-9]*)$/.exec(context.ref); + if (context.eventName !== 'push' || !match) { + throw new Error('Expected a copy-pr-bot branch push'); + } + const { data: pr } = await github.rest.pulls.get({ + ...context.repo, + pull_number: Number(match[1]), + }); + if (pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}` || pr.state !== 'open') { + throw new Error('Expected an open pull request targeting this repository'); + } + if (pr.head.sha !== context.sha) { + throw new Error('Copied PR is stale; approve the current SHA'); + } + return pr; +} + +async function setCopiedPullRequestOutputs({ github, context, core }) { + const pr = await readCopiedPullRequest({ github, context }); + // Diff the complete PR, excluding changes made only on the base branch. + const base = execFileSync('git', ['merge-base', pr.base.sha, context.sha], { encoding: 'utf8' }).trim(); + core.setOutput('number', String(pr.number)); + core.setOutput('head', pr.head.sha); + core.setOutput('base', base); +} + +module.exports = { readCopiedPullRequest, setCopiedPullRequestOutputs }; diff --git a/.github/scripts/openapi-pr.cjs b/.github/scripts/openapi-pr.cjs new file mode 100644 index 000000000..1689d5086 --- /dev/null +++ b/.github/scripts/openapi-pr.cjs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { readCopiedPullRequest } = require('./copied-pr.cjs'); + +const MARKER = ''; +const LEGACY_HEADING = '⚠️ **OpenAPI spec changes detected in this PR**'; + +async function reportOpenAPI({ github, context, changed }) { + const pr = await readCopiedPullRequest({ github, context }); + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: pr.number, + per_page: 100, + }); + const existing = comments.find((comment) => comment.user?.login === 'github-actions[bot]' && + (comment.body?.startsWith(MARKER) || comment.body?.startsWith(LEGACY_HEADING))); + if (!changed && !existing) return; + + const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const instructions = pr.head.repo.full_name === pr.base.repo.full_name + ? 'Run `make generate-ui-client-code` locally, or comment `/trigger-ui-codegen` if you have repository write access.' + : 'Run `make generate-ui-client-code` locally and push the regenerated files to your fork.'; + const body = [ + MARKER, + changed ? '⚠️ **OpenAPI schema regeneration needed**' : '✅ **OpenAPI schemas are up to date**', + '', + `Analyzed commit: \`${context.sha}\`.`, + '', + changed ? `${instructions} [View the schema patch in the workflow run](${run}).` + : `[View the workflow run](${run}).`, + ].join('\n'); + + // The PR may have advanced while comments were being fetched. + await readCopiedPullRequest({ github, context }); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ ...context.repo, issue_number: pr.number, body }); + } +} + +module.exports = { reportOpenAPI }; diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 11fc6e8bf..8567ba1ff 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -19,15 +19,10 @@ concurrency: permissions: contents: read - id-token: write jobs: import: - environment: gha-kumo-workflow-tests runs-on: ubuntu-24.04 - env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - steps: - name: Checkout repository and submodules uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -42,25 +37,18 @@ jobs: files: | ui/** - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python if: steps.ui-changes.outputs.only_changed != 'true' uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: '3.14' activate-environment: true - - name: Configure AWS credentials - if: steps.ui-changes.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Install Kumo package if: steps.ui-changes.outputs.only_changed != 'true' uses: ./.github/actions/setup with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} + github_token: ${{ github.token }} - name: Test imports if: steps.ui-changes.outputs.only_changed != 'true' diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a7a2b15cd..976cad880 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -41,10 +41,8 @@ concurrency: permissions: contents: read - id-token: write env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} GITHUB_REF: ${{secrets.GITHUB_REF}} GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} @@ -79,24 +77,8 @@ jobs: needs: [diskgraph-wheel] if: ${{ !cancelled() }} runs-on: ubuntu-24.04 - continue-on-error: true - environment: - name: azure-adls-tests - # This job uses environment-scoped credentials for tests; it does not - # deploy an application. Avoid one deployment record per matrix shard. - deployment: false permissions: contents: read - id-token: write - env: - KUMO_RUN_ADLS_TESTS: ${{ github.repository == 'kumo-ai/kumo' && contains(matrix.test_dir, 'test/io_utils/') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }} - AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} strategy: fail-fast: false @@ -118,7 +100,7 @@ jobs: # connection (devapp AWS SM creds or TEMPORAL_SERVER=cloud); it # has no local-dev-server fallback, so it can't run in this matrix. "test/pool --ignore=test/pool/test_provision_terminal_nonretry.py", - "test/api_contract test/release_tests/test_build_kumo_images.py test/release_tests/test_check_updated_release_branches.py test/release_tests/test_make_kumo3_hotfix_release.py test/release_tests/test_release_v3_guards.py test/release_tests/test_start_kumo3_release.py", + "test/api_contract test/scripts/test_ci_resources.py test/release_tests/test_build_kumo_images.py test/release_tests/test_check_updated_release_branches.py test/release_tests/test_make_kumo3_hotfix_release.py test/release_tests/test_release_v3_guards.py test/release_tests/test_start_kumo3_release.py", # Public API tests "-k graph-saas_static_multiclassclassification_base test/rest/integ/test_e2e_public_api.py", @@ -136,23 +118,18 @@ jobs: # Other REST tests (public and integ should always be separated) "test/rest/integ/test_graph_validation.py", "test/rest/auth/ test/rest/public/", - "test/rest/integ/black_box/test_public_api_key_auth_flow.py test/rest/integ/test_ml_backend.py test/rest/integ/test_schema.py test/rest/integ/test_job_tags.py", + "test/rest/integ/black_box/test_public_api_key_auth_flow.py test/rest/integ/test_ml_backend.py test/rest/integ/test_schema.py test/rest/integ/test_job_tags.py test/rest/integ/test_fixture_connector_args.py", "test/rest/integ/test_warm_start_training_e2e.py", "test/rest/integ/test_ui_to_sdk_e2e.py", "test/rest/backends test/rest/config test/rest/driver test/rest/routers test/rest/utils test/rest/uxrt", "test/temporalio_commons", "test/docker", "test/deploy/test_cloud_storage_preflight.py", - "test/rest/integ/test_data_variety/test_table_lifecycle.py", - # GCS e2e tests moved to testing_gcs.yml (requires IS_GCP_DEPLOY=1) "--splits 2 --group 1 test/rest/test_*.py", "--splits 2 --group 2 test/rest/test_*.py", "-k '[local]' test/rest/integ/test_diskgraph_e2e.py", - "-k '[s3]' test/rest/integ/test_diskgraph_e2e.py", # needs S3 dataplane "-k '[local]' test/rest/integ/test_diskgraph_public_api_e2e.py", - "-k '[s3]' test/rest/integ/test_diskgraph_public_api_e2e.py", # needs S3 dataplane "-k '[local]' test/rest/integ/test_diskgraph_custom_tt_e2e.py", - "-k '[s3]' test/rest/integ/test_diskgraph_custom_tt_e2e.py", # needs S3 dataplane "test/rest/integ/test_weighted_table_e2e.py", # custom-TT REST flow (unparked from test/legacy/) "test/statedb", ] @@ -194,37 +171,14 @@ jobs: tools/** benchmark/** - - name: Configure AWS credentials - if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up packages if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' uses: ./.github/actions/setup with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} + github_token: ${{ github.token }} diskgraph-wheel-artifact: diskgraph-wheel install-graphviz: ${{ matrix.test_dir == 'test/graph/' }} - - name: Log in to Azure with GitHub OIDC - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && env.KUMO_RUN_ADLS_TESTS == '1' - uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - - name: Authenticate to Google Cloud with GitHub OIDC - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - name: Set up coverage artifact name run: | # This name must avoid using special chars and must be unique per: @@ -258,17 +212,13 @@ jobs: # exceeds the per-group duration target (e.g. # test_db_file_storage::test_compression at ~511s in # test/rest/test_*.py). No-op when --splits is not in matrix.test_dir. - GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=signal --splitting-algorithm least_duration ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} + GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --resource-group public --durations=50 --timeout=1800 --timeout-method=signal --splitting-algorithm least_duration ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} move_coverage_artifact # Mirror local-test-relevant additions in setup.cfg [tool:pytest] env. env: # Set to "1" in a PR branch to re-enable io_utils.fsutil DEBUG logs # while debugging filesystem behavior. KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} AWS_DEFAULT_REGION: "us-west-2" DEPLOY_STAGE: "test" TEMPORAL_SERVER: "localhost" @@ -278,7 +228,6 @@ jobs: KUMO_ENABLE_GLOVE: "False" SPARK_VERSION: "3.5" KUMO_TESTING: "True" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Run full test suite if: env.RUN_FULL_TEST == 'true' && steps.non-pytest-changes.outputs.only_changed != 'true' @@ -291,16 +240,12 @@ jobs: # --timeout=4500 fires below the 90-min step cap so pytest-timeout # gets a chance to run; --timeout-method=signal so the on-failure # flake-diagnostics hook runs before the process exits. - FULL_TEST=1 GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=4500 --timeout-method=signal --splitting-algorithm least_duration ${{ matrix.test_dir }} + FULL_TEST=1 GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --resource-group public --durations=50 --timeout=4500 --timeout-method=signal --splitting-algorithm least_duration ${{ matrix.test_dir }} move_coverage_artifact env: # Set to "1" in a PR branch to re-enable io_utils.fsutil DEBUG logs # while debugging filesystem behavior. KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} AWS_DEFAULT_REGION: "us-west-2" DEPLOY_STAGE: "test" TEMPORAL_SERVER: "localhost" @@ -309,10 +254,9 @@ jobs: CUDA: "cpu" SPARK_VERSION: "3.5" KUMO_TESTING: "True" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Upload coverage artifact - if: always() + if: always() && env.ARTIFACT_NAME_SAFE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_SAFE }} @@ -594,7 +538,7 @@ jobs: - name: Set up packages uses: ./.github/actions/setup with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} + github_token: ${{ github.token }} diskgraph-wheel-artifact: diskgraph-wheel - name: Build and import Kumo-core wheel diff --git a/.github/workflows/testing_databricks.yml b/.github/workflows/testing_databricks.yml deleted file mode 100644 index c4a17eb33..000000000 --- a/.github/workflows/testing_databricks.yml +++ /dev/null @@ -1,193 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Databricks Testing and Coverage - -on: # yamllint disable-line rule:truthy - push: - branches: - - main - paths-ignore: ["ui/**", "tools/**"] - pull_request: - merge_group: - workflow_dispatch: - inputs: - full_test: - description: "Run the full test suite" - type: boolean - default: true - required: false - store_durations: - description: "Record pytest-split durations (forces minimal mode)" - type: boolean - default: false - required: false - workflow_call: - inputs: - full_test: - type: boolean - default: false - required: false - store_durations: - type: boolean - default: false - required: false - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -permissions: - contents: read - id-token: write - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} - STORE_DURATIONS: ${{ inputs.store_durations == true }} - RUN_FULL_TEST: ${{ github.event_name == 'push' || (inputs.full_test == true && inputs.store_durations != true) }} - FORCE_RUN_TESTS: ${{ inputs.store_durations == true }} - -jobs: - pytest: - environment: - name: gha-kumo-workflow-tests - # This job uses environment-scoped OIDC; it does not deploy an application. - deployment: false - runs-on: ubuntu-24.04 - continue-on-error: true - - strategy: - fail-fast: false - matrix: - test_dir: [ - # The former Databricks-compute E2E selectors were removed with the - # DiskGraph cutover. Databricks source/export coverage now lives in - # test_databricks_diskgraph_e2e.py. - "test/rest/test_databricks.py", - "test/rest/integ/test_databricks_diskgraph_e2e.py", - ] - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - ref: ${{ github.sha }} - fetch-depth: 40 - - - name: Set up Python - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - python-version: "3.14" - activate-environment: true - - # uv.lock is deliberately excluded: a lock-only dependency bump must - # still run the real install + test suite (see testing.yml). - - name: Check for only non pytest required file changes - id: non-pytest-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ui/** - CHANGELOG.md - tools/** - benchmark/** - - - name: Configure AWS credentials - if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - - name: Set up packages - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && !contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up packages (with diskgraph) - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up coverage artifact name - run: | - echo "ARTIFACT_NAME_SAFE=.coverage-dbx-${{ github.run_id }}-$(echo -n ${{ matrix.test_dir }} | sha256sum | awk '{print $1}')" >> $GITHUB_ENV - - - name: Run minimal test suite - if: env.RUN_FULL_TEST != 'true' && (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') - timeout-minutes: 50 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - start_ci_memory_monitor - GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} - move_coverage_artifact - env: - KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_STAGE: "test" - TEMPORAL_SERVER: "localhost" - DATA_SPARK_MEM: "1g" - MAX_DATA_BACKEND_WORKER_PROCESSES: "1" - CUDA: "cpu" - KUMO_ENABLE_GLOVE: "False" - KUMO_TESTING: "True" - SPARK_VERSION: "3.5" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - - - name: Run full test suite - if: env.RUN_FULL_TEST == 'true' && steps.non-pytest-changes.outputs.only_changed != 'true' - timeout-minutes: 90 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - FULL_TEST=1 GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 ${{ matrix.test_dir }} - move_coverage_artifact - env: - KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_STAGE: "test" - TEMPORAL_SERVER: "localhost" - DATA_SPARK_MEM: "1g" - MAX_DATA_BACKEND_WORKER_PROCESSES: "1" - CUDA: "cpu" - KUMO_TESTING: "True" - SPARK_VERSION: "3.5" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - - - name: Upload coverage artifact - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ${{ env.ARTIFACT_NAME_SAFE }} - path: ${{ env.ARTIFACT_NAME_SAFE }} - include-hidden-files: true - - - name: Upload durations artifact - if: success() && env.STORE_DURATIONS == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: durations-${{ github.run_id }}-${{ strategy.job-index }}-of-${{ strategy.job-total }} - path: .test_durations - include-hidden-files: true - if-no-files-found: ignore diff --git a/.github/workflows/testing_gcs.yml b/.github/workflows/testing_gcs.yml deleted file mode 100644 index a49396bae..000000000 --- a/.github/workflows/testing_gcs.yml +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: GCS Testing - -on: # yamllint disable-line rule:truthy - push: - branches: - - main - paths-ignore: ["ui/**", "tools/**"] - pull_request: - merge_group: - workflow_dispatch: - inputs: - full_test: - description: "Run the full test suite" - type: boolean - default: true - required: false - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -permissions: - contents: read - id-token: write - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} - -jobs: - pytest-gcs: - environment: - name: gha-kumo-workflow-tests - # This job uses environment-scoped OIDC; it does not deploy an application. - deployment: false - name: gcs-e2e-tests - runs-on: ubuntu-24.04 - continue-on-error: true - permissions: - contents: read - id-token: write - env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - strategy: - fail-fast: false - matrix: - test_dir: [ - test/rest/integ/test_gcs_e2e.py, - test/rest/integ/test_bigquery_diskgraph_e2e.py, - test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py, - ] - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - ref: ${{ github.sha }} - fetch-depth: 40 - - - name: Get GCS related changes - id: gcs-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 - with: - files: | - .github/actions/setup/action.yml - .github/workflows/testing_gcs.yml - kumo/connector/** - kumo/testing/gcs.py - io_utils/** - rest/** - setup.py - test/rest/integ/test_gcs_e2e.py - test/rest/integ/test_bigquery_diskgraph_e2e.py - test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py - test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json - rest/devserver/configs/diskgraph-test.json - kumo/testing/bigquery.py - kumo/testing/rest.py - - - name: Set up Python - if: steps.gcs-files-changed.outputs.any_changed == 'true' - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - python-version: "3.14" - activate-environment: true - - - name: Configure AWS credentials - if: steps.gcs-files-changed.outputs.any_changed == 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - - name: Set up packages - if: steps.gcs-files-changed.outputs.any_changed == 'true' && !contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up packages (with diskgraph) - if: steps.gcs-files-changed.outputs.any_changed == 'true' && contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Authenticate to Google Cloud with GitHub OIDC - if: steps.gcs-files-changed.outputs.any_changed == 'true' && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - - name: Run E2E tests - if: steps.gcs-files-changed.outputs.any_changed == 'true' - timeout-minutes: 60 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - GLOG_logtostdout=1 python -X faulthandler -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=thread ${{ matrix.test_dir }} - env: - # GCS-specific environment variables - IS_GCP_DEPLOY: "1" - GCS_TEST_BUCKET: 'kumo-test-bucket' - GCP_STORAGE_TRANSFER_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: "true" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - # Standard test environment variables - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_STAGE: "test" - TEMPORAL_SERVER: "localhost" - DATA_SPARK_MEM: "1g" - MAX_DATA_BACKEND_WORKER_PROCESSES: "1" - CUDA: "cpu" - KUMO_ENABLE_GLOVE: "False" - SPARK_VERSION: "3.5" - KUMO_TESTING: "True" diff --git a/.github/workflows/testing_private.yml b/.github/workflows/testing_private.yml new file mode 100644 index 000000000..47c35e36b --- /dev/null +++ b/.github/workflows/testing_private.yml @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Private resource tests + +on: # yamllint disable-line rule:truthy + push: + branches: [main, "pull-request/[0-9]+"] + workflow_dispatch: + inputs: + full_test: + description: Run full-suite-only cases as well + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} + +permissions: + contents: read + +jobs: + changes: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + matrix: ${{ steps.select.outputs.matrix }} + run: ${{ steps.select.outputs.run }} + gcp-e2e: ${{ steps.select.outputs.gcp-e2e }} + snowflake-e2e: ${{ steps.select.outputs.snowflake-e2e }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + persist-credentials: false + - name: Verify the copied PR and find its merge base + id: pr + if: startsWith(github.ref, 'refs/heads/pull-request/') + uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 + with: + script: | + const { setCopiedPullRequestOutputs } = require('./.github/scripts/copied-pr.cjs'); + await setCopiedPullRequestOutputs({ github, context, core }); + - name: Read complete changed-file list + id: files + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 + with: + base_sha: ${{ steps.pr.outputs.base || github.event.before }} + json: true + escape_json: false + - name: Select private jobs + id: select + env: + CHANGED_FILES: ${{ steps.files.outputs.all_changed_files }} + FORCE_PRIVATE_TESTS: ${{ github.event_name == 'workflow_dispatch' }} + run: python3 tools/ci_private_changes.py + + diskgraph-wheel: + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/_diskgraph-wheel.yml + permissions: + contents: read + + pytest: + name: private-${{ matrix.group }} + needs: [changes, diskgraph-wheel] + if: needs.changes.outputs.run == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.changes.outputs.matrix) }} + runs-on: + group: nv-cpu-general + labels: linux-amd64-cpu16 + timeout-minutes: ${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || inputs.full_test == true) && 90 || 25 }} + environment: + name: ${{ matrix.group == 'azure' && 'azure-adls-tests' || 'gha-kumo-workflow-tests' }} + deployment: false + permissions: + contents: read + id-token: write + env: + FULL_TEST: ${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || inputs.full_test == true) && '1' || '0' }} + CI_RUN_GCP_E2E: ${{ needs.changes.outputs.gcp-e2e }} + CI_RUN_SNOWFLAKE_E2E: ${{ needs.changes.outputs.snowflake-e2e }} + CI_TEST_RESULTS: ci-test-results + ARTIFACTS_DIR: /tmp/kumo-test-artifacts + AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 + AWS_DEFAULT_REGION: us-west-2 + DEPLOY_STAGE: test + TEMPORAL_SERVER: localhost + DATA_SPARK_MEM: 1g + MAX_DATA_BACKEND_WORKER_PROCESSES: '1' + CUDA: cpu + KUMO_ENABLE_GLOVE: 'False' + SPARK_VERSION: '3.5' + SPARK_LOCAL_IP: 127.0.0.1 + KUMO_TESTING: 'True' + KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: '0' + KUMO_RUN_ADLS_TESTS: ${{ matrix.group == 'azure' && '1' || '0' }} + KUMO_RUN_GCP_TESTS: ${{ startsWith(matrix.group, 'gcp') && '1' || '0' }} + KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ startsWith(matrix.group, 'gcp') && 'true' || 'false' }} + GCS_TEST_BUCKET: kumo-test-bucket + GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} + GCP_STORAGE_TRANSFER_BUCKET: ${{ vars.GCP_STORAGE_TRANSFER_BUCKET }} + GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} + GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} + GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} + AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }} + AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }} + DATABRICKS_HOST: ${{ matrix.group == 'databricks' && vars.DATABRICKS_HOST || '' }} + DATABRICKS_CLUSTER_ID: ${{ vars.DATABRICKS_CLUSTER_ID }} + DATABRICKS_WAREHOUSE_ID: ${{ vars.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_TOKEN: ${{ matrix.group == 'databricks' && secrets.DATABRICKS_TOKEN || '' }} + DATABRICKS_CLIENT_ID: ${{ matrix.group == 'databricks' && vars.DATABRICKS_CLIENT_ID || '' }} + DATABRICKS_CLIENT_SECRET: ${{ matrix.group == 'databricks' && secrets.DATABRICKS_CLIENT_SECRET || '' }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - name: Set up Python + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + python-version: '3.14' + activate-environment: true + - name: Set up packages + uses: ./.github/actions/setup + with: + github_token: ${{ github.token }} + diskgraph-wheel-artifact: diskgraph-wheel + install-graphviz: 'true' + - name: Configure AWS credentials + if: startsWith(matrix.group, 'aws') || matrix.group == 'gcp-e2e' || matrix.group == 'snowflake' || matrix.group == 'databricks' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests + aws-region: us-west-2 + - name: Authenticate to Google Cloud + if: startsWith(matrix.group, 'gcp') + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + project_id: ${{ vars.GCP_PROJECT_ID }} + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} + - name: Authenticate to Azure + if: matrix.group == 'azure' + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Verify Azure test storage configuration + if: matrix.group == 'azure' + run: | + python - <<'PY' + import os + missing = [name for name in ('AZURE_STORAGE_ACCOUNT', 'AZURE_CONTAINER_NAME') if not os.environ.get(name)] + assert not missing, f'Missing Azure test configuration: {missing}' + PY + - name: Verify federated Google credentials + if: startsWith(matrix.group, 'gcp') + run: | + python - <<'PY' + import json, os + from pathlib import Path + credentials = json.loads(Path(os.environ['GOOGLE_APPLICATION_CREDENTIALS']).read_text()) + assert credentials['type'] == 'external_account', 'Private GCP CI requires federation' + required = ['GCP_PROJECT_ID', 'GCP_TEST_SERVICE_ACCOUNT', 'GCP_STORAGE_TRANSFER_BUCKET'] + if os.environ['KUMO_RUN_GCP_TESTS'] == '1': + required += ['GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT', 'GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT'] + missing = [name for name in required if not os.environ.get(name)] + assert not missing, f'Missing GCP test configuration: {missing}' + PY + - name: Verify Databricks test configuration + if: matrix.group == 'databricks' + run: | + python - <<'PY' + import os + required = ['DATABRICKS_HOST', 'DATABRICKS_WAREHOUSE_ID', 'DATABRICKS_CLIENT_ID', 'DATABRICKS_CLIENT_SECRET', 'DATABRICKS_TOKEN'] + missing = [name for name in required if not os.environ.get(name)] + assert not missing, f'Missing Databricks test configuration: {missing}' + PY + - name: Run selected resource tests + env: + RESOURCE_GROUP: ${{ matrix.group }} + run: python -m tools.run_private_tests "$RESOURCE_GROUP" + - name: Upload selection, results and complete durations + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: private-tests-${{ matrix.group }}-${{ github.run_id }} + path: ci-test-results/ + if-no-files-found: error diff --git a/.github/workflows/testing_snowflake.yml b/.github/workflows/testing_snowflake.yml deleted file mode 100644 index e665ef75e..000000000 --- a/.github/workflows/testing_snowflake.yml +++ /dev/null @@ -1,224 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Snowflake Testing and Coverage - -on: # yamllint disable-line rule:truthy - push: - branches: - - main - paths-ignore: ["ui/**", "tools/**"] - pull_request: - merge_group: - workflow_dispatch: - inputs: - full_test: - description: "Run the full test suite" - type: boolean - default: true - required: false - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -permissions: - contents: read - id-token: write - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} - -jobs: - pytest: - environment: - name: gha-kumo-workflow-tests - # This job uses environment-scoped OIDC; it does not deploy an application. - deployment: false - name: all-snowflake-tests - runs-on: ubuntu-24.04 - continue-on-error: true - permissions: - contents: read - id-token: write - env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} - - strategy: - matrix: - test_name: - [ - 'connector (test/connector)', - 'rest (test/rest)', - 'fsutil (test/io_utils/fsutil)', - 'backend - data (test/data/ test/graph/', - 'backend - pquery (test/pquery/bp_generator)', - 'backend - pquery (test/pquery/train_table_generator)', - 'snowflake diskgraph e2e (test/rest/integ/test_snowflake_diskgraph_e2e.py)', - 'snowflake ingestion diskgraph e2e (test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py)' - ] - include: - - test_name: 'connector (test/connector)' - test_dir: 'test/connector' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - setup.py - test/connector/** - - test_name: 'rest (test/rest)' - test_dir: 'test/rest --ignore=test/rest/integ/test_snowflake_diskgraph_e2e.py --ignore=test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - setup.py - test/rest/** - - test_name: 'fsutil (test/io_utils/fsutil)' - test_dir: 'test/io_utils/fsutil' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - io_utils/** - setup.py - test/io_utils/fsutil/** - - test_name: 'backend - data (test/data/ test/graph/' - test_dir: 'test/data/ test/graph/**snowflake**' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/stats/** - io_utils/** - setup.py - test/data/ - test/graph/**snowflake** - - test_name: 'backend - pquery (test/pquery/bp_generator)' - test_dir: 'test/pquery/test_pql_batch_pred_generator.py' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/pquery/** - io_utils/** - kumo/data/data_processing_backend/** - setup.py - test/pquery/test_pql_batch_pred_generator.py - - test_name: 'backend - pquery (test/pquery/train_table_generator)' - test_dir: 'test/pquery/train_table_generator' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/pquery/** - io_utils/** - setup.py - test/pquery/train_table_generator/** - - test_name: 'snowflake diskgraph e2e (test/rest/integ/test_snowflake_diskgraph_e2e.py)' - test_dir: "test/rest/integ/test_snowflake_diskgraph_e2e.py" - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - rest/backends/data_backend/activities/** - test/rest/integ/test_snowflake_diskgraph_e2e.py - test/rest/integ/test_cases/test_snowflake_e2e_fixtures.json - rest/devserver/configs/diskgraph-test.json - - test_name: 'snowflake ingestion diskgraph e2e (test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py)' - test_dir: "test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py" - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - rest/backends/data_backend/activities/** - test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py - rest/devserver/configs/diskgraph-test.json - - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - ref: ${{ github.sha }} - fetch-depth: 40 - - - name: Get any data backend related changes - id: data-backend-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ${{ matrix.files_changed }} - - - name: Configure AWS credentials - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - - name: Set up packages - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && !contains(matrix.test_dir, 'diskgraph') && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up packages (with diskgraph) - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && contains(matrix.test_dir, 'diskgraph') && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Authenticate to Google Cloud with GitHub OIDC - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - # NOTE here we use coverage.py instead of pytest-cov to obtain JSON - # outputs that are easier to parse and manage (pytest-cov/455). - # coverage.py also has additional nice features that may be useful in the - # future for better coverage reports. - - name: Run minimal test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && github.event_name != 'push' && inputs.full_test != true && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - timeout-minutes: 120 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} - env: - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - - - name: Run full test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && (github.event_name == 'push' || inputs.full_test == true) && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - timeout-minutes: 120 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - FULL_TEST=1 GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} - env: - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} diff --git a/.github/workflows/ui-client-code-generate.yml b/.github/workflows/ui-client-code-generate.yml index 7f72736e9..d11b51199 100644 --- a/.github/workflows/ui-client-code-generate.yml +++ b/.github/workflows/ui-client-code-generate.yml @@ -6,73 +6,54 @@ name: Auto UI client code gen using Orval on: # yamllint disable-line rule:truthy workflow_dispatch: - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'rest/app.py' - - 'rest/config/**' - - 'rest/main.py' - - 'rest/routers/**' - - 'rest/scripts/download_schema.py' - - 'rest/utils/routers.py' - - 'ui/src/gen/spec/**' + # Schema analysis waits for approval of the exact commit by copy-pr-bot. + push: + branches: ["pull-request/[0-9]+"] issue_comment: types: [created] concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} + group: ${{ github.workflow }}-${{ github.event.issue.number || github.ref }}-${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'approved' || github.run_number }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: - contents: write - pull-requests: write - issues: write - packages: read + contents: read jobs: check-openapi-changes: - environment: gha-kumo-workflow-tests name: Check OpenAPI Changes runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' + timeout-minutes: 45 + if: github.event_name == 'push' permissions: - id-token: write # Required for configure-aws-credentials OIDC. contents: read pull-requests: write - issues: write - packages: read - outputs: - spec_changed: ${{steps.openapi-changes.outputs.changed}} steps: - - name: Checkout PR branch + - name: Checkout the approved commit uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: ref: ${{ github.sha }} - # Shallow checkout to avoid a full clone of git history - fetch-depth: 1 + fetch-depth: 0 + persist-credentials: false - - name: Set up Python - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Verify the copied PR and find its merge base + id: pr + uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 with: - python-version: "3.14" - activate-environment: true - enable-cache: false + script: | + const { setCopiedPullRequestOutputs } = require('./.github/scripts/copied-pr.cjs'); + await setCopiedPullRequestOutputs({ github, context, core }); - name: Check for backend/API changes id: backend-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 with: + base_sha: ${{ steps.pr.outputs.base }} files: | + .github/actions/setup/** + .github/scripts/copied-pr.cjs + .github/scripts/openapi-pr.cjs + .github/workflows/ui-client-code-generate.yml rest/app.py rest/config/** rest/main.py @@ -81,20 +62,13 @@ jobs: rest/utils/routers.py ui/src/gen/spec/** - - name: Configure AWS credentials - if: steps.backend-changes.outputs.any_changed == 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up backend packages if: steps.backend-changes.outputs.any_changed == 'true' uses: ./.github/actions/setup with: python-version: "3.14" - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - ref: ${{ github.event.pull_request.head.sha }} + github_token: ${{ github.token }} + ref: ${{ steps.pr.outputs.head }} - name: Download openapi.json files if: steps.backend-changes.outputs.any_changed == 'true' @@ -123,63 +97,53 @@ jobs: cd ui npm run lint-format-codegen --subdir=spec - - name: Check for Spec changes + - name: Check for schema changes if: steps.backend-changes.outputs.any_changed == 'true' id: openapi-changes run: | - if git diff --quiet --exit-code ui/src/gen/spec/; then - echo "changed=false" >> $GITHUB_OUTPUT + if git diff --quiet --exit-code -- ui/src/gen/spec/; then + echo "changed=false" >> "$GITHUB_OUTPUT" else - echo "changed=true" >> $GITHUB_OUTPUT + echo "changed=true" >> "$GITHUB_OUTPUT" + git diff --binary -- ui/src/gen/spec/ > "$RUNNER_TEMP/openapi-schema.patch" fi - - name: Post warning comment if changes detected - if: steps.backend-changes.outputs.any_changed == 'true' && steps.openapi-changes.outputs.changed == 'true' + - name: Upload the schema patch + if: steps.openapi-changes.outputs.changed == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: openapi-schema-patch + path: ${{ runner.temp }}/openapi-schema.patch + if-no-files-found: error + retention-days: 7 + + - name: Summarize schema analysis + env: + SCHEMA_CHANGED: ${{ steps.openapi-changes.outputs.changed }} + API_CHANGED: ${{ steps.backend-changes.outputs.any_changed }} + run: | + if [[ "$SCHEMA_CHANGED" == true ]]; then + echo 'OpenAPI schemas need regeneration. Download the schema patch from this run.' >> "$GITHUB_STEP_SUMMARY" + elif [[ "$API_CHANGED" == true ]]; then + echo 'OpenAPI schemas are up to date.' >> "$GITHUB_STEP_SUMMARY" + else + echo 'No OpenAPI-related files changed.' >> "$GITHUB_STEP_SUMMARY" + fi + echo "Analyzed commit: $GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" + + - name: Update the OpenAPI PR comment + env: + SCHEMA_CHANGED: ${{ steps.openapi-changes.outputs.changed }} uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const warningMessage = `⚠️ **OpenAPI spec changes detected in this PR** - - This means the UI client code needs to be regenerated. - - To regenerate the UI client code in this PR, comment: - \`/trigger-ui-codegen\``; - - // Find existing warning comment - const comments = await github.rest.issues.listComments({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }); - - const existingComment = comments.data.find(comment => - comment.body.includes('⚠️ **OpenAPI spec changes detected in this PR**') - ); - - // Update existing comment or create new one - if (existingComment) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existingComment.id, - body: warningMessage - }); - } else { - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: warningMessage - }); - } + const { reportOpenAPI } = require('./.github/scripts/openapi-pr.cjs'); + await reportOpenAPI({ github, context, changed: process.env.SCHEMA_CHANGED === 'true' }); trigger-codegen: - environment: gha-kumo-workflow-tests name: Trigger UI Codegen runs-on: ubuntu-24.04 permissions: - id-token: write # Required for configure-aws-credentials OIDC. contents: write pull-requests: write issues: write @@ -271,12 +235,6 @@ jobs: python-version: "3.14" activate-environment: true - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up backend packages uses: ./.github/actions/setup with: @@ -372,11 +330,9 @@ jobs: git push origin HEAD:${{ steps.pr-details.outputs.ref }} codegen: - environment: gha-kumo-workflow-tests name: Manual Codegen (workflow_dispatch) runs-on: ubuntu-24.04 permissions: - id-token: write # Required for configure-aws-credentials OIDC. contents: write pull-requests: write packages: read @@ -397,12 +353,6 @@ jobs: python-version: "3.14" activate-environment: true - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up backend packages uses: ./.github/actions/setup with: diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index 44c29f679..fbb629848 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -22,6 +22,6 @@ jobs: with: persist-credentials: false - name: Test workflow security policy - run: node --test .github/scripts/workflow-security.test.cjs + run: node --test .github/scripts/workflow-security.test.cjs .github/scripts/approved-pr.test.cjs - name: Reject new workflow security findings run: node .github/scripts/workflow-security.cjs . --baseline .github/workflow-security-baseline.json diff --git a/docs/connector-test-matrix.md b/docs/connector-test-matrix.md index f0cc3d673..9c82ad43e 100644 --- a/docs/connector-test-matrix.md +++ b/docs/connector-test-matrix.md @@ -86,18 +86,12 @@ The existing hosted workflows have different triggers and failure policies: | Workflow / coverage | When tests run | Failure behavior | | --- | --- | --- | -| [Testing and Coverage](../.github/workflows/testing.yml): hosted S3, ADLS and GCP tests | PR, master push, merge queue, manual or reusable-workflow invocation, subject to test-file filters and selected shards. ADLS/GCP authentication is limited to configured repository/shard and same-repository PR conditions. | The hosted `pytest` job has `continue-on-error: true`. Package-suite failures fail the separate `packages` job. | -| [GCS Testing](../.github/workflows/testing_gcs.yml): hosted GCS and BigQuery end-to-end tests | PR, master push, merge queue or manual run; test steps require matching changed files. GCP authentication also requires the configured repository and a same-repository PR for PR events. | Test job has `continue-on-error: true`. | -| [Snowflake Testing](../.github/workflows/testing_snowflake.yml): hosted Snowflake tests | PR, master push, merge queue or manual run; each shard's test steps require matching changed files. | Test job has `continue-on-error: true`. | -| [Databricks Testing](../.github/workflows/testing_databricks.yml): hosted REST and DiskGraph tests | PR, master push, merge queue, manual or reusable-workflow invocation; test steps apply changed-file and full/minimal-mode conditions. Durations recording can force a minimal run. | Test job has `continue-on-error: true`. | - -These workflows exclude pushes that only change `ui/` or `tools/`. -Their existence does not mean every PR runs cloud tests. Jobs marked -`continue-on-error` are informational for workflow failure propagation, not -evidence of a required merge gate. The package job does not enable the -Snowflake, Databricks or BigQuery live flags. Their source-reader/sink suites -remain distinct from the hosted workflows above. Access approvals and merge -rules are separate from these workflow triggers. +| [Testing and Coverage](../.github/workflows/testing.yml): public local/mocked Python tests | PR, main push, merge queue, manual or reusable invocation; resource-backed cases are deselected before fixture setup. | Test failures fail their jobs. | +| [Private resource tests](../.github/workflows/testing_private.yml): AWS, GCP, Azure, Snowflake, and Databricks | Approved copy-pr-bot branch push, main push, or manual dispatch on NVIDIA private runners; expensive provider E2E selections retain change filters. | Failures and missing-credential skips fail verification. | + +Public tests do not configure cloud identities. Private jobs use the shared resource classification and retain selections and results as artifacts. Workflow success does not establish coverage for a selection that was filtered out or intentionally skipped. Required merge checks are configured separately. Use `python -m pytest --resource-group public ` for local public selection; the private workflow uses `python -m tools.run_private_tests ` to run isolated provider sessions. Package source-reader/sink live suites retain their separate opt-in flags. + +Private discovery scans component test subdirectories. Root-level script tests, release tooling, and package suites retain separate entry points. Resource decorators and `private_resource` marks declare cloud access; `ci_profile`, `ci_group`, and `ci_gate` marks declare process environments, balanced workloads, and provider change gates. Fixture metadata propagates through dependency chains without resolving credentials. ## Real cloud tests on Kumo resources diff --git a/kumo/testing/_skip.py b/kumo/testing/_skip.py index c96a8ec30..311fefd14 100644 --- a/kumo/testing/_skip.py +++ b/kumo/testing/_skip.py @@ -42,3 +42,18 @@ def _sync_wrapper(*args, **kwargs): return func(*args, **kwargs) return _sync_wrapper + + +def requires_resource(provider: str, func: Callable | None = None, *, + profile: str | None = None) -> Callable: + """Describe live test/fixture dependencies without accessing credentials.""" + if func is None: + return functools.partial(requires_resource, provider, profile=profile) + resources = set(getattr(func, '__kumo_test_resources__', ())) + resources.add(provider) + setattr(func, '__kumo_test_resources__', tuple(sorted(resources))) + if profile: + profiles = set(getattr(func, '__kumo_ci_profiles__', ())) + profiles.add(profile) + func.__kumo_ci_profiles__ = tuple(sorted(profiles)) + return func diff --git a/kumo/testing/adls.py b/kumo/testing/adls.py index 2c638c5eb..fc4363061 100644 --- a/kumo/testing/adls.py +++ b/kumo/testing/adls.py @@ -104,6 +104,8 @@ def test_with_config(): } """ def decorator(test_func: Callable): + from kumo.testing._skip import requires_resource + test_func = requires_resource('azure', test_func) credentials_missing = not _adls_credentials_configured(config_path) if inspect.isgeneratorfunction(test_func): # A decorator below @pytest.fixture receives the raw generator. diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index db7ad10be..617ae6515 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -113,6 +113,9 @@ def withBigQueryCredentials(func: Optional[Callable] = None, import pytest def decorator(inner_func: Callable): + from kumo.testing._skip import requires_resource + inner_func = requires_resource('gcp', inner_func) + def _setup(): setup_bigquery_credentials(service_account_key) if (not os.getenv(ENV_GOOGLE_CREDENTIALS_JSON) and diff --git a/kumo/testing/databricks.py b/kumo/testing/databricks.py index adc5db588..b046c4bb9 100644 --- a/kumo/testing/databricks.py +++ b/kumo/testing/databricks.py @@ -24,7 +24,8 @@ class DatabricksTableConnector: def withDatabricksCredentials(func: Callable): """Skip unless DATABRICKS_HOST and DATABRICKS_TOKEN are set.""" - from kumo.testing._skip import skip_if + from kumo.testing._skip import requires_resource, skip_if + func = requires_resource('databricks', func, profile='databricks-pat') has_no_credentials = not (DATABRICKS_HOST.parse_value() and DATABRICKS_TOKEN.parse_value()) return skip_if(has_no_credentials, "No Databricks credentials found", func) @@ -32,7 +33,8 @@ def withDatabricksCredentials(func: Callable): def withDatabricksOauthCredentials(func: Callable): """Skip unless the host, client ID, and client secret variables are set.""" - from kumo.testing._skip import skip_if + from kumo.testing._skip import requires_resource, skip_if + func = requires_resource('databricks', func, profile='databricks-oauth') has_no_credentials = not (DATABRICKS_HOST.parse_value() and DATABRICKS_CLIENT_ID.parse_value() and DATABRICKS_CLIENT_SECRET.parse_value()) diff --git a/kumo/testing/gcs.py b/kumo/testing/gcs.py index a511fd19d..630d8e151 100644 --- a/kumo/testing/gcs.py +++ b/kumo/testing/gcs.py @@ -127,6 +127,8 @@ def test_gcs_operation(): # A collection-time mark prevents live GCS fixtures from being resolved # when credentials are absent. A runtime wrapper skips too late: pytest # sets up fixtures before invoking the wrapped test. + from kumo.testing._skip import requires_resource + func = requires_resource('gcp', func) credentials_missing = not _check_gcs_credentials() reason = "No GCS credentials found" if inspect.isgeneratorfunction(func): diff --git a/kumo/testing/s3.py b/kumo/testing/s3.py index 1c0f57b8f..3de7479cd 100644 --- a/kumo/testing/s3.py +++ b/kumo/testing/s3.py @@ -48,9 +48,12 @@ def _s3_credentials_configured() -> bool: def withS3Credentials(func: Callable | None = None): credentials_missing = not _s3_credentials_configured() reason = "No AWS credentials found for S3" - marker = pytest.mark.skipif(credentials_missing, reason=reason) + marker = pytest.mark.skipif(credentials_missing, reason=reason, + kumo_resource='aws') + from kumo.testing._skip import requires_resource if func is None: return marker + func = requires_resource('aws', func) if inspect.isgeneratorfunction(func): from kumo.testing._skip import skip_if return skip_if(credentials_missing, reason, func) diff --git a/kumo/testing/snowflake.py b/kumo/testing/snowflake.py index 6cb83a3d6..d6ba76521 100644 --- a/kumo/testing/snowflake.py +++ b/kumo/testing/snowflake.py @@ -38,7 +38,9 @@ def withSnowflakeCredentials(func: Callable): """Load credentials during pytest setup, before cloud resource fixtures.""" import pytest - return pytest.mark.usefixtures('snowflake_credentials')(func) + from kumo.testing._skip import requires_resource + return pytest.mark.usefixtures('snowflake_credentials')(requires_resource( + 'snowflake', func)) def setup_snowflake_credentials() -> None: diff --git a/test/ci_resources.py b/test/ci_resources.py new file mode 100644 index 000000000..804c093b6 --- /dev/null +++ b/test/ci_resources.py @@ -0,0 +1,188 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Select CI cases and describe their resource and process requirements.""" + +import json +import os +import re +from pathlib import Path + +import pytest + +GROUPS = ('public', 'aws', 'aws-public-api', 'gcp', 'gcp-e2e', 'azure', + 'snowflake', 'databricks') +PROFILE_ENV = { + 'diskgraph': {}, + 'gcp': { + 'IS_GCP_DEPLOY': '1' + }, + 'databricks-pat': { + 'DATABRICKS_CLIENT_ID': '', + 'DATABRICKS_CLIENT_SECRET': '' + }, + 'databricks-oauth': { + 'DATABRICKS_TOKEN': '' + }, +} +GATES = {'gcp-e2e': 'CI_RUN_GCP_E2E', 'snowflake-e2e': 'CI_RUN_SNOWFLAKE_E2E'} + + +def execution_profile(*profiles): + """Attach process requirements to a fixture without resolving it.""" + def decorate(func): + func.__kumo_ci_profiles__ = profiles + return func + + return decorate + + +def annotated_functions(item): + yield item.obj + # Fixture marks do not propagate to tests. Keep pytest-internal inspection + # here, and read metadata without resolving any fixture. + for name in item.fixturenames: + for definition in item._fixtureinfo.name2fixturedefs.get(name, ()): + yield definition.func + + +def resources_for_item(item): + resources = set() + for func in annotated_functions(item): + resources.update(getattr(func, '__kumo_test_resources__', ())) + for marker in item.iter_markers('skipif'): + if marker.kwargs.get('kumo_resource'): + resources.add(marker.kwargs['kumo_resource']) + for marker in item.iter_markers('private_resource'): + resources.update(marker.args) + unknown = resources - {'aws', 'gcp', 'azure', 'snowflake', 'databricks'} + if unknown: + raise pytest.UsageError( + f'Unknown resources for {item.nodeid}: {sorted(unknown)}') + return resources + + +def profiles_for_item(item): + profiles = set() + for func in annotated_functions(item): + profiles.update(getattr(func, '__kumo_ci_profiles__', ())) + for marker in item.iter_markers('ci_profile'): + profiles.update(marker.args) + if profiles - PROFILE_ENV.keys(): + raise pytest.UsageError( + f'Unknown CI profiles for {item.nodeid}: {sorted(profiles)}') + if {'databricks-pat', 'databricks-oauth'} <= profiles: + raise pytest.UsageError( + f'Conflicting Databricks auth profiles: {item.nodeid}') + return sorted(profiles) + + +def group_for_item(item): + resources = resources_for_item(item) + if not resources: + return 'public' + provider = next(p + for p in ('snowflake', 'databricks', 'gcp', 'azure', 'aws') + if p in resources) + marker = item.get_closest_marker('ci_group') + group = marker.args[0] if marker else provider + if group not in GROUPS or group.split('-', 1)[0] != provider: + raise pytest.UsageError(f'Invalid CI group {group} for {item.nodeid}') + return group + + +def pytest_addoption(parser): + parser.addoption('--resource-group', choices=GROUPS, default=None, + help='Select public tests or one private CI group') + parser.addoption( + '--resource-manifest', type=Path, + help='Write selected node IDs, resources and process profiles') + parser.addoption( + '--private-discovery', action='store_true', + help='Apply private CI provider change gates during collection') + parser.addoption('--require-resource-credentials', action='store_true', + help='Fail private CI on missing resource credentials') + + +def pytest_configure(config): + for marker in ('private_resource(provider): uses a live private resource', + 'ci_profile(name): requires a declared process environment', + 'ci_group(name): assigns resource cases to a CI workload', + 'ci_gate(name): follows a provider E2E changed-file gate'): + config.addinivalue_line('markers', marker) + config._resource_credential_skips = [] + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + group = config.getoption('--resource-group') + if group is None: + return + selected, deselected = [], [] + for item in items: + enabled = True + if config.getoption('--private-discovery'): + for marker in item.iter_markers('ci_gate'): + gate = marker.args[0] + if gate not in GATES: + raise pytest.UsageError(f'Unknown CI gate: {gate}') + enabled &= os.environ.get(GATES[gate]) != 'false' + (selected if enabled and group_for_item(item) == group else + deselected).append(item) + items[:] = selected + config.hook.pytest_deselected(items=deselected) + + +def pytest_collection_finish(session): + destination = session.config.getoption('--resource-manifest') + if destination: + manifest = {} + for item in session.items: + profiles = profiles_for_item(item) + manifest[item.nodeid] = { + 'group': + group_for_item(item), + 'resources': + sorted(resources_for_item(item)), + 'profiles': + profiles, + 'isolated_module': + item.nodeid.split('::', 1)[0] + if 'diskgraph' in profiles else '', + } + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(manifest, indent=2) + '\n') + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + report = (yield).get_result() + if report.when == 'call': + reporter = item.config.pluginmanager.get_plugin('terminalreporter') + if reporter: + for line in report.capstdout.splitlines(): + timing = re.match( + r'^CI_PHASE public-api [A-Za-z0-9_]+ elapsed_seconds=\d+\.\d{3}', + line) + if timing: + reporter.write_line(timing.group(0)) + if (item.config.getoption('--require-resource-credentials') + and report.skipped and group_for_item(item) != 'public'): + reason = str(report.longrepr).lower() + if any(fragment in reason + for fragment in ('credential', 'not configured', + 'access is unavailable')): + item.config._resource_credential_skips.append(item.nodeid) + + +def pytest_sessionfinish(session, exitstatus): + missing = session.config._resource_credential_skips + if missing: + reporter = session.config.pluginmanager.get_plugin('terminalreporter') + if reporter: + reporter.write_sep( + '=', 'Private CI skipped tests for missing credentials', + red=True) + for nodeid in missing: + reporter.write_line(nodeid) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/test/conftest.py b/test/conftest.py index 9b72acc80..296c7d3dd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,7 +5,7 @@ # Register fixtures defined under test/fixtures/ so they're discoverable # session-wide without per-test imports. Currently exposes # ``dg_indexed_financial_graph`` (see test/fixtures/dg_indexed_graph.py). -pytest_plugins = ["test.fixtures.dg_indexed_graph"] +pytest_plugins = ["test.fixtures.dg_indexed_graph", "test.ci_resources"] import asyncio # noqa: E402 import copy # noqa: E402 @@ -70,6 +70,7 @@ from kumo.graph.table import Table # noqa: E402 from kumo.task.task import Task # noqa: E402 from kumo.testing import withDatabricksCredentials # noqa: E402 +from kumo.testing._skip import requires_resource # noqa: E402 from kumo.testing.generator import EdgeSpec, GraphSpec, TableSpec # noqa: E402 from kumo.testing.snowflake import setup_snowflake_credentials # noqa: E402 from kumo.utils.embedding_models import embedding_model_store # noqa: E402 @@ -96,8 +97,42 @@ os.environ.setdefault(_ENABLE_FSUTIL_DEBUG_LOGS_ENV, '0') +@pytest.fixture(scope='session') +def test_execution_environment(request): + """Apply selected process configuration before any deployment/cache fixtures.""" + from test.ci_resources import profiles_for_item + + from service_framework import deploy_env_config as config + + if not any('diskgraph' in profiles_for_item(item) + for item in request.session.items): + yield + return + names = ('DEPLOY_ENV_CONFIG_FILE', 'DEPLOY_ENV_CONFIG_JSON', 'PATH') + previous = {name: os.environ.get(name) for name in names} + previous_deploy_env = config._deploy_env + previous_cache = data._active_cache + if not (os.environ.get('DEPLOY_ENV_CONFIG_FILE') + or os.environ.get('DEPLOY_ENV_CONFIG_JSON')): + os.environ['DEPLOY_ENV_CONFIG_FILE'] = str( + pathlib.Path(__file__).resolve().parents[1] / 'rest' / + 'devserver' / 'configs' / 'diskgraph-test.json') + config._reset_deploy_env() + data.reset_active_cache_for_test() + try: + yield + finally: + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + config._deploy_env = previous_deploy_env + data.reset_active_cache_for_test(previous_cache) + + @pytest.fixture(scope='session', autouse=True) -def configure_test_glove_artifacts(tmp_path_factory): +def configure_test_glove_artifacts(tmp_path_factory, test_execution_environment): artifacts = tmp_path_factory.mktemp('glove') tokens = [ 'get', @@ -160,6 +195,9 @@ def pytest_collection_modifyitems(config, items): if "nightly" in item.keywords: item.add_marker(skip_nightly) + if config.getoption('--resource-group') == 'snowflake': + return # The resource selector also includes unmarked live session tests. + # check if you got an option like --key=snowflake if not config.getoption("--runsnowflake"): skip_snowflake = pytest.mark.skip( @@ -1441,6 +1479,7 @@ def create_s3_graph() -> Callable: @pytest.fixture() +@requires_resource('snowflake') def create_snowflake_graph() -> Callable: return _create_snowflake_graph @@ -1451,6 +1490,7 @@ def create_task() -> Callable: @pytest.fixture(scope='session') +@requires_resource('snowflake') def snowflake_credentials() -> None: setup_snowflake_credentials() @@ -1587,7 +1627,7 @@ def run_before_and_after_tests(auto_setup_kumo_backend: bool, @pytest.fixture(scope='session', autouse=True) -def run_before_and_after_test_sessions(): +def run_before_and_after_test_sessions(test_execution_environment): # The data persist dir is derived from deploy_env() (kumo.data # .get_persist_dir); wipe it at session boundaries for clean seeding. data_persist_dir = data.get_persist_dir() @@ -1602,7 +1642,7 @@ def run_before_and_after_test_sessions(): @pytest.fixture(scope='session', autouse=True) -def _redirect_test_diskgraph_data_root(): +def _redirect_test_diskgraph_data_root(test_execution_environment): """Point the DiskGraph data_root at a writable location under TEST. The production default for ``DiskgraphConfig.data_root`` is diff --git a/test/connector/conftest.py b/test/connector/conftest.py index 5269e79ff..fe9732917 100644 --- a/test/connector/conftest.py +++ b/test/connector/conftest.py @@ -11,6 +11,7 @@ from kumo.connector.databricks_connector import DatabricksConnector from kumo.connector.snowflake_connector import SnowflakeConnector +from kumo.testing._skip import requires_resource from kumo.testing.databricks import DatabricksTableConnector from kumo.testing.rest import ( financial_snowflake_config, @@ -22,6 +23,7 @@ @pytest.fixture(scope='session') +@requires_resource('aws') def require_kumo_public_datasets_s3() -> None: """Skip remote dataset tests when this environment lacks S3 access.""" import boto3 diff --git a/test/connector/test_databricks_connector.py b/test/connector/test_databricks_connector.py index 700d30757..0138182cf 100644 --- a/test/connector/test_databricks_connector.py +++ b/test/connector/test_databricks_connector.py @@ -31,6 +31,7 @@ _HOST = os.environ.get('DATABRICKS_HOST', 'workspace.cloud.databricks.com') +# TODO: Use a Databricks volume for seed staging to remove the AWS dependency. _TEST_SEED_STAGING_PATH = os.environ.get( 'VOLUME_MOUNT_PATH', 's3://kumo-preprod-dataplane-bucket/databricks-integ-test-staging/') @@ -301,6 +302,7 @@ def test_databricks_table_info(databricks_hm_connector: DatabricksConnector): assert connector._get_table_source_size('transactions') == 776368084 +@pytest.mark.private_resource('aws') @withDatabricksOauthCredentials def test_databricks_write_df_and_write_artifacts_to_table( writable_databricks_table: DatabricksTableConnector): diff --git a/test/io_utils/fsutil/local/test_s3.py b/test/io_utils/fsutil/local/test_s3.py index 42fb860ba..fad5e2abf 100644 --- a/test/io_utils/fsutil/local/test_s3.py +++ b/test/io_utils/fsutil/local/test_s3.py @@ -211,7 +211,7 @@ def test_zip_files_passes_exact_keys_without_prefix_listing(monkeypatch): ) -def test_s3_credentials_decorator_does_not_probe_provider(monkeypatch): +def test_s3_credentials_decorator_does_not_probe_provider(monkeypatch, tmp_path): credential_env = ( 'AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY', @@ -229,6 +229,8 @@ def test_s3_credentials_decorator_does_not_probe_provider(monkeypatch): def test_target(): pass + # An existing developer credential file must not affect this no-credentials case. + monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(tmp_path / 'missing')) with patch.object(s3_filesystem.threadsafe_boto, 's3_client') as client: list_objects = client.return_value.list_objects_v2 decorated = withS3Credentials(test_target) diff --git a/test/io_utils/fsutil/test_all.py b/test/io_utils/fsutil/test_all.py index f2b661d39..c4c55233d 100644 --- a/test/io_utils/fsutil/test_all.py +++ b/test/io_utils/fsutil/test_all.py @@ -50,10 +50,14 @@ def root_ucv(temp_ucv) -> Generator[str, None, None]: @pytest.fixture( params=[ - "root_s3", - "root_gcs", - "root_ucv", - "root_adls", + pytest.param('root_s3', marks=pytest.mark.private_resource('aws')), + pytest.param('root_gcs', marks=pytest.mark.private_resource('gcp')), + pytest.param( + 'root_ucv', marks=[ + pytest.mark.private_resource('databricks'), + pytest.mark.ci_profile('databricks-pat'), + ]), + pytest.param('root_adls', marks=pytest.mark.private_resource('azure')), ], ids=["s3", "gcs", "ucv", "adls"], ) diff --git a/test/io_utils/fsutil/test_fsutil.py b/test/io_utils/fsutil/test_fsutil.py index b64500e9a..453f13b75 100644 --- a/test/io_utils/fsutil/test_fsutil.py +++ b/test/io_utils/fsutil/test_fsutil.py @@ -8,17 +8,14 @@ import pytest from io_utils import fsutil +from kumo.testing.s3 import withS3Credentials @pytest.fixture +@withS3Credentials def s3_scratch_prefix(): - """Unique S3 prefix per test, deleted after yield. Skips if no - AWS creds — hand-rolled to keep this fixture independent of REST app - initialization. - """ + """Unique S3 prefix per test, deleted after yield.""" import uuid - if not fsutil.exists('s3://kumo-public-datasets/financial/csv/'): - pytest.skip('No AWS credentials found for S3') prefix = ('s3://kumo-test-dataplane-bucket/fsutil-test/' f'copy_tree_threaded_{uuid.uuid4().hex[:12]}') yield prefix diff --git a/test/io_utils/snowflake/test_session.py b/test/io_utils/snowflake/test_session.py index 8c03b3ab4..0f28650df 100644 --- a/test/io_utils/snowflake/test_session.py +++ b/test/io_utils/snowflake/test_session.py @@ -2,11 +2,14 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import pytest + import io_utils.snowflake.session from io_utils.snowflake.session import snowpark_session_in_app_schema from kumo.testing.snowflake import setup_full_snowflake_session +@pytest.mark.private_resource('snowflake') def test_schema_no_change(monkeypatch, tempdir): session = setup_full_snowflake_session(tempdir) original_database = session.get_current_database() @@ -32,6 +35,7 @@ def test_schema_no_change(monkeypatch, tempdir): assert session.get_current_schema() == original_schema +@pytest.mark.private_resource('snowflake') def test_schema_change(monkeypatch, tempdir): session = setup_full_snowflake_session(tempdir) original_database = session.get_current_database() diff --git a/test/io_utils/snowflake/test_session_logging.py b/test/io_utils/snowflake/test_session_logging.py index 624101146..4b485a057 100644 --- a/test/io_utils/snowflake/test_session_logging.py +++ b/test/io_utils/snowflake/test_session_logging.py @@ -20,6 +20,7 @@ from kumo.testing.snowflake import setup_full_snowflake_session +@pytest.mark.private_resource('snowflake') def test_logged_connector_cursor_execute_logs_and_query_id(tempdir, caplog): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -36,6 +37,7 @@ def test_logged_connector_cursor_execute_logs_and_query_id(tempdir, caplog): assert "query_id=" in caplog.text +@pytest.mark.private_resource('snowflake') def test_logged_connector_cursor_execute_logs_error(tempdir, caplog): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -51,6 +53,7 @@ def test_logged_connector_cursor_execute_logs_error(tempdir, caplog): assert "SELECT * FROM definitely_missing_table" not in caplog.text +@pytest.mark.private_resource('snowflake') def test_long_running_query_operator_stats_report_uses_threshold_override( tempdir, caplog, monkeypatch): session = setup_full_snowflake_session(tempdir) @@ -72,6 +75,7 @@ def test_long_running_query_operator_stats_report_uses_threshold_override( assert any("operator_count=" in rec.message for rec in caplog.records) +@pytest.mark.private_resource('snowflake') def test_logged_connector_connection_wraps_commit_rollback_close( tempdir, caplog): session = setup_full_snowflake_session(tempdir) @@ -90,6 +94,7 @@ def test_logged_connector_connection_wraps_commit_rollback_close( assert "operation=connection.close status=success" in caplog.text +@pytest.mark.private_resource('snowflake') def test_ensure_logged_connector_connection_idempotent(tempdir): session = setup_full_snowflake_session(tempdir) wrapped_session = ensure_logged_snowpark_session(session) @@ -99,6 +104,7 @@ def test_ensure_logged_connector_connection_idempotent(tempdir): assert wrapped is wrapped_again +@pytest.mark.private_resource('snowflake') def test_logged_snowpark_session_connection_is_wrapped(tempdir): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -110,6 +116,7 @@ def test_logged_snowpark_session_connection_is_wrapped(tempdir): assert isinstance(conn_2, _LoggedSnowflakeConnection) +@pytest.mark.private_resource('snowflake') def test_wrapped_session_compatible_with_snowpark_dataframe(tempdir): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -122,6 +129,7 @@ def test_wrapped_session_compatible_with_snowpark_dataframe(tempdir): assert isinstance(df.session, Session) +@pytest.mark.private_resource('snowflake') def test_wrapped_session_logs_query_id_for_dataframe_collect(tempdir, caplog): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -133,6 +141,7 @@ def test_wrapped_session_logs_query_id_for_dataframe_collect(tempdir, caplog): assert any("query_id=" in rec.message for rec in caplog.records) +@pytest.mark.private_resource('snowflake') def test_wrapped_session_file_put_logs_cursor_execute(tempdir, tmp_path, caplog): session = setup_full_snowflake_session(tempdir) @@ -328,6 +337,7 @@ def cursor(self): assert 'stats-secret-canary' not in caplog.text +@pytest.mark.private_resource('snowflake') def test_logged_cursor_ineligible_for_retry_after_multi_statement( tempdir, monkeypatch, caplog): monkeypatch.setattr(session_logging, "_QUERY_RETRY_TRIES", 3) diff --git a/test/io_utils/test_json.py b/test/io_utils/test_json.py index 7ea7cb768..a9eb6f4c3 100644 --- a/test/io_utils/test_json.py +++ b/test/io_utils/test_json.py @@ -256,6 +256,7 @@ def test_json_safe_float(): assert json_safe_float_array([4, float('inf')]) == [4.0, None] +@pytest.mark.private_resource('aws') def test_json_load_no_such_key(): path = "s3://kumo-public-datasets/something_not_exist" assert load_json(path, str, default="not_exists", @@ -283,6 +284,7 @@ def _raise_if_read(*args, **kwargs): assert read_called is False +@pytest.mark.private_resource('aws') def test_json_load_zipfile(): path = "s3://kumo-public-datasets/test_zip_file/" \ "materialization_task_info.json.gz" diff --git a/test/io_utils/test_parquet_util.py b/test/io_utils/test_parquet_util.py index e6b15350b..b5b625bc0 100644 --- a/test/io_utils/test_parquet_util.py +++ b/test/io_utils/test_parquet_util.py @@ -305,6 +305,7 @@ def test_count_rows_paritions(tmp_path: pathlib.Path): assert info.num_rows == 10 +@pytest.mark.private_resource('aws') def test_get_parquet_metadata_wildcard(tmp_path: pathlib.Path, deploy_env_config): # parquet_util.get_parquet_metadata on subsets of .parquet files in a @@ -1094,6 +1095,7 @@ def test_is_empty_helper_ucv(temp_ucv, tmp_path: pathlib.Path): assert parquet_util._is_empty_helper(empty_dir.as_url()) +@pytest.mark.private_resource('aws') def test_read_parquet_offset_size(): file_path = 's3://kumo-public-datasets/large_input_data/test1G128/' diff --git a/test/rest/config/test_artifact_export_file_output.py b/test/rest/config/test_artifact_export_file_output.py index 2a36297b7..8ff3238df 100644 --- a/test/rest/config/test_artifact_export_file_output.py +++ b/test/rest/config/test_artifact_export_file_output.py @@ -142,7 +142,9 @@ def test_prediction_artifact_preserves_multiple_destinations(): 'abfss://data@account.dfs.core.windows.net/output/'), ]) def test_destination_resolves_for_matching_connector_mode( - connector, config, expected): + connector, config, expected, monkeypatch): + monkeypatch.setattr( + 'io_utils.fsutil.s3_filesystem.select_s3_region', lambda path: None) assert resolve_file_output_path(config, connector) == expected diff --git a/test/rest/conftest.py b/test/rest/conftest.py index 390dd58b0..593de495e 100644 --- a/test/rest/conftest.py +++ b/test/rest/conftest.py @@ -263,7 +263,8 @@ def fixture_export_path( @pytest.fixture(scope="session") -def deploy_env_config(temporal_workflow_env) -> DeployEnvConfig: +def deploy_env_config(temporal_workflow_env, + test_execution_environment) -> DeployEnvConfig: os.environ['DEPLOY_STAGE'] = 'test' # The shared REST app is explicitly authentication-disabled. Worker-side # capability redemption reads this deployment setting independently of the diff --git a/test/rest/driver/test_pquery_api_driver.py b/test/rest/driver/test_pquery_api_driver.py index b388c7af9..185a229e1 100644 --- a/test/rest/driver/test_pquery_api_driver.py +++ b/test/rest/driver/test_pquery_api_driver.py @@ -3380,8 +3380,8 @@ def test_get_presigned_artifact_urls_artifact_type_not_found(artifact_type): @pytest.mark.parametrize('artifact_type', _ARTIFACT_TYPES) -def test_get_presigned_artifact_urls_file_not_found(artifact_type): - file_storage_path = 's3://kumo-public-datasets/db/rest' +def test_get_presigned_artifact_urls_file_not_found(artifact_type, tmp_path): + file_storage_path = str(tmp_path) pquery = _pquery_driver( ml_job_backend=_create_mock_ml_job_backend(file_storage_path), ) diff --git a/test/rest/integ/conftest.py b/test/rest/integ/conftest.py index 3c8589ca9..04d66fa1d 100644 --- a/test/rest/integ/conftest.py +++ b/test/rest/integ/conftest.py @@ -8,6 +8,7 @@ import sys import uuid from pathlib import Path +from test.ci_resources import execution_profile from typing import Dict, Generator, List import pytest @@ -226,7 +227,9 @@ def pquery_db_dump(request) -> FixtureDefinition: # --------------------------------------------------------------------------- -@pytest.fixture(scope="session", params=["local", "s3"]) +@pytest.fixture(scope="session", params=[ + "local", pytest.param("s3", marks=pytest.mark.private_resource('aws')), +]) def storage_backend(request): """Parametrize diskgraph tests over local and S3 storage backends.""" return request.param @@ -234,30 +237,13 @@ def storage_backend(request): @pytest.fixture(scope="session") def _diskgraph_deploy_config(tmp_path_factory, storage_backend, request, - temporal_workflow_env): - """Load diskgraph JSON config, override storage root, and set - DEPLOY_ENV_CONFIG_JSON so deploy_env() picks it up. - - DEPLOY_ENV_CONFIG_FILE must be set by the calling test module - (typically defaulted at module import time to - ``rest/devserver/configs/diskgraph-test.json``); CI sets it - explicitly via .github/workflows/testing.yml. - """ + temporal_workflow_env, test_execution_environment): + """Configure storage and workers for the selected DiskGraph backend.""" import json - # The fixture is parametrized over ``storage_backend`` (``local`` / - # ``s3``), so it runs once per parametrization. The first run pops - # ``DEPLOY_ENV_CONFIG_FILE`` after copying its contents into - # ``DEPLOY_ENV_CONFIG_JSON`` (see bottom of this fixture); the - # second invocation would otherwise hit the assertion below. - # Default it back to the diskgraph-test config if absent — same - # path the e2e test modules set at import time. - config_file = os.environ.get('DEPLOY_ENV_CONFIG_FILE') - if not config_file: - config_file = str( - Path(__file__).resolve().parents[3] / 'rest' / 'devserver' / - 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = config_file + config_file = os.environ.get('DEPLOY_ENV_CONFIG_FILE') or str( + Path(__file__).resolve().parents[3] / 'rest' / 'devserver' / + 'configs' / 'diskgraph-test.json') with open(config_file) as f: config = json.load(f) @@ -431,6 +417,7 @@ def _kill_pool_subprocesses(procs: List, log) -> None: @pytest.fixture(scope="session") +@execution_profile('diskgraph') def _pool_subprocesses(_diskgraph_deploy_config, request): """Spawn the pool controller AutoRPC server and a singlehost pool worker as long-running subprocesses for the duration of the session. diff --git a/test/rest/integ/fixture_connector_args.py b/test/rest/integ/fixture_connector_args.py index 3617df9cf..758d23ded 100644 --- a/test/rest/integ/fixture_connector_args.py +++ b/test/rest/integ/fixture_connector_args.py @@ -18,6 +18,8 @@ from kumoapi.source_table import S3RelativeSource from io_utils import fsutil +from io_utils.json import load_json +from kumo.testing.rest import randomize_fixture_ids_v2 from rest.config.api_command import FixtureDefinition from rest.config.connector import FileConnectorResourceConfig from rest.config.connector_db import InternalConnectorDBEntry @@ -31,6 +33,22 @@ 'connector_args/test_e2e_connector_args.json') +def load_selected_fixture(path: str | Path, + pquery_name: str) -> FixtureDefinition: + fixture = load_json(str(path), FixtureDefinition) + fixture.queries = [ + q for q in fixture.queries if q.pquery_name == pquery_name + ] + assert len(fixture.queries) == 1 + graph_ids = {query.graph_fpid for query in fixture.queries} + fixture.graphs = [ + graph for graph in fixture.graphs if graph.fpid in graph_ids + ] + assert len(fixture.graphs) == 1 + fixture.tables = list(fixture.graphs[0].tables.values()) + return randomize_fixture_ids_v2(fixture) + + def load_fixture_connectors( fixture: FixtureDefinition, project_id: ProjectID, @@ -247,6 +265,7 @@ async def provision_fixture_connectors( *, api_prefix: str = '/v0', fixture_root: Path | None = None, + selected_sources_only: bool = False, ) -> tuple[ProjectID, list[InternalConnectorDBEntry]]: """Upload checked-in local fixture data into the owning project. @@ -267,18 +286,38 @@ async def provision_fixture_connectors( } for old_connector_id, source_root in roots_by_connector.items(): if not Path(source_root).is_dir(): - raise ValueError( - f'Local fixture source directory does not exist: ' - f'{source_root}') + raise ValueError(f'Local fixture source directory does not exist: ' + f'{source_root}') namespace = posixpath.join('fixture-data', str(old_connector_id)) destination = posixpath.join(connector.config.root_dir.rstrip('/'), namespace) - await fsutil.copy.coroutine( - source_root, - destination, - overwrite=True, - raise_on_exists=False, - ) + if selected_sources_only: + source_connector = next(source for source in source_connectors + if source.id == old_connector_id) + resolver = dataclasses.replace( + source_connector.config, + name='fixture_source_resolver').as_connector() + sources = { + str(table.get_source_table_name()): table.source_table + for graph in fixture.graphs + for table in graph.tables.values() + if table.get_connector_id() == old_connector_id + } + for logical_path, source_table in sources.items(): + physical_path, suffix = await _physical_fixture_source( + resolver, source_table) + physical_destination = posixpath.join(destination, logical_path) + suffix + fsutil.makedirs(posixpath.dirname(physical_destination), exist_ok=True) + await fsutil.copy.coroutine( + physical_path, physical_destination, + overwrite=True, raise_on_exists=False) + else: + await fsutil.copy.coroutine( + source_root, + destination, + overwrite=True, + raise_on_exists=False, + ) remapped = _project_upload_fixture(fixture, connector.id, roots_by_connector) diff --git a/test/rest/integ/test_bigquery_diskgraph_e2e.py b/test/rest/integ/test_bigquery_diskgraph_e2e.py index 749b8d11a..94e086c0a 100644 --- a/test/rest/integ/test_bigquery_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_diskgraph_e2e.py @@ -23,7 +23,6 @@ --timeout=1800 -m bigquery """ import logging -import os from pathlib import Path from test.rest.integ.test_diskgraph_e2e import ( _check_diskgraph_train_table, @@ -51,12 +50,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks up the -# diskgraph config even when the env var isn't set by CI. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path( __file__).parent / "test_cases" / "test_bigquery_e2e_fixtures.json" @@ -67,8 +60,13 @@ # storage_backend is pinned to s3: BigQuery exports to GCS then downloads to # the dataplane, so local storage is not a meaningful test target here. pytestmark = [ + pytest.mark.ci_group('gcp-e2e'), + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py index 4f92ba144..b877f23bb 100644 --- a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py @@ -48,10 +48,9 @@ Prerequisites: - Temporal dev server: ``make up-local-temporal`` - ``diskgraph`` Python package installed (CI uses - ``install-diskgraph: 'true'`` in ``testing_gcs.yml``) + ``install-diskgraph: 'true'`` in ``testing_private.yml``) - ``DEPLOY_ENV_CONFIG_FILE`` pointing at a diskgraph deploy - config (CI sets it for the ``bigquery_*_diskgraph`` matrix - entries; this module also defaults it at import time) + config (the shared execution fixture supplies a default) - BigQuery credentials available via the ``withBigQueryCredentials`` decorator (from AWS Secrets Manager in CI, ``GOOGLE_CREDENTIALS_STR_JSON`` fallback @@ -65,7 +64,6 @@ -xvs --timeout=900 """ import logging -import os import shutil import time from pathlib import Path @@ -107,16 +105,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks -# up the diskgraph config even when the env var isn't set by CI. -# Doing this at module import (not in an autouse fixture) because -# ``deploy_env_config`` caches its DeployEnvConfig the first time -# any test in the session imports the conftest — see notes in -# ``test_diskgraph_e2e.py``. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) _CONNECTOR_NAME = "financial_bigquery_diskgraph" _PROJECT_ID = "kumo-engineering" @@ -138,6 +126,9 @@ # assertions are scoped to local storage; pinning avoids a second skipped S3 # parametrization and keeps the CI job from spawning an unused pool. pytestmark = [ + pytest.mark.ci_group('gcp-e2e'), + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), pytest.mark.parametrize("storage_backend", ["local"], indirect=True), ] diff --git a/test/rest/integ/test_data_variety/test_table_lifecycle.py b/test/rest/integ/test_data_variety/test_table_lifecycle.py index d3dbfba85..41722c375 100644 --- a/test/rest/integ/test_data_variety/test_table_lifecycle.py +++ b/test/rest/integ/test_data_variety/test_table_lifecycle.py @@ -2,9 +2,7 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import os from dataclasses import replace -from pathlib import Path from test.rest.conftest import TestClient from test.rest.integ import fixture_connector_args from typing import Any, Dict @@ -44,17 +42,6 @@ unsupported_parquet_args, ) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks up the -# diskgraph config even when the env var isn't set by CI. Mirrors the -# sibling DG ingestion e2e modules (test_snowflake_ingestion_diskgraph_e2e.py, -# test_bigquery_ingestion_diskgraph_e2e.py): set at module import (not in a -# fixture) because deploy_env_config caches its DeployEnvConfig the first -# time the session imports the conftest. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[4] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) - # Pool fixtures (storage_backend, _diskgraph_deploy_config, _pool_subprocesses) # live in test/rest/integ/conftest.py; the diskgraph per-table refresh # (TableRefreshWorkflow -> ingest_table_data_activity_v2) needs the pool @@ -64,7 +51,9 @@ # within one session trips a known harness deploy_env leak. pytestmark = [ pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/rest/integ/test_databricks_diskgraph_e2e.py b/test/rest/integ/test_databricks_diskgraph_e2e.py index 60a84a74d..d61b576fe 100644 --- a/test/rest/integ/test_databricks_diskgraph_e2e.py +++ b/test/rest/integ/test_databricks_diskgraph_e2e.py @@ -9,10 +9,8 @@ """ import asyncio import dataclasses -import os import time import uuid -from pathlib import Path from test.rest.integ.seed_helpers import seed_and_train import kumoapi.data_source as data_source_api @@ -63,11 +61,6 @@ TrainingJobID, ) -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str( - Path(__file__).resolve().parents[3] / 'rest' / 'devserver' / 'configs' / - 'diskgraph-test.json') - pytestmark = pytest.mark.usefixtures("_pool_subprocesses") _SOURCE_CONNECTOR = 'databricks_source' diff --git a/test/rest/integ/test_diskgraph_custom_tt_e2e.py b/test/rest/integ/test_diskgraph_custom_tt_e2e.py index 9e04fc661..e3012057e 100644 --- a/test/rest/integ/test_diskgraph_custom_tt_e2e.py +++ b/test/rest/integ/test_diskgraph_custom_tt_e2e.py @@ -71,13 +71,6 @@ log = logging.getLogger(__name__) -# Set DEPLOY_ENV_CONFIG_FILE at module import time so the deploy_env_config -# fixture picks up the diskgraph-test config before any fixture imports it -# (autouse fires too late). -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path(__file__).parent / "test_cases" / "test_e2e_fixtures.json" diff --git a/test/rest/integ/test_diskgraph_e2e.py b/test/rest/integ/test_diskgraph_e2e.py index d4b986df9..418db22c3 100644 --- a/test/rest/integ/test_diskgraph_e2e.py +++ b/test/rest/integ/test_diskgraph_e2e.py @@ -23,7 +23,6 @@ """ import logging import math -import os import os.path as osp import time from pathlib import Path @@ -72,20 +71,6 @@ log = logging.getLogger(__name__) -# Diskgraph deploy config is loaded from DEPLOY_ENV_CONFIG_FILE (set by -# .github/workflows/testing.yml). The autouse fixture below overrides -# ``data_plane_storage_root`` with either a local temp dir or an S3 prefix, -# controlled by the ``storage_backend`` parametrize fixture. -# -# Default DEPLOY_ENV_CONFIG_FILE at module import time so the -# ``deploy_env_config`` fixture (created by test/rest/conftest.py via -# ``DeployEnvConfig.for_test``) reads the diskgraph-test config — not -# the default in-memory test config. Doing this in an autouse fixture is too -# late: deploy_env_config has already cached its DeployEnvConfig by then. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path(__file__).parent / "test_cases" / "test_e2e_fixtures.json" diff --git a/test/rest/integ/test_diskgraph_public_api_e2e.py b/test/rest/integ/test_diskgraph_public_api_e2e.py index f237d52c3..33d06a19f 100644 --- a/test/rest/integ/test_diskgraph_public_api_e2e.py +++ b/test/rest/integ/test_diskgraph_public_api_e2e.py @@ -44,7 +44,10 @@ import os import time from pathlib import Path -from test.rest.integ.fixture_connector_args import provision_fixture_connectors +from test.rest.integ.fixture_connector_args import ( + load_selected_fixture, + provision_fixture_connectors, +) from test.rest.integ.public_response_reader import ( flatten_public_parquet_refs, read_public_response_bytes, @@ -70,11 +73,7 @@ from kumoapi.model_plan import RunMode from io_utils.json import load_json -from kumo.testing.rest import ( - get_and_expect_success, - post_and_expect_success, - randomize_fixture_ids_v2, -) +from kumo.testing.rest import get_and_expect_success, post_and_expect_success from rest.config.api_command import FixtureDefinition from rest.config.graph_v2 import GraphDefinition from rest.config.status import EmptyResponse, IDResponse @@ -88,16 +87,6 @@ # Constants — same dataset / expectations as test_diskgraph_e2e.py # --------------------------------------------------------------------------- -# Default DEPLOY_ENV_CONFIG_FILE at module import time so the -# ``deploy_env_config`` fixture (created by test/rest/conftest.py via -# ``DeployEnvConfig.for_test``) reads the diskgraph-test config rather -# than the default in-memory test config. Same pattern as -# test_diskgraph_e2e.py — must happen before any fixture imports -# deploy_env_config (autouse is too late; the cache fires first). -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path(__file__).parent / "test_cases" / "test_e2e_fixtures.json" @@ -107,7 +96,10 @@ # even though this file exercises the public API surface — workflow dispatch # routes through ``SnapshotActivities`` on the pool task queues, so the # subprocesses must be running for ingest/train/predict to make progress. -pytestmark = pytest.mark.usefixtures("_pool_subprocesses") +pytestmark = [ + pytest.mark.ci_group('aws-public-api'), + pytest.mark.usefixtures("_pool_subprocesses"), +] PQL_TEST_CASE = "saas_temporal_regression_base" @@ -213,8 +205,8 @@ async def _create_connectors_v1( fixture: FixtureDefinition, ) -> str: """Provision project-owned integration connectors.""" - project_id, _ = await provision_fixture_connectors(client, fixture, - api_prefix='/v1') + project_id, _ = await provision_fixture_connectors( + client, fixture, api_prefix='/v1', selected_sources_only=True) return str(project_id) @@ -663,17 +655,24 @@ async def test_diskgraph_decoupled_training_and_prediction( client = public_api_client # Load and randomize fixture - fixture = load_json(str(FIXTURE_PATH), FixtureDefinition) - fixture.queries = [ - q for q in fixture.queries if q.pquery_name == PQL_TEST_CASE - ] - assert len(fixture.queries) == 1 - fixture = randomize_fixture_ids_v2(fixture) + fixture = load_selected_fixture(FIXTURE_PATH, PQL_TEST_CASE) + + phase_started = time.monotonic() + previous_phase = 'setup' + + def _record_phase(next_phase: str) -> None: + nonlocal phase_started, previous_phase + now = time.monotonic() + print( + f"CI_PHASE public-api {previous_phase} elapsed_seconds={now - phase_started:.3f}", + flush=True) + previous_phase, phase_started = next_phase, now # ==== Phase 1: Setup ==== # SDK: connector = kumo.S3Connector(...); graph = kumo.Graph(...); # graph.save(); pquery = kumo.PredictiveQuery(graph, query); # pquery.save() + _record_phase("provision") log.info("Phase 1: Creating connectors, PQuery, Graph via V1") await _create_connectors_v1(client, fixture) pquery_id, graph_id, args = await _setup_pquery_and_graph(client, fixture) @@ -683,6 +682,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # SDK: graph.snapshot(force_refresh=True) — called explicitly or # implicitly by trainer.fit(). We create it here for stats validation # and reuse it in trainer.fit() / trainer.predict() below. + _record_phase("snapshot") log.info("Phase 2: Creating graph snapshot via V1") snapshot_id = GraphSnapshotID( post_and_expect_success( @@ -706,6 +706,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # SDK: pquery.generate_training_table(plan=plan) # Note: SDK passes graph_snapshot_id=None — server triggers its own # graph refresh internally. + _record_phase("training_table") log.info("Phase 3: Generating training table via V1") plan_payload = plan_api.SuggestModelPlanRequest( @@ -765,6 +766,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # trainer.fit(graph=graph, train_table=train_table) # Internally, fit() calls graph.snapshot() and passes the snapshot_id # along with train_table.job_id to TrainingJobRequest. + _record_phase("training_A") log.info("Phase 4a: Training with seed=42 via V1") model_plan_info = post_and_expect_success( @@ -838,6 +840,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # ==== Phase 4b: Training Job B (seed=123, reuse train table) ==== # SDK: Same trainer.fit() with different model_plan seed but same # train_table.job_id — both jobs share the same train table. + _record_phase("training_B") log.info("Phase 4b: Training with seed=123, reusing train table via V1") model_plan_b = copy.deepcopy(model_plan) @@ -882,6 +885,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # ==== Phase 5: Prediction Table Generation ==== # SDK: pquery.generate_prediction_table() # Note: SDK passes graph_snapshot_id=None — server handles graph refresh. + _record_phase("prediction_table") log.info("Phase 5: Generating prediction table via V1") pred_table_job_id = post_and_expect_success( @@ -922,6 +926,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # output_config=OutputConfig(...)) # Internally, predict() calls graph.snapshot() and passes snapshot_id + # pred_table_job_id to BatchPredictionRequest. + _record_phase("prediction_and_artifacts") log.info("Phase 6: Running batch prediction via V1") from kumo.validation import ValidationResponse @@ -966,6 +971,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # Verify prediction config has diskgraph backend (supplementary) _check_prediction_config_diskgraph(bp_job_id) + _record_phase("complete") log.info("Test 1 PASSED: full diskgraph decoupled flow via V1.") @@ -980,12 +986,7 @@ async def test_diskgraph_snapshot_reuse(public_api_client: TestClient, ): """Single graph snapshot reused for two train-table-gen jobs.""" client = public_api_client - fixture = load_json(str(FIXTURE_PATH), FixtureDefinition) - fixture.queries = [ - q for q in fixture.queries if q.pquery_name == PQL_TEST_CASE - ] - assert len(fixture.queries) == 1 - fixture = randomize_fixture_ids_v2(fixture) + fixture = load_selected_fixture(FIXTURE_PATH, PQL_TEST_CASE) await _create_connectors_v1(client, fixture) pquery_id, graph_id, args = await _setup_pquery_and_graph(client, fixture) @@ -1055,12 +1056,7 @@ async def test_diskgraph_graph_refresh_idempotency( """Re-creating a graph snapshot produces consistent stats.""" client = public_api_client - fixture = load_json(str(FIXTURE_PATH), FixtureDefinition) - fixture.queries = [ - q for q in fixture.queries if q.pquery_name == PQL_TEST_CASE - ] - assert len(fixture.queries) == 1 - fixture = randomize_fixture_ids_v2(fixture) + fixture = load_selected_fixture(FIXTURE_PATH, PQL_TEST_CASE) await _create_connectors_v1(client, fixture) _, graph_id, _ = await _setup_pquery_and_graph(client, fixture) diff --git a/test/rest/integ/test_fixture_connector_args.py b/test/rest/integ/test_fixture_connector_args.py index ddabcdfaa..edeaf892a 100644 --- a/test/rest/integ/test_fixture_connector_args.py +++ b/test/rest/integ/test_fixture_connector_args.py @@ -352,3 +352,39 @@ async def test_project_upload_fixture_is_hidden_on_private_and_public_routes( SourceTableListResponse, ) assert 'events' in source_tables.table_names + + +@pytest.mark.asyncio +async def test_public_api_fixture_uploads_only_its_resolved_input_bytes( + tmp_path, monkeypatch): + from test.rest.integ import fixture_connector_args as uploads + from test.rest.integ.fixture_connector_args import load_selected_fixture + + fixture = load_selected_fixture( + 'test/rest/integ/test_cases/test_e2e_fixtures.json', + 'saas_temporal_regression_base') + assert len(fixture.queries) == len(fixture.graphs) == 1 + assert set(fixture.graphs[0].tables) == {'ACCOUNT', 'LOAN', 'TRANS'} + assert len(fixture.tables) == 3 + connector = SimpleNamespace( + id=ConnectorID(f'connector_{"b" * 32}'), + config=SimpleNamespace(root_dir=str(tmp_path / 'uploads'))) + monkeypatch.setattr(uploads, 'ensure_test_project', + AsyncMock(return_value=ProjectID('default'))) + monkeypatch.setattr(uploads, '_canonical_project_upload_connector', + AsyncMock(return_value=connector)) + monkeypatch.setattr(uploads, 'bind_test_client_project', lambda *_: None) + await uploads.provision_fixture_connectors(None, fixture, + selected_sources_only=True) + copied = sorted(path for path in (tmp_path / 'uploads').rglob('*') + if path.is_file()) + assert sorted(path.name for path in copied) == [ + 'ACCOUNT.parquet', 'LOAN.parquet', 'TRANS.parquet' + ] + expected = { + 'ACCOUNT.parquet': Path('test/csv_data/FINANCIAL/ACCOUNT.parquet'), + 'TRANS.parquet': Path('test/csv_data/FINANCIAL/TRANS.parquet'), + 'LOAN.parquet': Path('test/csv_data/FINANCIAL_E2E/LOAN.parquet'), + } + for path in copied: + assert path.read_bytes() == expected[path.name].read_bytes() diff --git a/test/rest/integ/test_gcs_e2e.py b/test/rest/integ/test_gcs_e2e.py index 56f9d284a..bb226cd1b 100644 --- a/test/rest/integ/test_gcs_e2e.py +++ b/test/rest/integ/test_gcs_e2e.py @@ -66,6 +66,8 @@ # This suite covers GCS-connector e2e, not storage-backend coverage — # only test_diskgraph_e2e.py needs the [s3] parametrize. pytestmark = [ + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), pytest.mark.parametrize("storage_backend", ["local"], indirect=True), ] diff --git a/test/rest/integ/test_gcs_public_api_e2e.py b/test/rest/integ/test_gcs_public_api_e2e.py index b62a32462..930798f81 100644 --- a/test/rest/integ/test_gcs_public_api_e2e.py +++ b/test/rest/integ/test_gcs_public_api_e2e.py @@ -50,6 +50,8 @@ # (routed via ``DiskgraphIngestWorkflow`` -> ``pool.wf_coordinator``) hangs on a # no-worker AutoRPC queue and the workflow times out. pytestmark = [ + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), pytest.mark.parametrize("storage_backend", ["local"], indirect=True), ] diff --git a/test/rest/integ/test_snowflake_diskgraph_e2e.py b/test/rest/integ/test_snowflake_diskgraph_e2e.py index 2f5a81f97..cac7b16a7 100644 --- a/test/rest/integ/test_snowflake_diskgraph_e2e.py +++ b/test/rest/integ/test_snowflake_diskgraph_e2e.py @@ -53,12 +53,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks up the -# diskgraph config even when the env var isn't set by CI. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path( __file__).parent / "test_cases" / "test_snowflake_e2e_fixtures.json" @@ -69,8 +63,11 @@ # storage_backend is pinned to s3 to exercise Kumo-owned remote persistence # before the pool-local diskgraph download. pytestmark = [ + pytest.mark.ci_gate('snowflake-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py b/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py index 29bb03bf9..71fc8060e 100644 --- a/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py +++ b/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py @@ -58,7 +58,7 @@ Prerequisites: - Temporal dev server: ``make up-local-temporal`` - ``diskgraph`` Python package installed (CI uses - ``install-diskgraph: 'true'`` in ``testing_snowflake.yml``) + ``install-diskgraph: 'true'`` in ``testing_private.yml``) - ``DEPLOY_ENV_CONFIG_FILE`` pointing at a diskgraph deploy config - Snowflake credentials available via ``withSnowflakeCredentials`` (AWS Secrets Manager ``xva19026.snowflakecomputing.com`` or @@ -77,7 +77,6 @@ import os import time from decimal import Decimal -from pathlib import Path from test.rest.integ.test_diskgraph_e2e import ( MAX_POLL_SECONDS, POLL_INTERVAL_SECONDS, @@ -130,13 +129,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks -# up the diskgraph config even when the env var isn't set by CI. -# Mirrors test_snowflake_diskgraph_e2e.py. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) _CONNECTOR_NAME = "financial_snowflake_diskgraph" @@ -162,8 +154,11 @@ # parametrization hits a kumo-side stats persistence issue # unrelated to the Snowflake connector path. pytestmark = [ + pytest.mark.ci_gate('snowflake-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/scripts/test_ci_resources.py b/test/scripts/test_ci_resources.py new file mode 100644 index 000000000..232644214 --- /dev/null +++ b/test/scripts/test_ci_resources.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise real pytest collection/fixture ordering without cloud access.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + + +def run_pytest(tmp_path, source, group=None, strict=False, collect=False): + test = tmp_path / 'test_selection.py' + test.write_text(source) + manifest = tmp_path / f'{group or "all"}.json' + command = [ + sys.executable, '-m', 'pytest', '-c', '/dev/null', '--noconftest', '-p', + 'test.ci_resources', '-q', '--resource-manifest', + str(manifest), + str(test) + ] + if group: + command += ['--resource-group', group] + if strict: + command += ['--require-resource-credentials'] + if collect: + command += ['--collect-only'] + result = subprocess.run( + command, capture_output=True, text=True, timeout=45, env={ + **os.environ, 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1' + }) + return result, json.loads(manifest.read_text()) + + +def test_public_selection_never_resolves_private_fixture(tmp_path): + source = ''' +import pytest +import runpy +requires_resource = runpy.run_path('kumo/testing/_skip.py')['requires_resource'] + +@pytest.fixture +@requires_resource('gcp') +def private_input(): + raise AssertionError('Private fixture ran on a public runner') + +@pytest.fixture +def indirect_input(private_input): + return private_input + +def test_local(): + pass + +@pytest.mark.private_resource('aws') +@pytest.mark.ci_group('aws-public-api') +def test_balanced(): + pass + +@pytest.mark.parametrize('storage_backend', [ + pytest.param('s3', marks=pytest.mark.private_resource('aws'))]) +def test_live(indirect_input, storage_backend): + pass + +@pytest.mark.parametrize('storage_backend', [ + 'local', pytest.param('s3', marks=pytest.mark.private_resource('aws'))]) +def test_backend(storage_backend): + assert storage_backend == 'local' +''' + result, public = run_pytest(tmp_path, source, 'public') + assert result.returncode == 0, result.stdout + result.stderr + assert len(public) == 2 + assert all(not value['resources'] for value in public.values()) + _, complete = run_pytest(tmp_path, source, collect=True) + _, aws = run_pytest(tmp_path, source, 'aws', collect=True) + _, gcp = run_pytest(tmp_path, source, 'gcp', collect=True) + _, balanced = run_pytest(tmp_path, source, 'aws-public-api', collect=True) + assert len(balanced) == 1 + assert set(complete) == set(public) | set(aws) | set(gcp) | set(balanced) + assert sum(map(len, (public, aws, gcp, balanced))) == len(complete) + assert not (set(public) & (set(aws) | set(gcp))) + assert not (set(aws) & set(gcp)) + assert [row['resources'] for row in gcp.values()] == [['aws', 'gcp']] + + +def test_private_ci_rejects_missing_credentials_but_preserves_intentional_skips( + tmp_path): + source = ''' +import pytest +@pytest.mark.private_resource('aws') +def test_missing(): + pytest.skip('No AWS credentials found for S3') +''' + result, _ = run_pytest(tmp_path, source, 'aws', strict=True) + assert result.returncode == 1, result.stdout + result.stderr + assert 'Private CI skipped tests' in result.stdout + source = source.replace('No AWS credentials found for S3', + 'Only run in the full suite') + result, _ = run_pytest(tmp_path, source, 'aws', strict=True) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_discovery_runs_declared_profiles_in_fresh_processes(tmp_path): + component = tmp_path / 'test' / 'new_component' + component.mkdir(parents=True) + helper = repr(str(Path('kumo/testing/_skip.py').resolve())) + source = f''' +import os +import runpy +import pytest +from test.ci_resources import execution_profile +requires_resource = runpy.run_path({helper})['requires_resource'] +IMPORTED_AUTH = {{key: os.environ.get(key) for key in ( + 'DATABRICKS_TOKEN', 'DATABRICKS_CLIENT_ID', 'DATABRICKS_CLIENT_SECRET')}} + +@pytest.fixture(scope='session') +@execution_profile('diskgraph') +def process_state(): + assert not os.path.exists(os.environ['PROFILE_PROCESS_STATE']) + with open(os.environ['PROFILE_PROCESS_STATE'], 'w') as state: + state.write(str(os.getpid())) + yield + os.unlink(os.environ['PROFILE_PROCESS_STATE']) + +@requires_resource('databricks', profile='databricks-pat') +def test_pat(process_state): + assert IMPORTED_AUTH == {{'DATABRICKS_TOKEN': 'fake-pat', + 'DATABRICKS_CLIENT_ID': '', + 'DATABRICKS_CLIENT_SECRET': ''}} + +@requires_resource('databricks', profile='databricks-oauth') +def test_oauth(process_state): + assert IMPORTED_AUTH == {{'DATABRICKS_TOKEN': '', + 'DATABRICKS_CLIENT_ID': 'fake-client', + 'DATABRICKS_CLIENT_SECRET': 'fake-secret'}} +''' + for name in ('one', 'two'): + (component / f'test_{name}.py').write_text(source) + (component / 'test_gcp.py').write_text(''' +import os +import pytest +IMPORTED_GCP = os.environ.get('IS_GCP_DEPLOY') + +@pytest.mark.private_resource('gcp') +@pytest.mark.ci_profile('gcp') +@pytest.mark.parametrize('storage_backend', [ + pytest.param('s3', marks=pytest.mark.private_resource('aws'))]) +def test_combined_identity(storage_backend): + assert IMPORTED_GCP == '1' + +@pytest.mark.private_resource('gcp') +@pytest.mark.ci_gate('gcp-e2e') +def test_gated(): + raise AssertionError('Disabled provider suite ran') +''') + # Unsupported release/script entry points must not enter component discovery. + (tmp_path / 'test' / 'test_script.py').write_text('raise RuntimeError()') + release = tmp_path / 'test' / 'release_tests' + release.mkdir() + (release / 'test_release.py').write_text('raise RuntimeError()') + env = { + **os.environ, + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': + '1', + 'PYTHONPATH': + os.pathsep.join([ + str(Path(__file__).resolve().parents[2]), + os.environ.get('PYTHONPATH', '') + ]), + 'PYTEST_ADDOPTS': + '--noconftest -c /dev/null -p test.ci_resources -p pytest_timeout', + 'DATABRICKS_TOKEN': + 'fake-pat', + 'DATABRICKS_CLIENT_ID': + 'fake-client', + 'DATABRICKS_CLIENT_SECRET': + 'fake-secret', + 'IS_GCP_DEPLOY': + '0', + 'CI_RUN_GCP_E2E': + 'false', + 'PROFILE_PROCESS_STATE': + str(tmp_path / 'process-state'), + } + for group, sessions, cases in [('databricks', 4, 4), ('gcp', 1, 1)]: + output = tmp_path / group + result = subprocess.run( + [sys.executable, '-m', 'tools.run_private_tests', group], + cwd=tmp_path, env={ + **env, 'CI_TEST_RESULTS': str(output) + }, capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stdout + result.stderr + discovered = json.loads((output / 'discovery.json').read_text()) + selections = [ + json.loads(path.read_text()) + for path in output.glob('*-selection.json') + ] + assert len(selections) == sessions + assert len(discovered) == cases + assert discovered == { + key: value + for part in selections + for key, value in part.items() + } + assert len(list(output.glob('*.xml'))) == sessions + if group == 'gcp': + assert [row['resources'] + for row in discovered.values()] == [['aws', 'gcp']] + + +def test_phase_timings_keep_only_safe_fields(tmp_path): + source = """ +def test_phase(): + print('CI_PHASE public-api provision elapsed_seconds=1.234https://example.test/data?X-Amz-Signature=sensitive') +""" + result, _ = run_pytest(tmp_path, source, 'public') + assert result.returncode == 0, result.stdout + result.stderr + assert 'CI_PHASE public-api provision elapsed_seconds=1.234' in result.stdout + assert 'X-Amz-Signature' not in result.stdout + + +def test_private_output_redacts_download_queries(monkeypatch): + from tools.run_private_tests import redact_download_urls + monkeypatch.setenv('DATABRICKS_CLIENT_SECRET', 'synthetic-private-secret') + text = 'download https://example.test/data?sig=sensitive&token=value end' + assert redact_download_urls( + text) == 'download https://example.test/data?REDACTED end' + assert redact_download_urls( + 'secret synthetic-private-secret') == 'secret REDACTED' + + +def test_provider_fixture_changes_keep_e2e_enabled(): + from tools.ci_private_changes import selection + matrix, gcp, _ = selection( + ['test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json']) + assert gcp + assert {'group': 'gcp-e2e'} in matrix['include'] + matrix, _, _ = selection(['docs/connector-test-matrix.md']) + assert matrix == {'include': [{'group': 'none'}]} + matrix, gcp, snowflake = selection(['README.md'], force=True) + assert len(matrix['include']) == 7 + assert gcp and snowflake diff --git a/tools/ci_private_changes.py b/tools/ci_private_changes.py new file mode 100644 index 000000000..aca158c1c --- /dev/null +++ b/tools/ci_private_changes.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Choose private jobs from the approved PR's complete changed-file list.""" + +import fnmatch +import json +import os + + +def matches(files, patterns): + return any( + fnmatch.fnmatchcase(path, pattern) for path in files + for pattern in patterns) + + +def selection(files, force=False): + code = force or any( + not matches([path], ('ui/**', 'docs/**', 'helm/**', 'tools/**', + 'benchmark/**', '*.md')) or path in + ('tools/run_private_tests.py', 'tools/ci_private_changes.py') + for path in files) + shared = ('.github/actions/setup/**', '.github/scripts/copied-pr.cjs', + '.github/workflows/testing_private.yml', 'test/ci_resources.py', + 'kumo/testing/**', 'setup.py', 'uv.lock', + 'tools/run_private_tests.py', 'tools/ci_private_changes.py') + gcp = force or matches( + files, (*shared, 'kumo/connector/**', 'io_utils/**', 'rest/**', + 'packages/kumo-core/**', 'test/rest/integ/test_gcs*', + 'test/rest/integ/test_bigquery*', + 'test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json')) + snowflake = force or matches( + files, (*shared, 'kumo/connector/**', + 'rest/backends/data_backend/activities/**', + 'test/rest/integ/test_snowflake*', + 'test/rest/integ/test_cases/test_snowflake_e2e_fixtures.json', + 'rest/devserver/configs/diskgraph-test.json')) + groups = [ + 'aws', 'aws-public-api', 'gcp', 'azure', 'snowflake', 'databricks' + ] if code else [] + if code and gcp: + groups.append('gcp-e2e') + # A sentinel avoids GitHub's invalid empty matrix; the test job skips it. + matrix = { + 'include': [{ + 'group': group + } for group in groups] or [{ + 'group': 'none' + }] + } + return matrix, gcp, snowflake + + +if __name__ == '__main__': + matrix, gcp, snowflake = selection( + json.loads(os.environ['CHANGED_FILES']), + os.environ.get('FORCE_PRIVATE_TESTS') == 'true') + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'matrix={json.dumps(matrix)}\n') + output.write( + f'run={str(matrix["include"][0]["group"] != "none").lower()}\n') + output.write(f'gcp-e2e={str(gcp).lower()}\n') + output.write(f'snowflake-e2e={str(snowflake).lower()}\n') diff --git a/tools/run_private_tests.py b/tools/run_private_tests.py new file mode 100644 index 000000000..9ce2b0d8c --- /dev/null +++ b/tools/run_private_tests.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Discover private cases and run their declared profiles in fresh processes.""" + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from test.ci_resources import GROUPS, PROFILE_ENV + + +def redact_download_urls(text): + for name, value in os.environ.items(): + if value and (name.endswith( + ('_TOKEN', '_SECRET', '_SECRET_ACCESS_KEY', '_PASSWORD', + '_PRIVATE_KEY')) or name == 'AWS_ACCESS_KEY_ID'): + text = text.replace(value, 'REDACTED') + return re.sub( + r"https?://[^\s'\"<>]+", + lambda match: match[0].split('?', 1)[0] + '?REDACTED' + if '?' in match[0] else match[0], text) + + +def run_pytest(args, env, output, name, *, coverage=False): + command = [sys.executable, '-X', 'faulthandler', '-m'] + if coverage: + command += [ + 'coverage', 'run', '--data-file', + str(output / f'{name}.coverage'), '-m' + ] + command += [ + 'pytest', '--rootdir=.', '--capture=fd', '--verbosity=0', + '--disable-warnings', *args + ] + with (output / f'{name}.log').open('w') as log: + process = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + for line in process.stdout: + line = redact_download_urls(line) + print(line, end='', flush=True) + log.write(line) + result = process.wait() + junit = output / f'{name}.xml' + if junit.exists(): + junit.write_text(redact_download_urls(junit.read_text())) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('group', choices=GROUPS[1:]) + parser.add_argument('paths', nargs='*', + help='Override component collection roots') + parser.add_argument('--collect-only', action='store_true') + args = parser.parse_args() + output = Path(os.environ.get('CI_TEST_RESULTS', 'ci-test-results')) + output.mkdir(parents=True, exist_ok=True) + # Hosted component suites are subdirectories of test/. Root-level script + # tests and release tooling have separate entry points and prerequisites. + roots = args.paths or [ + str(path) for path in sorted(Path('test').iterdir()) if path.is_dir() + and not path.name.startswith('_') and path.name != 'release_tests' + ] + discovery = output / 'discovery.json' + result = run_pytest([ + *roots, '--collect-only', '--resource-group', args.group, + '--private-discovery', '--resource-manifest', + str(discovery) + ], os.environ.copy(), output, 'discovery') + if result: + return result + sessions = {} + for nodeid, metadata in json.loads(discovery.read_text()).items(): + key = (tuple(metadata['profiles']), metadata['isolated_module']) + sessions.setdefault(key, {})[nodeid] = metadata + if not sessions: + raise RuntimeError( + f'No tests discovered for private group {args.group}') + failed = False + for index, ((profiles, module), + expected) in enumerate(sorted(sessions.items())): + name = f'{args.group}-{index}' + selection = output / f'{name}-selection.json' + selection.write_text(json.dumps(expected, indent=2) + '\n') + print(f'CI_SELECTION {name} profiles={profiles} module={module}', + flush=True) + if args.collect_only: + continue + env = os.environ.copy() + for profile in profiles: + env.update(PROFILE_ENV[profile]) + failed |= run_pytest([ + *expected, + '--resource-group', + args.group, + '--require-resource-credentials', + '--resource-manifest', + str(selection), + '--junitxml', + str(output / f'{name}.xml'), + '--durations=0', + '--timeout=1200', + '--timeout-method=signal', + ], env, output, name, coverage=True) != 0 + if json.loads(selection.read_text()) != expected: + print( + f'CI selection changed between discovery and execution: {name}', + flush=True) + failed = True + return int(failed) + + +if __name__ == '__main__': + raise SystemExit(main())