From 8b24cd4b632eb2a7738a2087125ba5518cd17865 Mon Sep 17 00:00:00 2001 From: Aaron Braunstein Date: Tue, 20 Oct 2026 16:05:05 +0000 Subject: [PATCH 1/3] Separate public CI from private resource tests --- .github/workflows/install.yml | 16 +- .github/workflows/testing.yml | 68 +---- .github/workflows/testing_databricks.yml | 193 -------------- .github/workflows/testing_gcs.yml | 154 ----------- .github/workflows/testing_private.yml | 204 +++++++++++++++ .github/workflows/testing_snowflake.yml | 224 ---------------- .github/workflows/ui-client-code-generate.yml | 26 -- docs/connector-test-matrix.md | 18 +- kumo/testing/_skip.py | 15 ++ kumo/testing/adls.py | 2 + kumo/testing/bigquery.py | 3 + kumo/testing/databricks.py | 6 +- kumo/testing/gcs.py | 2 + kumo/testing/s3.py | 5 +- kumo/testing/snowflake.py | 4 +- test/ci_resources.py | 188 ++++++++++++++ test/conftest.py | 48 +++- test/connector/conftest.py | 2 + test/connector/test_databricks_connector.py | 2 + test/io_utils/fsutil/local/test_s3.py | 4 +- test/io_utils/fsutil/test_all.py | 12 +- test/io_utils/fsutil/test_fsutil.py | 9 +- test/io_utils/snowflake/test_session.py | 4 + .../snowflake/test_session_logging.py | 10 + test/io_utils/test_json.py | 2 + test/io_utils/test_parquet_util.py | 2 + .../test_artifact_export_file_output.py | 4 +- test/rest/conftest.py | 3 +- test/rest/driver/test_pquery_api_driver.py | 4 +- test/rest/integ/conftest.py | 33 +-- test/rest/integ/fixture_connector_args.py | 57 ++++- .../rest/integ/test_bigquery_diskgraph_e2e.py | 14 +- .../test_bigquery_ingestion_diskgraph_e2e.py | 19 +- .../test_data_variety/test_table_lifecycle.py | 17 +- .../integ/test_databricks_diskgraph_e2e.py | 7 - .../integ/test_diskgraph_custom_tt_e2e.py | 7 - test/rest/integ/test_diskgraph_e2e.py | 15 -- .../integ/test_diskgraph_public_api_e2e.py | 70 +++-- .../rest/integ/test_fixture_connector_args.py | 36 +++ test/rest/integ/test_gcs_e2e.py | 2 + test/rest/integ/test_gcs_public_api_e2e.py | 2 + .../integ/test_snowflake_diskgraph_e2e.py | 11 +- .../test_snowflake_ingestion_diskgraph_e2e.py | 15 +- test/scripts/test_ci_resources.py | 241 ++++++++++++++++++ tools/ci_private_changes.py | 63 +++++ tools/run_private_tests.py | 118 +++++++++ 46 files changed, 1102 insertions(+), 859 deletions(-) delete mode 100644 .github/workflows/testing_databricks.yml delete mode 100644 .github/workflows/testing_gcs.yml create mode 100644 .github/workflows/testing_private.yml delete mode 100644 .github/workflows/testing_snowflake.yml create mode 100644 test/ci_resources.py create mode 100644 test/scripts/test_ci_resources.py create mode 100644 tools/ci_private_changes.py create mode 100644 tools/run_private_tests.py diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 11fc6e8bf..8567ba1ff 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -19,15 +19,10 @@ concurrency: permissions: contents: read - id-token: write jobs: import: - environment: gha-kumo-workflow-tests runs-on: ubuntu-24.04 - env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - steps: - name: Checkout repository and submodules uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -42,25 +37,18 @@ jobs: files: | ui/** - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python if: steps.ui-changes.outputs.only_changed != 'true' uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: '3.14' activate-environment: true - - name: Configure AWS credentials - if: steps.ui-changes.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Install Kumo package if: steps.ui-changes.outputs.only_changed != 'true' uses: ./.github/actions/setup with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} + github_token: ${{ github.token }} - name: Test imports if: steps.ui-changes.outputs.only_changed != 'true' diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ebe3e8da4..a8452e648 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -41,10 +41,8 @@ concurrency: permissions: contents: read - id-token: write env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} GITHUB_REF: ${{secrets.GITHUB_REF}} GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} @@ -79,24 +77,8 @@ jobs: needs: [diskgraph-wheel] if: ${{ !cancelled() }} runs-on: ubuntu-24.04 - continue-on-error: true - environment: - name: azure-adls-tests - # This job uses environment-scoped credentials for tests; it does not - # deploy an application. Avoid one deployment record per matrix shard. - deployment: false permissions: contents: read - id-token: write - env: - KUMO_RUN_ADLS_TESTS: ${{ github.repository == 'kumo-ai/kumo' && contains(matrix.test_dir, 'test/io_utils/') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }} - AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} strategy: fail-fast: false @@ -118,7 +100,7 @@ jobs: # connection (devapp AWS SM creds or TEMPORAL_SERVER=cloud); it # has no local-dev-server fallback, so it can't run in this matrix. "test/pool --ignore=test/pool/test_provision_terminal_nonretry.py", - "test/api_contract test/release_tests/test_build_kumo_images.py test/release_tests/test_check_updated_release_branches.py test/release_tests/test_make_kumo3_hotfix_release.py test/release_tests/test_release_v3_guards.py test/release_tests/test_start_kumo3_release.py", + "test/api_contract test/scripts/test_ci_resources.py test/release_tests/test_build_kumo_images.py test/release_tests/test_check_updated_release_branches.py test/release_tests/test_make_kumo3_hotfix_release.py test/release_tests/test_release_v3_guards.py test/release_tests/test_start_kumo3_release.py", # Public API tests "-k graph-saas_static_multiclassclassification_base test/rest/integ/test_e2e_public_api.py", @@ -136,23 +118,18 @@ jobs: # Other REST tests (public and integ should always be separated) "test/rest/integ/test_graph_validation.py", "test/rest/auth/ test/rest/public/", - "test/rest/integ/black_box/test_public_api_key_auth_flow.py test/rest/integ/test_ml_backend.py test/rest/integ/test_schema.py test/rest/integ/test_job_tags.py", + "test/rest/integ/black_box/test_public_api_key_auth_flow.py test/rest/integ/test_ml_backend.py test/rest/integ/test_schema.py test/rest/integ/test_job_tags.py test/rest/integ/test_fixture_connector_args.py", "test/rest/integ/test_warm_start_training_e2e.py", "test/rest/integ/test_ui_to_sdk_e2e.py", "test/rest/backends test/rest/config test/rest/driver test/rest/routers test/rest/utils test/rest/uxrt", "test/temporalio_commons", "test/docker", "test/deploy/test_cloud_storage_preflight.py", - "test/rest/integ/test_data_variety/test_table_lifecycle.py", - # GCS e2e tests moved to testing_gcs.yml (requires IS_GCP_DEPLOY=1) "--splits 2 --group 1 test/rest/test_*.py", "--splits 2 --group 2 test/rest/test_*.py", "-k '[local]' test/rest/integ/test_diskgraph_e2e.py", - "-k '[s3]' test/rest/integ/test_diskgraph_e2e.py", # needs S3 dataplane "-k '[local]' test/rest/integ/test_diskgraph_public_api_e2e.py", - "-k '[s3]' test/rest/integ/test_diskgraph_public_api_e2e.py", # needs S3 dataplane "-k '[local]' test/rest/integ/test_diskgraph_custom_tt_e2e.py", - "-k '[s3]' test/rest/integ/test_diskgraph_custom_tt_e2e.py", # needs S3 dataplane "test/rest/integ/test_weighted_table_e2e.py", # custom-TT REST flow (unparked from test/legacy/) "test/statedb", ] @@ -194,37 +171,14 @@ jobs: tools/** benchmark/** - - name: Configure AWS credentials - if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up packages if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' uses: ./.github/actions/setup with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} + github_token: ${{ github.token }} diskgraph-wheel-artifact: diskgraph-wheel install-graphviz: ${{ matrix.test_dir == 'test/graph/' }} - - name: Log in to Azure with GitHub OIDC - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && env.KUMO_RUN_ADLS_TESTS == '1' - uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - - name: Authenticate to Google Cloud with GitHub OIDC - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - name: Set up coverage artifact name run: | # This name must avoid using special chars and must be unique per: @@ -258,17 +212,13 @@ jobs: # exceeds the per-group duration target (e.g. # test_db_file_storage::test_compression at ~511s in # test/rest/test_*.py). No-op when --splits is not in matrix.test_dir. - GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=signal --splitting-algorithm least_duration ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} + GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --resource-group public --durations=50 --timeout=1800 --timeout-method=signal --splitting-algorithm least_duration ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} move_coverage_artifact # Mirror local-test-relevant additions in setup.cfg [tool:pytest] env. env: # Set to "1" in a PR branch to re-enable io_utils.fsutil DEBUG logs # while debugging filesystem behavior. KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} AWS_DEFAULT_REGION: "us-west-2" DEPLOY_STAGE: "test" TEMPORAL_SERVER: "localhost" @@ -278,7 +228,6 @@ jobs: KUMO_ENABLE_GLOVE: "False" SPARK_VERSION: "3.5" KUMO_TESTING: "True" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Run full test suite if: env.RUN_FULL_TEST == 'true' && steps.non-pytest-changes.outputs.only_changed != 'true' @@ -291,16 +240,12 @@ jobs: # --timeout=4500 fires below the 90-min step cap so pytest-timeout # gets a chance to run; --timeout-method=signal so the on-failure # flake-diagnostics hook runs before the process exits. - FULL_TEST=1 GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=4500 --timeout-method=signal --splitting-algorithm least_duration ${{ matrix.test_dir }} + FULL_TEST=1 GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --resource-group public --durations=50 --timeout=4500 --timeout-method=signal --splitting-algorithm least_duration ${{ matrix.test_dir }} move_coverage_artifact env: # Set to "1" in a PR branch to re-enable io_utils.fsutil DEBUG logs # while debugging filesystem behavior. KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} AWS_DEFAULT_REGION: "us-west-2" DEPLOY_STAGE: "test" TEMPORAL_SERVER: "localhost" @@ -309,7 +254,6 @@ jobs: CUDA: "cpu" SPARK_VERSION: "3.5" KUMO_TESTING: "True" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Upload coverage artifact if: always() @@ -519,7 +463,7 @@ jobs: - name: Set up packages uses: ./.github/actions/setup with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} + github_token: ${{ github.token }} diskgraph-wheel-artifact: diskgraph-wheel - name: Build and import Kumo-core wheel diff --git a/.github/workflows/testing_databricks.yml b/.github/workflows/testing_databricks.yml deleted file mode 100644 index c4a17eb33..000000000 --- a/.github/workflows/testing_databricks.yml +++ /dev/null @@ -1,193 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Databricks Testing and Coverage - -on: # yamllint disable-line rule:truthy - push: - branches: - - main - paths-ignore: ["ui/**", "tools/**"] - pull_request: - merge_group: - workflow_dispatch: - inputs: - full_test: - description: "Run the full test suite" - type: boolean - default: true - required: false - store_durations: - description: "Record pytest-split durations (forces minimal mode)" - type: boolean - default: false - required: false - workflow_call: - inputs: - full_test: - type: boolean - default: false - required: false - store_durations: - type: boolean - default: false - required: false - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -permissions: - contents: read - id-token: write - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} - STORE_DURATIONS: ${{ inputs.store_durations == true }} - RUN_FULL_TEST: ${{ github.event_name == 'push' || (inputs.full_test == true && inputs.store_durations != true) }} - FORCE_RUN_TESTS: ${{ inputs.store_durations == true }} - -jobs: - pytest: - environment: - name: gha-kumo-workflow-tests - # This job uses environment-scoped OIDC; it does not deploy an application. - deployment: false - runs-on: ubuntu-24.04 - continue-on-error: true - - strategy: - fail-fast: false - matrix: - test_dir: [ - # The former Databricks-compute E2E selectors were removed with the - # DiskGraph cutover. Databricks source/export coverage now lives in - # test_databricks_diskgraph_e2e.py. - "test/rest/test_databricks.py", - "test/rest/integ/test_databricks_diskgraph_e2e.py", - ] - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - ref: ${{ github.sha }} - fetch-depth: 40 - - - name: Set up Python - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - python-version: "3.14" - activate-environment: true - - # uv.lock is deliberately excluded: a lock-only dependency bump must - # still run the real install + test suite (see testing.yml). - - name: Check for only non pytest required file changes - id: non-pytest-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ui/** - CHANGELOG.md - tools/** - benchmark/** - - - name: Configure AWS credentials - if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - - name: Set up packages - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && !contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up packages (with diskgraph) - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up coverage artifact name - run: | - echo "ARTIFACT_NAME_SAFE=.coverage-dbx-${{ github.run_id }}-$(echo -n ${{ matrix.test_dir }} | sha256sum | awk '{print $1}')" >> $GITHUB_ENV - - - name: Run minimal test suite - if: env.RUN_FULL_TEST != 'true' && (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') - timeout-minutes: 50 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - start_ci_memory_monitor - GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} - move_coverage_artifact - env: - KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_STAGE: "test" - TEMPORAL_SERVER: "localhost" - DATA_SPARK_MEM: "1g" - MAX_DATA_BACKEND_WORKER_PROCESSES: "1" - CUDA: "cpu" - KUMO_ENABLE_GLOVE: "False" - KUMO_TESTING: "True" - SPARK_VERSION: "3.5" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - - - name: Run full test suite - if: env.RUN_FULL_TEST == 'true' && steps.non-pytest-changes.outputs.only_changed != 'true' - timeout-minutes: 90 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - FULL_TEST=1 GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 ${{ matrix.test_dir }} - move_coverage_artifact - env: - KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_STAGE: "test" - TEMPORAL_SERVER: "localhost" - DATA_SPARK_MEM: "1g" - MAX_DATA_BACKEND_WORKER_PROCESSES: "1" - CUDA: "cpu" - KUMO_TESTING: "True" - SPARK_VERSION: "3.5" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - - - name: Upload coverage artifact - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ${{ env.ARTIFACT_NAME_SAFE }} - path: ${{ env.ARTIFACT_NAME_SAFE }} - include-hidden-files: true - - - name: Upload durations artifact - if: success() && env.STORE_DURATIONS == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: durations-${{ github.run_id }}-${{ strategy.job-index }}-of-${{ strategy.job-total }} - path: .test_durations - include-hidden-files: true - if-no-files-found: ignore diff --git a/.github/workflows/testing_gcs.yml b/.github/workflows/testing_gcs.yml deleted file mode 100644 index a49396bae..000000000 --- a/.github/workflows/testing_gcs.yml +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: GCS Testing - -on: # yamllint disable-line rule:truthy - push: - branches: - - main - paths-ignore: ["ui/**", "tools/**"] - pull_request: - merge_group: - workflow_dispatch: - inputs: - full_test: - description: "Run the full test suite" - type: boolean - default: true - required: false - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -permissions: - contents: read - id-token: write - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} - -jobs: - pytest-gcs: - environment: - name: gha-kumo-workflow-tests - # This job uses environment-scoped OIDC; it does not deploy an application. - deployment: false - name: gcs-e2e-tests - runs-on: ubuntu-24.04 - continue-on-error: true - permissions: - contents: read - id-token: write - env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - strategy: - fail-fast: false - matrix: - test_dir: [ - test/rest/integ/test_gcs_e2e.py, - test/rest/integ/test_bigquery_diskgraph_e2e.py, - test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py, - ] - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - ref: ${{ github.sha }} - fetch-depth: 40 - - - name: Get GCS related changes - id: gcs-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 - with: - files: | - .github/actions/setup/action.yml - .github/workflows/testing_gcs.yml - kumo/connector/** - kumo/testing/gcs.py - io_utils/** - rest/** - setup.py - test/rest/integ/test_gcs_e2e.py - test/rest/integ/test_bigquery_diskgraph_e2e.py - test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py - test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json - rest/devserver/configs/diskgraph-test.json - kumo/testing/bigquery.py - kumo/testing/rest.py - - - name: Set up Python - if: steps.gcs-files-changed.outputs.any_changed == 'true' - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - python-version: "3.14" - activate-environment: true - - - name: Configure AWS credentials - if: steps.gcs-files-changed.outputs.any_changed == 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - - name: Set up packages - if: steps.gcs-files-changed.outputs.any_changed == 'true' && !contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up packages (with diskgraph) - if: steps.gcs-files-changed.outputs.any_changed == 'true' && contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Authenticate to Google Cloud with GitHub OIDC - if: steps.gcs-files-changed.outputs.any_changed == 'true' && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - - name: Run E2E tests - if: steps.gcs-files-changed.outputs.any_changed == 'true' - timeout-minutes: 60 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - GLOG_logtostdout=1 python -X faulthandler -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=thread ${{ matrix.test_dir }} - env: - # GCS-specific environment variables - IS_GCP_DEPLOY: "1" - GCS_TEST_BUCKET: 'kumo-test-bucket' - GCP_STORAGE_TRANSFER_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: "true" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - # Standard test environment variables - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_STAGE: "test" - TEMPORAL_SERVER: "localhost" - DATA_SPARK_MEM: "1g" - MAX_DATA_BACKEND_WORKER_PROCESSES: "1" - CUDA: "cpu" - KUMO_ENABLE_GLOVE: "False" - SPARK_VERSION: "3.5" - KUMO_TESTING: "True" diff --git a/.github/workflows/testing_private.yml b/.github/workflows/testing_private.yml new file mode 100644 index 000000000..4d6002580 --- /dev/null +++ b/.github/workflows/testing_private.yml @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Private resource tests + +on: # yamllint disable-line rule:truthy + push: + branches: [main, "pull-request/[0-9]+"] + workflow_dispatch: + inputs: + full_test: + description: Run full-suite-only cases as well + type: boolean + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} + +permissions: + contents: read + +jobs: + changes: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + matrix: ${{ steps.select.outputs.matrix }} + run: ${{ steps.select.outputs.run }} + gcp-e2e: ${{ steps.select.outputs.gcp-e2e }} + snowflake-e2e: ${{ steps.select.outputs.snowflake-e2e }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + persist-credentials: false + - name: Read copied PR context + id: pr + if: startsWith(github.ref, 'refs/heads/pull-request/') + uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf + - name: Verify the copied commit + if: startsWith(github.ref, 'refs/heads/pull-request/') + env: + PR_INFO: ${{ steps.pr.outputs.pr-info }} + run: | + python3 - <<'PY' + import json, os + pr = json.loads(os.environ['PR_INFO']) + assert pr['head']['sha'] == os.environ['GITHUB_SHA'], 'Copied PR is stale; approve the current SHA' + PY + - name: Read complete changed-file list + id: files + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 + with: + base_sha: ${{ steps.pr.outputs.pr-info && fromJSON(steps.pr.outputs.pr-info).base.sha || github.event.before }} + json: true + escape_json: false + - name: Select private jobs + id: select + env: + CHANGED_FILES: ${{ steps.files.outputs.all_changed_files }} + FORCE_PRIVATE_TESTS: ${{ github.event_name == 'workflow_dispatch' }} + run: python3 tools/ci_private_changes.py + + diskgraph-wheel: + needs: changes + if: needs.changes.outputs.run == 'true' + uses: ./.github/workflows/_diskgraph-wheel.yml + permissions: + contents: read + + pytest: + name: private-${{ matrix.group }} + needs: [changes, diskgraph-wheel] + if: needs.changes.outputs.run == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.changes.outputs.matrix) }} + runs-on: + group: nv-cpu-general + labels: linux-amd64-cpu16 + timeout-minutes: ${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || inputs.full_test == true) && 90 || 25 }} + environment: + name: ${{ matrix.group == 'azure' && 'azure-adls-tests' || 'gha-kumo-workflow-tests' }} + deployment: false + permissions: + contents: read + id-token: write + env: + FULL_TEST: ${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || inputs.full_test == true) && '1' || '0' }} + CI_RUN_GCP_E2E: ${{ needs.changes.outputs.gcp-e2e }} + CI_RUN_SNOWFLAKE_E2E: ${{ needs.changes.outputs.snowflake-e2e }} + CI_TEST_RESULTS: ci-test-results + ARTIFACTS_DIR: /tmp/kumo-test-artifacts + AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 + AWS_DEFAULT_REGION: us-west-2 + DEPLOY_STAGE: test + TEMPORAL_SERVER: localhost + DATA_SPARK_MEM: 1g + MAX_DATA_BACKEND_WORKER_PROCESSES: '1' + CUDA: cpu + KUMO_ENABLE_GLOVE: 'False' + SPARK_VERSION: '3.5' + SPARK_LOCAL_IP: 127.0.0.1 + KUMO_TESTING: 'True' + KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: '0' + KUMO_RUN_ADLS_TESTS: ${{ matrix.group == 'azure' && '1' || '0' }} + KUMO_RUN_GCP_TESTS: ${{ startsWith(matrix.group, 'gcp') && '1' || '0' }} + KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ startsWith(matrix.group, 'gcp') && 'true' || 'false' }} + GCS_TEST_BUCKET: kumo-test-bucket + GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} + GCP_STORAGE_TRANSFER_BUCKET: ${{ vars.GCP_STORAGE_TRANSFER_BUCKET }} + GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} + GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} + GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} + AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }} + AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }} + DATABRICKS_HOST: ${{ matrix.group == 'databricks' && vars.DATABRICKS_HOST || '' }} + DATABRICKS_CLUSTER_ID: ${{ vars.DATABRICKS_CLUSTER_ID }} + DATABRICKS_WAREHOUSE_ID: ${{ vars.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_TOKEN: ${{ matrix.group == 'databricks' && secrets.DATABRICKS_TOKEN || '' }} + DATABRICKS_CLIENT_ID: ${{ matrix.group == 'databricks' && vars.DATABRICKS_CLIENT_ID || '' }} + DATABRICKS_CLIENT_SECRET: ${{ matrix.group == 'databricks' && secrets.DATABRICKS_CLIENT_SECRET || '' }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - name: Set up Python + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + python-version: '3.14' + activate-environment: true + - name: Set up packages + uses: ./.github/actions/setup + with: + github_token: ${{ github.token }} + diskgraph-wheel-artifact: diskgraph-wheel + install-graphviz: 'true' + - name: Configure AWS credentials + if: startsWith(matrix.group, 'aws') || matrix.group == 'gcp-e2e' || matrix.group == 'snowflake' || matrix.group == 'databricks' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests + aws-region: us-west-2 + - name: Authenticate to Google Cloud + if: startsWith(matrix.group, 'gcp') + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + project_id: ${{ vars.GCP_PROJECT_ID }} + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} + - name: Authenticate to Azure + if: matrix.group == 'azure' + uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Verify Azure test storage configuration + if: matrix.group == 'azure' + run: | + python - <<'PY' + import os + missing = [name for name in ('AZURE_STORAGE_ACCOUNT', 'AZURE_CONTAINER_NAME') if not os.environ.get(name)] + assert not missing, f'Missing Azure test configuration: {missing}' + PY + - name: Verify federated Google credentials + if: startsWith(matrix.group, 'gcp') + run: | + python - <<'PY' + import json, os + from pathlib import Path + credentials = json.loads(Path(os.environ['GOOGLE_APPLICATION_CREDENTIALS']).read_text()) + assert credentials['type'] == 'external_account', 'Private GCP CI requires federation' + required = ['GCP_PROJECT_ID', 'GCP_TEST_SERVICE_ACCOUNT', 'GCP_STORAGE_TRANSFER_BUCKET'] + if os.environ['KUMO_RUN_GCP_TESTS'] == '1': + required += ['GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT', 'GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT'] + missing = [name for name in required if not os.environ.get(name)] + assert not missing, f'Missing GCP test configuration: {missing}' + PY + - name: Verify Databricks test configuration + if: matrix.group == 'databricks' + run: | + python - <<'PY' + import os + required = ['DATABRICKS_HOST', 'DATABRICKS_WAREHOUSE_ID', 'DATABRICKS_CLIENT_ID', 'DATABRICKS_CLIENT_SECRET', 'DATABRICKS_TOKEN'] + missing = [name for name in required if not os.environ.get(name)] + assert not missing, f'Missing Databricks test configuration: {missing}' + PY + - name: Run selected resource tests + env: + RESOURCE_GROUP: ${{ matrix.group }} + run: python -m tools.run_private_tests "$RESOURCE_GROUP" + - name: Upload selection, results and complete durations + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: private-tests-${{ matrix.group }}-${{ github.run_id }} + path: ci-test-results/ + if-no-files-found: error diff --git a/.github/workflows/testing_snowflake.yml b/.github/workflows/testing_snowflake.yml deleted file mode 100644 index e665ef75e..000000000 --- a/.github/workflows/testing_snowflake.yml +++ /dev/null @@ -1,224 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Snowflake Testing and Coverage - -on: # yamllint disable-line rule:truthy - push: - branches: - - main - paths-ignore: ["ui/**", "tools/**"] - pull_request: - merge_group: - workflow_dispatch: - inputs: - full_test: - description: "Run the full test suite" - type: boolean - default: true - required: false - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -permissions: - contents: read - id-token: write - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} - -jobs: - pytest: - environment: - name: gha-kumo-workflow-tests - # This job uses environment-scoped OIDC; it does not deploy an application. - deployment: false - name: all-snowflake-tests - runs-on: ubuntu-24.04 - continue-on-error: true - permissions: - contents: read - id-token: write - env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} - - strategy: - matrix: - test_name: - [ - 'connector (test/connector)', - 'rest (test/rest)', - 'fsutil (test/io_utils/fsutil)', - 'backend - data (test/data/ test/graph/', - 'backend - pquery (test/pquery/bp_generator)', - 'backend - pquery (test/pquery/train_table_generator)', - 'snowflake diskgraph e2e (test/rest/integ/test_snowflake_diskgraph_e2e.py)', - 'snowflake ingestion diskgraph e2e (test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py)' - ] - include: - - test_name: 'connector (test/connector)' - test_dir: 'test/connector' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - setup.py - test/connector/** - - test_name: 'rest (test/rest)' - test_dir: 'test/rest --ignore=test/rest/integ/test_snowflake_diskgraph_e2e.py --ignore=test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - setup.py - test/rest/** - - test_name: 'fsutil (test/io_utils/fsutil)' - test_dir: 'test/io_utils/fsutil' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - io_utils/** - setup.py - test/io_utils/fsutil/** - - test_name: 'backend - data (test/data/ test/graph/' - test_dir: 'test/data/ test/graph/**snowflake**' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/stats/** - io_utils/** - setup.py - test/data/ - test/graph/**snowflake** - - test_name: 'backend - pquery (test/pquery/bp_generator)' - test_dir: 'test/pquery/test_pql_batch_pred_generator.py' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/pquery/** - io_utils/** - kumo/data/data_processing_backend/** - setup.py - test/pquery/test_pql_batch_pred_generator.py - - test_name: 'backend - pquery (test/pquery/train_table_generator)' - test_dir: 'test/pquery/train_table_generator' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/pquery/** - io_utils/** - setup.py - test/pquery/train_table_generator/** - - test_name: 'snowflake diskgraph e2e (test/rest/integ/test_snowflake_diskgraph_e2e.py)' - test_dir: "test/rest/integ/test_snowflake_diskgraph_e2e.py" - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - rest/backends/data_backend/activities/** - test/rest/integ/test_snowflake_diskgraph_e2e.py - test/rest/integ/test_cases/test_snowflake_e2e_fixtures.json - rest/devserver/configs/diskgraph-test.json - - test_name: 'snowflake ingestion diskgraph e2e (test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py)' - test_dir: "test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py" - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - rest/backends/data_backend/activities/** - test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py - rest/devserver/configs/diskgraph-test.json - - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - ref: ${{ github.sha }} - fetch-depth: 40 - - - name: Get any data backend related changes - id: data-backend-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ${{ matrix.files_changed }} - - - name: Configure AWS credentials - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - - name: Set up packages - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && !contains(matrix.test_dir, 'diskgraph') && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up packages (with diskgraph) - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && contains(matrix.test_dir, 'diskgraph') && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Authenticate to Google Cloud with GitHub OIDC - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - # NOTE here we use coverage.py instead of pytest-cov to obtain JSON - # outputs that are easier to parse and manage (pytest-cov/455). - # coverage.py also has additional nice features that may be useful in the - # future for better coverage reports. - - name: Run minimal test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && github.event_name != 'push' && inputs.full_test != true && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - timeout-minutes: 120 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} - env: - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - - - name: Run full test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && (github.event_name == 'push' || inputs.full_test == true) && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' - timeout-minutes: 120 - run: | - set -o pipefail - export SPARK_LOCAL_IP="127.0.0.1" - source ./tools/ci_pytest_log_controls.sh - configure_pytest_ci_output - FULL_TEST=1 GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} - env: - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} - AWS_DEFAULT_REGION: "us-west-2" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} diff --git a/.github/workflows/ui-client-code-generate.yml b/.github/workflows/ui-client-code-generate.yml index 7f72736e9..e5a431c17 100644 --- a/.github/workflows/ui-client-code-generate.yml +++ b/.github/workflows/ui-client-code-generate.yml @@ -25,7 +25,6 @@ concurrency: cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} GITHUB_REF: ${{secrets.GITHUB_REF}} GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} @@ -41,12 +40,10 @@ permissions: jobs: check-openapi-changes: - environment: gha-kumo-workflow-tests name: Check OpenAPI Changes runs-on: ubuntu-24.04 if: github.event_name == 'pull_request' permissions: - id-token: write # Required for configure-aws-credentials OIDC. contents: read pull-requests: write issues: write @@ -81,13 +78,6 @@ jobs: rest/utils/routers.py ui/src/gen/spec/** - - name: Configure AWS credentials - if: steps.backend-changes.outputs.any_changed == 'true' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up backend packages if: steps.backend-changes.outputs.any_changed == 'true' uses: ./.github/actions/setup @@ -175,11 +165,9 @@ jobs: } trigger-codegen: - environment: gha-kumo-workflow-tests name: Trigger UI Codegen runs-on: ubuntu-24.04 permissions: - id-token: write # Required for configure-aws-credentials OIDC. contents: write pull-requests: write issues: write @@ -271,12 +259,6 @@ jobs: python-version: "3.14" activate-environment: true - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up backend packages uses: ./.github/actions/setup with: @@ -372,11 +354,9 @@ jobs: git push origin HEAD:${{ steps.pr-details.outputs.ref }} codegen: - environment: gha-kumo-workflow-tests name: Manual Codegen (workflow_dispatch) runs-on: ubuntu-24.04 permissions: - id-token: write # Required for configure-aws-credentials OIDC. contents: write pull-requests: write packages: read @@ -397,12 +377,6 @@ jobs: python-version: "3.14" activate-environment: true - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests - aws-region: us-west-2 - - name: Set up backend packages uses: ./.github/actions/setup with: diff --git a/docs/connector-test-matrix.md b/docs/connector-test-matrix.md index 2e6404e04..25d50ff9e 100644 --- a/docs/connector-test-matrix.md +++ b/docs/connector-test-matrix.md @@ -86,18 +86,12 @@ The existing hosted workflows have different triggers and failure policies: | Workflow / coverage | When tests run | Failure behavior | | --- | --- | --- | -| [Testing and Coverage](../.github/workflows/testing.yml): hosted S3, ADLS and GCP tests | PR, master push, merge queue, manual or reusable-workflow invocation, subject to test-file filters and selected shards. ADLS/GCP authentication is limited to configured repository/shard and same-repository PR conditions. | The hosted `pytest` job has `continue-on-error: true`. Package-suite failures fail the separate `packages` job. | -| [GCS Testing](../.github/workflows/testing_gcs.yml): hosted GCS and BigQuery end-to-end tests | PR, master push, merge queue or manual run; test steps require matching changed files. GCP authentication also requires the configured repository and a same-repository PR for PR events. | Test job has `continue-on-error: true`. | -| [Snowflake Testing](../.github/workflows/testing_snowflake.yml): hosted Snowflake tests | PR, master push, merge queue or manual run; each shard's test steps require matching changed files. | Test job has `continue-on-error: true`. | -| [Databricks Testing](../.github/workflows/testing_databricks.yml): hosted REST and DiskGraph tests | PR, master push, merge queue, manual or reusable-workflow invocation; test steps apply changed-file and full/minimal-mode conditions. Durations recording can force a minimal run. | Test job has `continue-on-error: true`. | - -These workflows exclude pushes that only change `ui/` or `tools/`. -Their existence does not mean every PR runs cloud tests. Jobs marked -`continue-on-error` are informational for workflow failure propagation, not -evidence of a required merge gate. The package job does not enable the -Snowflake, Databricks or BigQuery live flags. Their source-reader/sink suites -remain distinct from the hosted workflows above. Access approvals and merge -rules are separate from these workflow triggers. +| [Testing and Coverage](../.github/workflows/testing.yml): public local/mocked Python tests | PR, main push, merge queue, manual or reusable invocation; resource-backed cases are deselected before fixture setup. | Test failures fail their jobs. | +| [Private resource tests](../.github/workflows/testing_private.yml): AWS, GCP, Azure, Snowflake, and Databricks | Approved copy-pr-bot branch push, main push, or manual dispatch on NVIDIA private runners; expensive provider E2E selections retain change filters. | Failures and missing-credential skips fail verification. | + +Public tests do not configure cloud identities. Private jobs use the shared resource classification and retain selections and results as artifacts. Workflow success does not establish coverage for a selection that was filtered out or intentionally skipped. Required merge checks are configured separately. Use `python -m pytest --resource-group public ` for local public selection; the private workflow uses `python -m tools.run_private_tests ` to run isolated provider sessions. Package source-reader/sink live suites retain their separate opt-in flags. + +Private discovery scans component test subdirectories. Root-level script tests, release tooling, and package suites retain separate entry points. Resource decorators and `private_resource` marks declare cloud access; `ci_profile`, `ci_group`, and `ci_gate` marks declare process environments, balanced workloads, and provider change gates. Fixture metadata propagates through dependency chains without resolving credentials. ## Real cloud tests on Kumo resources diff --git a/kumo/testing/_skip.py b/kumo/testing/_skip.py index c96a8ec30..311fefd14 100644 --- a/kumo/testing/_skip.py +++ b/kumo/testing/_skip.py @@ -42,3 +42,18 @@ def _sync_wrapper(*args, **kwargs): return func(*args, **kwargs) return _sync_wrapper + + +def requires_resource(provider: str, func: Callable | None = None, *, + profile: str | None = None) -> Callable: + """Describe live test/fixture dependencies without accessing credentials.""" + if func is None: + return functools.partial(requires_resource, provider, profile=profile) + resources = set(getattr(func, '__kumo_test_resources__', ())) + resources.add(provider) + setattr(func, '__kumo_test_resources__', tuple(sorted(resources))) + if profile: + profiles = set(getattr(func, '__kumo_ci_profiles__', ())) + profiles.add(profile) + func.__kumo_ci_profiles__ = tuple(sorted(profiles)) + return func diff --git a/kumo/testing/adls.py b/kumo/testing/adls.py index 2c638c5eb..fc4363061 100644 --- a/kumo/testing/adls.py +++ b/kumo/testing/adls.py @@ -104,6 +104,8 @@ def test_with_config(): } """ def decorator(test_func: Callable): + from kumo.testing._skip import requires_resource + test_func = requires_resource('azure', test_func) credentials_missing = not _adls_credentials_configured(config_path) if inspect.isgeneratorfunction(test_func): # A decorator below @pytest.fixture receives the raw generator. diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index db7ad10be..617ae6515 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -113,6 +113,9 @@ def withBigQueryCredentials(func: Optional[Callable] = None, import pytest def decorator(inner_func: Callable): + from kumo.testing._skip import requires_resource + inner_func = requires_resource('gcp', inner_func) + def _setup(): setup_bigquery_credentials(service_account_key) if (not os.getenv(ENV_GOOGLE_CREDENTIALS_JSON) and diff --git a/kumo/testing/databricks.py b/kumo/testing/databricks.py index adc5db588..b046c4bb9 100644 --- a/kumo/testing/databricks.py +++ b/kumo/testing/databricks.py @@ -24,7 +24,8 @@ class DatabricksTableConnector: def withDatabricksCredentials(func: Callable): """Skip unless DATABRICKS_HOST and DATABRICKS_TOKEN are set.""" - from kumo.testing._skip import skip_if + from kumo.testing._skip import requires_resource, skip_if + func = requires_resource('databricks', func, profile='databricks-pat') has_no_credentials = not (DATABRICKS_HOST.parse_value() and DATABRICKS_TOKEN.parse_value()) return skip_if(has_no_credentials, "No Databricks credentials found", func) @@ -32,7 +33,8 @@ def withDatabricksCredentials(func: Callable): def withDatabricksOauthCredentials(func: Callable): """Skip unless the host, client ID, and client secret variables are set.""" - from kumo.testing._skip import skip_if + from kumo.testing._skip import requires_resource, skip_if + func = requires_resource('databricks', func, profile='databricks-oauth') has_no_credentials = not (DATABRICKS_HOST.parse_value() and DATABRICKS_CLIENT_ID.parse_value() and DATABRICKS_CLIENT_SECRET.parse_value()) diff --git a/kumo/testing/gcs.py b/kumo/testing/gcs.py index a511fd19d..630d8e151 100644 --- a/kumo/testing/gcs.py +++ b/kumo/testing/gcs.py @@ -127,6 +127,8 @@ def test_gcs_operation(): # A collection-time mark prevents live GCS fixtures from being resolved # when credentials are absent. A runtime wrapper skips too late: pytest # sets up fixtures before invoking the wrapped test. + from kumo.testing._skip import requires_resource + func = requires_resource('gcp', func) credentials_missing = not _check_gcs_credentials() reason = "No GCS credentials found" if inspect.isgeneratorfunction(func): diff --git a/kumo/testing/s3.py b/kumo/testing/s3.py index 1c0f57b8f..3de7479cd 100644 --- a/kumo/testing/s3.py +++ b/kumo/testing/s3.py @@ -48,9 +48,12 @@ def _s3_credentials_configured() -> bool: def withS3Credentials(func: Callable | None = None): credentials_missing = not _s3_credentials_configured() reason = "No AWS credentials found for S3" - marker = pytest.mark.skipif(credentials_missing, reason=reason) + marker = pytest.mark.skipif(credentials_missing, reason=reason, + kumo_resource='aws') + from kumo.testing._skip import requires_resource if func is None: return marker + func = requires_resource('aws', func) if inspect.isgeneratorfunction(func): from kumo.testing._skip import skip_if return skip_if(credentials_missing, reason, func) diff --git a/kumo/testing/snowflake.py b/kumo/testing/snowflake.py index 6cb83a3d6..d6ba76521 100644 --- a/kumo/testing/snowflake.py +++ b/kumo/testing/snowflake.py @@ -38,7 +38,9 @@ def withSnowflakeCredentials(func: Callable): """Load credentials during pytest setup, before cloud resource fixtures.""" import pytest - return pytest.mark.usefixtures('snowflake_credentials')(func) + from kumo.testing._skip import requires_resource + return pytest.mark.usefixtures('snowflake_credentials')(requires_resource( + 'snowflake', func)) def setup_snowflake_credentials() -> None: diff --git a/test/ci_resources.py b/test/ci_resources.py new file mode 100644 index 000000000..804c093b6 --- /dev/null +++ b/test/ci_resources.py @@ -0,0 +1,188 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Select CI cases and describe their resource and process requirements.""" + +import json +import os +import re +from pathlib import Path + +import pytest + +GROUPS = ('public', 'aws', 'aws-public-api', 'gcp', 'gcp-e2e', 'azure', + 'snowflake', 'databricks') +PROFILE_ENV = { + 'diskgraph': {}, + 'gcp': { + 'IS_GCP_DEPLOY': '1' + }, + 'databricks-pat': { + 'DATABRICKS_CLIENT_ID': '', + 'DATABRICKS_CLIENT_SECRET': '' + }, + 'databricks-oauth': { + 'DATABRICKS_TOKEN': '' + }, +} +GATES = {'gcp-e2e': 'CI_RUN_GCP_E2E', 'snowflake-e2e': 'CI_RUN_SNOWFLAKE_E2E'} + + +def execution_profile(*profiles): + """Attach process requirements to a fixture without resolving it.""" + def decorate(func): + func.__kumo_ci_profiles__ = profiles + return func + + return decorate + + +def annotated_functions(item): + yield item.obj + # Fixture marks do not propagate to tests. Keep pytest-internal inspection + # here, and read metadata without resolving any fixture. + for name in item.fixturenames: + for definition in item._fixtureinfo.name2fixturedefs.get(name, ()): + yield definition.func + + +def resources_for_item(item): + resources = set() + for func in annotated_functions(item): + resources.update(getattr(func, '__kumo_test_resources__', ())) + for marker in item.iter_markers('skipif'): + if marker.kwargs.get('kumo_resource'): + resources.add(marker.kwargs['kumo_resource']) + for marker in item.iter_markers('private_resource'): + resources.update(marker.args) + unknown = resources - {'aws', 'gcp', 'azure', 'snowflake', 'databricks'} + if unknown: + raise pytest.UsageError( + f'Unknown resources for {item.nodeid}: {sorted(unknown)}') + return resources + + +def profiles_for_item(item): + profiles = set() + for func in annotated_functions(item): + profiles.update(getattr(func, '__kumo_ci_profiles__', ())) + for marker in item.iter_markers('ci_profile'): + profiles.update(marker.args) + if profiles - PROFILE_ENV.keys(): + raise pytest.UsageError( + f'Unknown CI profiles for {item.nodeid}: {sorted(profiles)}') + if {'databricks-pat', 'databricks-oauth'} <= profiles: + raise pytest.UsageError( + f'Conflicting Databricks auth profiles: {item.nodeid}') + return sorted(profiles) + + +def group_for_item(item): + resources = resources_for_item(item) + if not resources: + return 'public' + provider = next(p + for p in ('snowflake', 'databricks', 'gcp', 'azure', 'aws') + if p in resources) + marker = item.get_closest_marker('ci_group') + group = marker.args[0] if marker else provider + if group not in GROUPS or group.split('-', 1)[0] != provider: + raise pytest.UsageError(f'Invalid CI group {group} for {item.nodeid}') + return group + + +def pytest_addoption(parser): + parser.addoption('--resource-group', choices=GROUPS, default=None, + help='Select public tests or one private CI group') + parser.addoption( + '--resource-manifest', type=Path, + help='Write selected node IDs, resources and process profiles') + parser.addoption( + '--private-discovery', action='store_true', + help='Apply private CI provider change gates during collection') + parser.addoption('--require-resource-credentials', action='store_true', + help='Fail private CI on missing resource credentials') + + +def pytest_configure(config): + for marker in ('private_resource(provider): uses a live private resource', + 'ci_profile(name): requires a declared process environment', + 'ci_group(name): assigns resource cases to a CI workload', + 'ci_gate(name): follows a provider E2E changed-file gate'): + config.addinivalue_line('markers', marker) + config._resource_credential_skips = [] + + +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(config, items): + group = config.getoption('--resource-group') + if group is None: + return + selected, deselected = [], [] + for item in items: + enabled = True + if config.getoption('--private-discovery'): + for marker in item.iter_markers('ci_gate'): + gate = marker.args[0] + if gate not in GATES: + raise pytest.UsageError(f'Unknown CI gate: {gate}') + enabled &= os.environ.get(GATES[gate]) != 'false' + (selected if enabled and group_for_item(item) == group else + deselected).append(item) + items[:] = selected + config.hook.pytest_deselected(items=deselected) + + +def pytest_collection_finish(session): + destination = session.config.getoption('--resource-manifest') + if destination: + manifest = {} + for item in session.items: + profiles = profiles_for_item(item) + manifest[item.nodeid] = { + 'group': + group_for_item(item), + 'resources': + sorted(resources_for_item(item)), + 'profiles': + profiles, + 'isolated_module': + item.nodeid.split('::', 1)[0] + if 'diskgraph' in profiles else '', + } + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(manifest, indent=2) + '\n') + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + report = (yield).get_result() + if report.when == 'call': + reporter = item.config.pluginmanager.get_plugin('terminalreporter') + if reporter: + for line in report.capstdout.splitlines(): + timing = re.match( + r'^CI_PHASE public-api [A-Za-z0-9_]+ elapsed_seconds=\d+\.\d{3}', + line) + if timing: + reporter.write_line(timing.group(0)) + if (item.config.getoption('--require-resource-credentials') + and report.skipped and group_for_item(item) != 'public'): + reason = str(report.longrepr).lower() + if any(fragment in reason + for fragment in ('credential', 'not configured', + 'access is unavailable')): + item.config._resource_credential_skips.append(item.nodeid) + + +def pytest_sessionfinish(session, exitstatus): + missing = session.config._resource_credential_skips + if missing: + reporter = session.config.pluginmanager.get_plugin('terminalreporter') + if reporter: + reporter.write_sep( + '=', 'Private CI skipped tests for missing credentials', + red=True) + for nodeid in missing: + reporter.write_line(nodeid) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/test/conftest.py b/test/conftest.py index 9b72acc80..296c7d3dd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,7 +5,7 @@ # Register fixtures defined under test/fixtures/ so they're discoverable # session-wide without per-test imports. Currently exposes # ``dg_indexed_financial_graph`` (see test/fixtures/dg_indexed_graph.py). -pytest_plugins = ["test.fixtures.dg_indexed_graph"] +pytest_plugins = ["test.fixtures.dg_indexed_graph", "test.ci_resources"] import asyncio # noqa: E402 import copy # noqa: E402 @@ -70,6 +70,7 @@ from kumo.graph.table import Table # noqa: E402 from kumo.task.task import Task # noqa: E402 from kumo.testing import withDatabricksCredentials # noqa: E402 +from kumo.testing._skip import requires_resource # noqa: E402 from kumo.testing.generator import EdgeSpec, GraphSpec, TableSpec # noqa: E402 from kumo.testing.snowflake import setup_snowflake_credentials # noqa: E402 from kumo.utils.embedding_models import embedding_model_store # noqa: E402 @@ -96,8 +97,42 @@ os.environ.setdefault(_ENABLE_FSUTIL_DEBUG_LOGS_ENV, '0') +@pytest.fixture(scope='session') +def test_execution_environment(request): + """Apply selected process configuration before any deployment/cache fixtures.""" + from test.ci_resources import profiles_for_item + + from service_framework import deploy_env_config as config + + if not any('diskgraph' in profiles_for_item(item) + for item in request.session.items): + yield + return + names = ('DEPLOY_ENV_CONFIG_FILE', 'DEPLOY_ENV_CONFIG_JSON', 'PATH') + previous = {name: os.environ.get(name) for name in names} + previous_deploy_env = config._deploy_env + previous_cache = data._active_cache + if not (os.environ.get('DEPLOY_ENV_CONFIG_FILE') + or os.environ.get('DEPLOY_ENV_CONFIG_JSON')): + os.environ['DEPLOY_ENV_CONFIG_FILE'] = str( + pathlib.Path(__file__).resolve().parents[1] / 'rest' / + 'devserver' / 'configs' / 'diskgraph-test.json') + config._reset_deploy_env() + data.reset_active_cache_for_test() + try: + yield + finally: + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + config._deploy_env = previous_deploy_env + data.reset_active_cache_for_test(previous_cache) + + @pytest.fixture(scope='session', autouse=True) -def configure_test_glove_artifacts(tmp_path_factory): +def configure_test_glove_artifacts(tmp_path_factory, test_execution_environment): artifacts = tmp_path_factory.mktemp('glove') tokens = [ 'get', @@ -160,6 +195,9 @@ def pytest_collection_modifyitems(config, items): if "nightly" in item.keywords: item.add_marker(skip_nightly) + if config.getoption('--resource-group') == 'snowflake': + return # The resource selector also includes unmarked live session tests. + # check if you got an option like --key=snowflake if not config.getoption("--runsnowflake"): skip_snowflake = pytest.mark.skip( @@ -1441,6 +1479,7 @@ def create_s3_graph() -> Callable: @pytest.fixture() +@requires_resource('snowflake') def create_snowflake_graph() -> Callable: return _create_snowflake_graph @@ -1451,6 +1490,7 @@ def create_task() -> Callable: @pytest.fixture(scope='session') +@requires_resource('snowflake') def snowflake_credentials() -> None: setup_snowflake_credentials() @@ -1587,7 +1627,7 @@ def run_before_and_after_tests(auto_setup_kumo_backend: bool, @pytest.fixture(scope='session', autouse=True) -def run_before_and_after_test_sessions(): +def run_before_and_after_test_sessions(test_execution_environment): # The data persist dir is derived from deploy_env() (kumo.data # .get_persist_dir); wipe it at session boundaries for clean seeding. data_persist_dir = data.get_persist_dir() @@ -1602,7 +1642,7 @@ def run_before_and_after_test_sessions(): @pytest.fixture(scope='session', autouse=True) -def _redirect_test_diskgraph_data_root(): +def _redirect_test_diskgraph_data_root(test_execution_environment): """Point the DiskGraph data_root at a writable location under TEST. The production default for ``DiskgraphConfig.data_root`` is diff --git a/test/connector/conftest.py b/test/connector/conftest.py index 5269e79ff..fe9732917 100644 --- a/test/connector/conftest.py +++ b/test/connector/conftest.py @@ -11,6 +11,7 @@ from kumo.connector.databricks_connector import DatabricksConnector from kumo.connector.snowflake_connector import SnowflakeConnector +from kumo.testing._skip import requires_resource from kumo.testing.databricks import DatabricksTableConnector from kumo.testing.rest import ( financial_snowflake_config, @@ -22,6 +23,7 @@ @pytest.fixture(scope='session') +@requires_resource('aws') def require_kumo_public_datasets_s3() -> None: """Skip remote dataset tests when this environment lacks S3 access.""" import boto3 diff --git a/test/connector/test_databricks_connector.py b/test/connector/test_databricks_connector.py index 700d30757..0138182cf 100644 --- a/test/connector/test_databricks_connector.py +++ b/test/connector/test_databricks_connector.py @@ -31,6 +31,7 @@ _HOST = os.environ.get('DATABRICKS_HOST', 'workspace.cloud.databricks.com') +# TODO: Use a Databricks volume for seed staging to remove the AWS dependency. _TEST_SEED_STAGING_PATH = os.environ.get( 'VOLUME_MOUNT_PATH', 's3://kumo-preprod-dataplane-bucket/databricks-integ-test-staging/') @@ -301,6 +302,7 @@ def test_databricks_table_info(databricks_hm_connector: DatabricksConnector): assert connector._get_table_source_size('transactions') == 776368084 +@pytest.mark.private_resource('aws') @withDatabricksOauthCredentials def test_databricks_write_df_and_write_artifacts_to_table( writable_databricks_table: DatabricksTableConnector): diff --git a/test/io_utils/fsutil/local/test_s3.py b/test/io_utils/fsutil/local/test_s3.py index 42fb860ba..fad5e2abf 100644 --- a/test/io_utils/fsutil/local/test_s3.py +++ b/test/io_utils/fsutil/local/test_s3.py @@ -211,7 +211,7 @@ def test_zip_files_passes_exact_keys_without_prefix_listing(monkeypatch): ) -def test_s3_credentials_decorator_does_not_probe_provider(monkeypatch): +def test_s3_credentials_decorator_does_not_probe_provider(monkeypatch, tmp_path): credential_env = ( 'AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY', @@ -229,6 +229,8 @@ def test_s3_credentials_decorator_does_not_probe_provider(monkeypatch): def test_target(): pass + # An existing developer credential file must not affect this no-credentials case. + monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(tmp_path / 'missing')) with patch.object(s3_filesystem.threadsafe_boto, 's3_client') as client: list_objects = client.return_value.list_objects_v2 decorated = withS3Credentials(test_target) diff --git a/test/io_utils/fsutil/test_all.py b/test/io_utils/fsutil/test_all.py index f2b661d39..c4c55233d 100644 --- a/test/io_utils/fsutil/test_all.py +++ b/test/io_utils/fsutil/test_all.py @@ -50,10 +50,14 @@ def root_ucv(temp_ucv) -> Generator[str, None, None]: @pytest.fixture( params=[ - "root_s3", - "root_gcs", - "root_ucv", - "root_adls", + pytest.param('root_s3', marks=pytest.mark.private_resource('aws')), + pytest.param('root_gcs', marks=pytest.mark.private_resource('gcp')), + pytest.param( + 'root_ucv', marks=[ + pytest.mark.private_resource('databricks'), + pytest.mark.ci_profile('databricks-pat'), + ]), + pytest.param('root_adls', marks=pytest.mark.private_resource('azure')), ], ids=["s3", "gcs", "ucv", "adls"], ) diff --git a/test/io_utils/fsutil/test_fsutil.py b/test/io_utils/fsutil/test_fsutil.py index b64500e9a..453f13b75 100644 --- a/test/io_utils/fsutil/test_fsutil.py +++ b/test/io_utils/fsutil/test_fsutil.py @@ -8,17 +8,14 @@ import pytest from io_utils import fsutil +from kumo.testing.s3 import withS3Credentials @pytest.fixture +@withS3Credentials def s3_scratch_prefix(): - """Unique S3 prefix per test, deleted after yield. Skips if no - AWS creds — hand-rolled to keep this fixture independent of REST app - initialization. - """ + """Unique S3 prefix per test, deleted after yield.""" import uuid - if not fsutil.exists('s3://kumo-public-datasets/financial/csv/'): - pytest.skip('No AWS credentials found for S3') prefix = ('s3://kumo-test-dataplane-bucket/fsutil-test/' f'copy_tree_threaded_{uuid.uuid4().hex[:12]}') yield prefix diff --git a/test/io_utils/snowflake/test_session.py b/test/io_utils/snowflake/test_session.py index 8c03b3ab4..0f28650df 100644 --- a/test/io_utils/snowflake/test_session.py +++ b/test/io_utils/snowflake/test_session.py @@ -2,11 +2,14 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import pytest + import io_utils.snowflake.session from io_utils.snowflake.session import snowpark_session_in_app_schema from kumo.testing.snowflake import setup_full_snowflake_session +@pytest.mark.private_resource('snowflake') def test_schema_no_change(monkeypatch, tempdir): session = setup_full_snowflake_session(tempdir) original_database = session.get_current_database() @@ -32,6 +35,7 @@ def test_schema_no_change(monkeypatch, tempdir): assert session.get_current_schema() == original_schema +@pytest.mark.private_resource('snowflake') def test_schema_change(monkeypatch, tempdir): session = setup_full_snowflake_session(tempdir) original_database = session.get_current_database() diff --git a/test/io_utils/snowflake/test_session_logging.py b/test/io_utils/snowflake/test_session_logging.py index 624101146..4b485a057 100644 --- a/test/io_utils/snowflake/test_session_logging.py +++ b/test/io_utils/snowflake/test_session_logging.py @@ -20,6 +20,7 @@ from kumo.testing.snowflake import setup_full_snowflake_session +@pytest.mark.private_resource('snowflake') def test_logged_connector_cursor_execute_logs_and_query_id(tempdir, caplog): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -36,6 +37,7 @@ def test_logged_connector_cursor_execute_logs_and_query_id(tempdir, caplog): assert "query_id=" in caplog.text +@pytest.mark.private_resource('snowflake') def test_logged_connector_cursor_execute_logs_error(tempdir, caplog): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -51,6 +53,7 @@ def test_logged_connector_cursor_execute_logs_error(tempdir, caplog): assert "SELECT * FROM definitely_missing_table" not in caplog.text +@pytest.mark.private_resource('snowflake') def test_long_running_query_operator_stats_report_uses_threshold_override( tempdir, caplog, monkeypatch): session = setup_full_snowflake_session(tempdir) @@ -72,6 +75,7 @@ def test_long_running_query_operator_stats_report_uses_threshold_override( assert any("operator_count=" in rec.message for rec in caplog.records) +@pytest.mark.private_resource('snowflake') def test_logged_connector_connection_wraps_commit_rollback_close( tempdir, caplog): session = setup_full_snowflake_session(tempdir) @@ -90,6 +94,7 @@ def test_logged_connector_connection_wraps_commit_rollback_close( assert "operation=connection.close status=success" in caplog.text +@pytest.mark.private_resource('snowflake') def test_ensure_logged_connector_connection_idempotent(tempdir): session = setup_full_snowflake_session(tempdir) wrapped_session = ensure_logged_snowpark_session(session) @@ -99,6 +104,7 @@ def test_ensure_logged_connector_connection_idempotent(tempdir): assert wrapped is wrapped_again +@pytest.mark.private_resource('snowflake') def test_logged_snowpark_session_connection_is_wrapped(tempdir): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -110,6 +116,7 @@ def test_logged_snowpark_session_connection_is_wrapped(tempdir): assert isinstance(conn_2, _LoggedSnowflakeConnection) +@pytest.mark.private_resource('snowflake') def test_wrapped_session_compatible_with_snowpark_dataframe(tempdir): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -122,6 +129,7 @@ def test_wrapped_session_compatible_with_snowpark_dataframe(tempdir): assert isinstance(df.session, Session) +@pytest.mark.private_resource('snowflake') def test_wrapped_session_logs_query_id_for_dataframe_collect(tempdir, caplog): session = setup_full_snowflake_session(tempdir) wrapped = ensure_logged_snowpark_session(session) @@ -133,6 +141,7 @@ def test_wrapped_session_logs_query_id_for_dataframe_collect(tempdir, caplog): assert any("query_id=" in rec.message for rec in caplog.records) +@pytest.mark.private_resource('snowflake') def test_wrapped_session_file_put_logs_cursor_execute(tempdir, tmp_path, caplog): session = setup_full_snowflake_session(tempdir) @@ -328,6 +337,7 @@ def cursor(self): assert 'stats-secret-canary' not in caplog.text +@pytest.mark.private_resource('snowflake') def test_logged_cursor_ineligible_for_retry_after_multi_statement( tempdir, monkeypatch, caplog): monkeypatch.setattr(session_logging, "_QUERY_RETRY_TRIES", 3) diff --git a/test/io_utils/test_json.py b/test/io_utils/test_json.py index 7ea7cb768..a9eb6f4c3 100644 --- a/test/io_utils/test_json.py +++ b/test/io_utils/test_json.py @@ -256,6 +256,7 @@ def test_json_safe_float(): assert json_safe_float_array([4, float('inf')]) == [4.0, None] +@pytest.mark.private_resource('aws') def test_json_load_no_such_key(): path = "s3://kumo-public-datasets/something_not_exist" assert load_json(path, str, default="not_exists", @@ -283,6 +284,7 @@ def _raise_if_read(*args, **kwargs): assert read_called is False +@pytest.mark.private_resource('aws') def test_json_load_zipfile(): path = "s3://kumo-public-datasets/test_zip_file/" \ "materialization_task_info.json.gz" diff --git a/test/io_utils/test_parquet_util.py b/test/io_utils/test_parquet_util.py index e6b15350b..b5b625bc0 100644 --- a/test/io_utils/test_parquet_util.py +++ b/test/io_utils/test_parquet_util.py @@ -305,6 +305,7 @@ def test_count_rows_paritions(tmp_path: pathlib.Path): assert info.num_rows == 10 +@pytest.mark.private_resource('aws') def test_get_parquet_metadata_wildcard(tmp_path: pathlib.Path, deploy_env_config): # parquet_util.get_parquet_metadata on subsets of .parquet files in a @@ -1094,6 +1095,7 @@ def test_is_empty_helper_ucv(temp_ucv, tmp_path: pathlib.Path): assert parquet_util._is_empty_helper(empty_dir.as_url()) +@pytest.mark.private_resource('aws') def test_read_parquet_offset_size(): file_path = 's3://kumo-public-datasets/large_input_data/test1G128/' diff --git a/test/rest/config/test_artifact_export_file_output.py b/test/rest/config/test_artifact_export_file_output.py index 2a36297b7..8ff3238df 100644 --- a/test/rest/config/test_artifact_export_file_output.py +++ b/test/rest/config/test_artifact_export_file_output.py @@ -142,7 +142,9 @@ def test_prediction_artifact_preserves_multiple_destinations(): 'abfss://data@account.dfs.core.windows.net/output/'), ]) def test_destination_resolves_for_matching_connector_mode( - connector, config, expected): + connector, config, expected, monkeypatch): + monkeypatch.setattr( + 'io_utils.fsutil.s3_filesystem.select_s3_region', lambda path: None) assert resolve_file_output_path(config, connector) == expected diff --git a/test/rest/conftest.py b/test/rest/conftest.py index 390dd58b0..593de495e 100644 --- a/test/rest/conftest.py +++ b/test/rest/conftest.py @@ -263,7 +263,8 @@ def fixture_export_path( @pytest.fixture(scope="session") -def deploy_env_config(temporal_workflow_env) -> DeployEnvConfig: +def deploy_env_config(temporal_workflow_env, + test_execution_environment) -> DeployEnvConfig: os.environ['DEPLOY_STAGE'] = 'test' # The shared REST app is explicitly authentication-disabled. Worker-side # capability redemption reads this deployment setting independently of the diff --git a/test/rest/driver/test_pquery_api_driver.py b/test/rest/driver/test_pquery_api_driver.py index b388c7af9..185a229e1 100644 --- a/test/rest/driver/test_pquery_api_driver.py +++ b/test/rest/driver/test_pquery_api_driver.py @@ -3380,8 +3380,8 @@ def test_get_presigned_artifact_urls_artifact_type_not_found(artifact_type): @pytest.mark.parametrize('artifact_type', _ARTIFACT_TYPES) -def test_get_presigned_artifact_urls_file_not_found(artifact_type): - file_storage_path = 's3://kumo-public-datasets/db/rest' +def test_get_presigned_artifact_urls_file_not_found(artifact_type, tmp_path): + file_storage_path = str(tmp_path) pquery = _pquery_driver( ml_job_backend=_create_mock_ml_job_backend(file_storage_path), ) diff --git a/test/rest/integ/conftest.py b/test/rest/integ/conftest.py index 3c8589ca9..04d66fa1d 100644 --- a/test/rest/integ/conftest.py +++ b/test/rest/integ/conftest.py @@ -8,6 +8,7 @@ import sys import uuid from pathlib import Path +from test.ci_resources import execution_profile from typing import Dict, Generator, List import pytest @@ -226,7 +227,9 @@ def pquery_db_dump(request) -> FixtureDefinition: # --------------------------------------------------------------------------- -@pytest.fixture(scope="session", params=["local", "s3"]) +@pytest.fixture(scope="session", params=[ + "local", pytest.param("s3", marks=pytest.mark.private_resource('aws')), +]) def storage_backend(request): """Parametrize diskgraph tests over local and S3 storage backends.""" return request.param @@ -234,30 +237,13 @@ def storage_backend(request): @pytest.fixture(scope="session") def _diskgraph_deploy_config(tmp_path_factory, storage_backend, request, - temporal_workflow_env): - """Load diskgraph JSON config, override storage root, and set - DEPLOY_ENV_CONFIG_JSON so deploy_env() picks it up. - - DEPLOY_ENV_CONFIG_FILE must be set by the calling test module - (typically defaulted at module import time to - ``rest/devserver/configs/diskgraph-test.json``); CI sets it - explicitly via .github/workflows/testing.yml. - """ + temporal_workflow_env, test_execution_environment): + """Configure storage and workers for the selected DiskGraph backend.""" import json - # The fixture is parametrized over ``storage_backend`` (``local`` / - # ``s3``), so it runs once per parametrization. The first run pops - # ``DEPLOY_ENV_CONFIG_FILE`` after copying its contents into - # ``DEPLOY_ENV_CONFIG_JSON`` (see bottom of this fixture); the - # second invocation would otherwise hit the assertion below. - # Default it back to the diskgraph-test config if absent — same - # path the e2e test modules set at import time. - config_file = os.environ.get('DEPLOY_ENV_CONFIG_FILE') - if not config_file: - config_file = str( - Path(__file__).resolve().parents[3] / 'rest' / 'devserver' / - 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = config_file + config_file = os.environ.get('DEPLOY_ENV_CONFIG_FILE') or str( + Path(__file__).resolve().parents[3] / 'rest' / 'devserver' / + 'configs' / 'diskgraph-test.json') with open(config_file) as f: config = json.load(f) @@ -431,6 +417,7 @@ def _kill_pool_subprocesses(procs: List, log) -> None: @pytest.fixture(scope="session") +@execution_profile('diskgraph') def _pool_subprocesses(_diskgraph_deploy_config, request): """Spawn the pool controller AutoRPC server and a singlehost pool worker as long-running subprocesses for the duration of the session. diff --git a/test/rest/integ/fixture_connector_args.py b/test/rest/integ/fixture_connector_args.py index 3617df9cf..758d23ded 100644 --- a/test/rest/integ/fixture_connector_args.py +++ b/test/rest/integ/fixture_connector_args.py @@ -18,6 +18,8 @@ from kumoapi.source_table import S3RelativeSource from io_utils import fsutil +from io_utils.json import load_json +from kumo.testing.rest import randomize_fixture_ids_v2 from rest.config.api_command import FixtureDefinition from rest.config.connector import FileConnectorResourceConfig from rest.config.connector_db import InternalConnectorDBEntry @@ -31,6 +33,22 @@ 'connector_args/test_e2e_connector_args.json') +def load_selected_fixture(path: str | Path, + pquery_name: str) -> FixtureDefinition: + fixture = load_json(str(path), FixtureDefinition) + fixture.queries = [ + q for q in fixture.queries if q.pquery_name == pquery_name + ] + assert len(fixture.queries) == 1 + graph_ids = {query.graph_fpid for query in fixture.queries} + fixture.graphs = [ + graph for graph in fixture.graphs if graph.fpid in graph_ids + ] + assert len(fixture.graphs) == 1 + fixture.tables = list(fixture.graphs[0].tables.values()) + return randomize_fixture_ids_v2(fixture) + + def load_fixture_connectors( fixture: FixtureDefinition, project_id: ProjectID, @@ -247,6 +265,7 @@ async def provision_fixture_connectors( *, api_prefix: str = '/v0', fixture_root: Path | None = None, + selected_sources_only: bool = False, ) -> tuple[ProjectID, list[InternalConnectorDBEntry]]: """Upload checked-in local fixture data into the owning project. @@ -267,18 +286,38 @@ async def provision_fixture_connectors( } for old_connector_id, source_root in roots_by_connector.items(): if not Path(source_root).is_dir(): - raise ValueError( - f'Local fixture source directory does not exist: ' - f'{source_root}') + raise ValueError(f'Local fixture source directory does not exist: ' + f'{source_root}') namespace = posixpath.join('fixture-data', str(old_connector_id)) destination = posixpath.join(connector.config.root_dir.rstrip('/'), namespace) - await fsutil.copy.coroutine( - source_root, - destination, - overwrite=True, - raise_on_exists=False, - ) + if selected_sources_only: + source_connector = next(source for source in source_connectors + if source.id == old_connector_id) + resolver = dataclasses.replace( + source_connector.config, + name='fixture_source_resolver').as_connector() + sources = { + str(table.get_source_table_name()): table.source_table + for graph in fixture.graphs + for table in graph.tables.values() + if table.get_connector_id() == old_connector_id + } + for logical_path, source_table in sources.items(): + physical_path, suffix = await _physical_fixture_source( + resolver, source_table) + physical_destination = posixpath.join(destination, logical_path) + suffix + fsutil.makedirs(posixpath.dirname(physical_destination), exist_ok=True) + await fsutil.copy.coroutine( + physical_path, physical_destination, + overwrite=True, raise_on_exists=False) + else: + await fsutil.copy.coroutine( + source_root, + destination, + overwrite=True, + raise_on_exists=False, + ) remapped = _project_upload_fixture(fixture, connector.id, roots_by_connector) diff --git a/test/rest/integ/test_bigquery_diskgraph_e2e.py b/test/rest/integ/test_bigquery_diskgraph_e2e.py index 749b8d11a..94e086c0a 100644 --- a/test/rest/integ/test_bigquery_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_diskgraph_e2e.py @@ -23,7 +23,6 @@ --timeout=1800 -m bigquery """ import logging -import os from pathlib import Path from test.rest.integ.test_diskgraph_e2e import ( _check_diskgraph_train_table, @@ -51,12 +50,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks up the -# diskgraph config even when the env var isn't set by CI. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path( __file__).parent / "test_cases" / "test_bigquery_e2e_fixtures.json" @@ -67,8 +60,13 @@ # storage_backend is pinned to s3: BigQuery exports to GCS then downloads to # the dataplane, so local storage is not a meaningful test target here. pytestmark = [ + pytest.mark.ci_group('gcp-e2e'), + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py index 4f92ba144..b877f23bb 100644 --- a/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py +++ b/test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py @@ -48,10 +48,9 @@ Prerequisites: - Temporal dev server: ``make up-local-temporal`` - ``diskgraph`` Python package installed (CI uses - ``install-diskgraph: 'true'`` in ``testing_gcs.yml``) + ``install-diskgraph: 'true'`` in ``testing_private.yml``) - ``DEPLOY_ENV_CONFIG_FILE`` pointing at a diskgraph deploy - config (CI sets it for the ``bigquery_*_diskgraph`` matrix - entries; this module also defaults it at import time) + config (the shared execution fixture supplies a default) - BigQuery credentials available via the ``withBigQueryCredentials`` decorator (from AWS Secrets Manager in CI, ``GOOGLE_CREDENTIALS_STR_JSON`` fallback @@ -65,7 +64,6 @@ -xvs --timeout=900 """ import logging -import os import shutil import time from pathlib import Path @@ -107,16 +105,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks -# up the diskgraph config even when the env var isn't set by CI. -# Doing this at module import (not in an autouse fixture) because -# ``deploy_env_config`` caches its DeployEnvConfig the first time -# any test in the session imports the conftest — see notes in -# ``test_diskgraph_e2e.py``. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) _CONNECTOR_NAME = "financial_bigquery_diskgraph" _PROJECT_ID = "kumo-engineering" @@ -138,6 +126,9 @@ # assertions are scoped to local storage; pinning avoids a second skipped S3 # parametrization and keeps the CI job from spawning an unused pool. pytestmark = [ + pytest.mark.ci_group('gcp-e2e'), + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), pytest.mark.parametrize("storage_backend", ["local"], indirect=True), ] diff --git a/test/rest/integ/test_data_variety/test_table_lifecycle.py b/test/rest/integ/test_data_variety/test_table_lifecycle.py index d3dbfba85..41722c375 100644 --- a/test/rest/integ/test_data_variety/test_table_lifecycle.py +++ b/test/rest/integ/test_data_variety/test_table_lifecycle.py @@ -2,9 +2,7 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import os from dataclasses import replace -from pathlib import Path from test.rest.conftest import TestClient from test.rest.integ import fixture_connector_args from typing import Any, Dict @@ -44,17 +42,6 @@ unsupported_parquet_args, ) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks up the -# diskgraph config even when the env var isn't set by CI. Mirrors the -# sibling DG ingestion e2e modules (test_snowflake_ingestion_diskgraph_e2e.py, -# test_bigquery_ingestion_diskgraph_e2e.py): set at module import (not in a -# fixture) because deploy_env_config caches its DeployEnvConfig the first -# time the session imports the conftest. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[4] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) - # Pool fixtures (storage_backend, _diskgraph_deploy_config, _pool_subprocesses) # live in test/rest/integ/conftest.py; the diskgraph per-table refresh # (TableRefreshWorkflow -> ingest_table_data_activity_v2) needs the pool @@ -64,7 +51,9 @@ # within one session trips a known harness deploy_env leak. pytestmark = [ pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/rest/integ/test_databricks_diskgraph_e2e.py b/test/rest/integ/test_databricks_diskgraph_e2e.py index 60a84a74d..d61b576fe 100644 --- a/test/rest/integ/test_databricks_diskgraph_e2e.py +++ b/test/rest/integ/test_databricks_diskgraph_e2e.py @@ -9,10 +9,8 @@ """ import asyncio import dataclasses -import os import time import uuid -from pathlib import Path from test.rest.integ.seed_helpers import seed_and_train import kumoapi.data_source as data_source_api @@ -63,11 +61,6 @@ TrainingJobID, ) -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str( - Path(__file__).resolve().parents[3] / 'rest' / 'devserver' / 'configs' / - 'diskgraph-test.json') - pytestmark = pytest.mark.usefixtures("_pool_subprocesses") _SOURCE_CONNECTOR = 'databricks_source' diff --git a/test/rest/integ/test_diskgraph_custom_tt_e2e.py b/test/rest/integ/test_diskgraph_custom_tt_e2e.py index 9e04fc661..e3012057e 100644 --- a/test/rest/integ/test_diskgraph_custom_tt_e2e.py +++ b/test/rest/integ/test_diskgraph_custom_tt_e2e.py @@ -71,13 +71,6 @@ log = logging.getLogger(__name__) -# Set DEPLOY_ENV_CONFIG_FILE at module import time so the deploy_env_config -# fixture picks up the diskgraph-test config before any fixture imports it -# (autouse fires too late). -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path(__file__).parent / "test_cases" / "test_e2e_fixtures.json" diff --git a/test/rest/integ/test_diskgraph_e2e.py b/test/rest/integ/test_diskgraph_e2e.py index d4b986df9..418db22c3 100644 --- a/test/rest/integ/test_diskgraph_e2e.py +++ b/test/rest/integ/test_diskgraph_e2e.py @@ -23,7 +23,6 @@ """ import logging import math -import os import os.path as osp import time from pathlib import Path @@ -72,20 +71,6 @@ log = logging.getLogger(__name__) -# Diskgraph deploy config is loaded from DEPLOY_ENV_CONFIG_FILE (set by -# .github/workflows/testing.yml). The autouse fixture below overrides -# ``data_plane_storage_root`` with either a local temp dir or an S3 prefix, -# controlled by the ``storage_backend`` parametrize fixture. -# -# Default DEPLOY_ENV_CONFIG_FILE at module import time so the -# ``deploy_env_config`` fixture (created by test/rest/conftest.py via -# ``DeployEnvConfig.for_test``) reads the diskgraph-test config — not -# the default in-memory test config. Doing this in an autouse fixture is too -# late: deploy_env_config has already cached its DeployEnvConfig by then. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path(__file__).parent / "test_cases" / "test_e2e_fixtures.json" diff --git a/test/rest/integ/test_diskgraph_public_api_e2e.py b/test/rest/integ/test_diskgraph_public_api_e2e.py index f237d52c3..33d06a19f 100644 --- a/test/rest/integ/test_diskgraph_public_api_e2e.py +++ b/test/rest/integ/test_diskgraph_public_api_e2e.py @@ -44,7 +44,10 @@ import os import time from pathlib import Path -from test.rest.integ.fixture_connector_args import provision_fixture_connectors +from test.rest.integ.fixture_connector_args import ( + load_selected_fixture, + provision_fixture_connectors, +) from test.rest.integ.public_response_reader import ( flatten_public_parquet_refs, read_public_response_bytes, @@ -70,11 +73,7 @@ from kumoapi.model_plan import RunMode from io_utils.json import load_json -from kumo.testing.rest import ( - get_and_expect_success, - post_and_expect_success, - randomize_fixture_ids_v2, -) +from kumo.testing.rest import get_and_expect_success, post_and_expect_success from rest.config.api_command import FixtureDefinition from rest.config.graph_v2 import GraphDefinition from rest.config.status import EmptyResponse, IDResponse @@ -88,16 +87,6 @@ # Constants — same dataset / expectations as test_diskgraph_e2e.py # --------------------------------------------------------------------------- -# Default DEPLOY_ENV_CONFIG_FILE at module import time so the -# ``deploy_env_config`` fixture (created by test/rest/conftest.py via -# ``DeployEnvConfig.for_test``) reads the diskgraph-test config rather -# than the default in-memory test config. Same pattern as -# test_diskgraph_e2e.py — must happen before any fixture imports -# deploy_env_config (autouse is too late; the cache fires first). -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path(__file__).parent / "test_cases" / "test_e2e_fixtures.json" @@ -107,7 +96,10 @@ # even though this file exercises the public API surface — workflow dispatch # routes through ``SnapshotActivities`` on the pool task queues, so the # subprocesses must be running for ingest/train/predict to make progress. -pytestmark = pytest.mark.usefixtures("_pool_subprocesses") +pytestmark = [ + pytest.mark.ci_group('aws-public-api'), + pytest.mark.usefixtures("_pool_subprocesses"), +] PQL_TEST_CASE = "saas_temporal_regression_base" @@ -213,8 +205,8 @@ async def _create_connectors_v1( fixture: FixtureDefinition, ) -> str: """Provision project-owned integration connectors.""" - project_id, _ = await provision_fixture_connectors(client, fixture, - api_prefix='/v1') + project_id, _ = await provision_fixture_connectors( + client, fixture, api_prefix='/v1', selected_sources_only=True) return str(project_id) @@ -663,17 +655,24 @@ async def test_diskgraph_decoupled_training_and_prediction( client = public_api_client # Load and randomize fixture - fixture = load_json(str(FIXTURE_PATH), FixtureDefinition) - fixture.queries = [ - q for q in fixture.queries if q.pquery_name == PQL_TEST_CASE - ] - assert len(fixture.queries) == 1 - fixture = randomize_fixture_ids_v2(fixture) + fixture = load_selected_fixture(FIXTURE_PATH, PQL_TEST_CASE) + + phase_started = time.monotonic() + previous_phase = 'setup' + + def _record_phase(next_phase: str) -> None: + nonlocal phase_started, previous_phase + now = time.monotonic() + print( + f"CI_PHASE public-api {previous_phase} elapsed_seconds={now - phase_started:.3f}", + flush=True) + previous_phase, phase_started = next_phase, now # ==== Phase 1: Setup ==== # SDK: connector = kumo.S3Connector(...); graph = kumo.Graph(...); # graph.save(); pquery = kumo.PredictiveQuery(graph, query); # pquery.save() + _record_phase("provision") log.info("Phase 1: Creating connectors, PQuery, Graph via V1") await _create_connectors_v1(client, fixture) pquery_id, graph_id, args = await _setup_pquery_and_graph(client, fixture) @@ -683,6 +682,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # SDK: graph.snapshot(force_refresh=True) — called explicitly or # implicitly by trainer.fit(). We create it here for stats validation # and reuse it in trainer.fit() / trainer.predict() below. + _record_phase("snapshot") log.info("Phase 2: Creating graph snapshot via V1") snapshot_id = GraphSnapshotID( post_and_expect_success( @@ -706,6 +706,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # SDK: pquery.generate_training_table(plan=plan) # Note: SDK passes graph_snapshot_id=None — server triggers its own # graph refresh internally. + _record_phase("training_table") log.info("Phase 3: Generating training table via V1") plan_payload = plan_api.SuggestModelPlanRequest( @@ -765,6 +766,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # trainer.fit(graph=graph, train_table=train_table) # Internally, fit() calls graph.snapshot() and passes the snapshot_id # along with train_table.job_id to TrainingJobRequest. + _record_phase("training_A") log.info("Phase 4a: Training with seed=42 via V1") model_plan_info = post_and_expect_success( @@ -838,6 +840,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # ==== Phase 4b: Training Job B (seed=123, reuse train table) ==== # SDK: Same trainer.fit() with different model_plan seed but same # train_table.job_id — both jobs share the same train table. + _record_phase("training_B") log.info("Phase 4b: Training with seed=123, reusing train table via V1") model_plan_b = copy.deepcopy(model_plan) @@ -882,6 +885,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # ==== Phase 5: Prediction Table Generation ==== # SDK: pquery.generate_prediction_table() # Note: SDK passes graph_snapshot_id=None — server handles graph refresh. + _record_phase("prediction_table") log.info("Phase 5: Generating prediction table via V1") pred_table_job_id = post_and_expect_success( @@ -922,6 +926,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # output_config=OutputConfig(...)) # Internally, predict() calls graph.snapshot() and passes snapshot_id + # pred_table_job_id to BatchPredictionRequest. + _record_phase("prediction_and_artifacts") log.info("Phase 6: Running batch prediction via V1") from kumo.validation import ValidationResponse @@ -966,6 +971,7 @@ async def test_diskgraph_decoupled_training_and_prediction( # Verify prediction config has diskgraph backend (supplementary) _check_prediction_config_diskgraph(bp_job_id) + _record_phase("complete") log.info("Test 1 PASSED: full diskgraph decoupled flow via V1.") @@ -980,12 +986,7 @@ async def test_diskgraph_snapshot_reuse(public_api_client: TestClient, ): """Single graph snapshot reused for two train-table-gen jobs.""" client = public_api_client - fixture = load_json(str(FIXTURE_PATH), FixtureDefinition) - fixture.queries = [ - q for q in fixture.queries if q.pquery_name == PQL_TEST_CASE - ] - assert len(fixture.queries) == 1 - fixture = randomize_fixture_ids_v2(fixture) + fixture = load_selected_fixture(FIXTURE_PATH, PQL_TEST_CASE) await _create_connectors_v1(client, fixture) pquery_id, graph_id, args = await _setup_pquery_and_graph(client, fixture) @@ -1055,12 +1056,7 @@ async def test_diskgraph_graph_refresh_idempotency( """Re-creating a graph snapshot produces consistent stats.""" client = public_api_client - fixture = load_json(str(FIXTURE_PATH), FixtureDefinition) - fixture.queries = [ - q for q in fixture.queries if q.pquery_name == PQL_TEST_CASE - ] - assert len(fixture.queries) == 1 - fixture = randomize_fixture_ids_v2(fixture) + fixture = load_selected_fixture(FIXTURE_PATH, PQL_TEST_CASE) await _create_connectors_v1(client, fixture) _, graph_id, _ = await _setup_pquery_and_graph(client, fixture) diff --git a/test/rest/integ/test_fixture_connector_args.py b/test/rest/integ/test_fixture_connector_args.py index ddabcdfaa..edeaf892a 100644 --- a/test/rest/integ/test_fixture_connector_args.py +++ b/test/rest/integ/test_fixture_connector_args.py @@ -352,3 +352,39 @@ async def test_project_upload_fixture_is_hidden_on_private_and_public_routes( SourceTableListResponse, ) assert 'events' in source_tables.table_names + + +@pytest.mark.asyncio +async def test_public_api_fixture_uploads_only_its_resolved_input_bytes( + tmp_path, monkeypatch): + from test.rest.integ import fixture_connector_args as uploads + from test.rest.integ.fixture_connector_args import load_selected_fixture + + fixture = load_selected_fixture( + 'test/rest/integ/test_cases/test_e2e_fixtures.json', + 'saas_temporal_regression_base') + assert len(fixture.queries) == len(fixture.graphs) == 1 + assert set(fixture.graphs[0].tables) == {'ACCOUNT', 'LOAN', 'TRANS'} + assert len(fixture.tables) == 3 + connector = SimpleNamespace( + id=ConnectorID(f'connector_{"b" * 32}'), + config=SimpleNamespace(root_dir=str(tmp_path / 'uploads'))) + monkeypatch.setattr(uploads, 'ensure_test_project', + AsyncMock(return_value=ProjectID('default'))) + monkeypatch.setattr(uploads, '_canonical_project_upload_connector', + AsyncMock(return_value=connector)) + monkeypatch.setattr(uploads, 'bind_test_client_project', lambda *_: None) + await uploads.provision_fixture_connectors(None, fixture, + selected_sources_only=True) + copied = sorted(path for path in (tmp_path / 'uploads').rglob('*') + if path.is_file()) + assert sorted(path.name for path in copied) == [ + 'ACCOUNT.parquet', 'LOAN.parquet', 'TRANS.parquet' + ] + expected = { + 'ACCOUNT.parquet': Path('test/csv_data/FINANCIAL/ACCOUNT.parquet'), + 'TRANS.parquet': Path('test/csv_data/FINANCIAL/TRANS.parquet'), + 'LOAN.parquet': Path('test/csv_data/FINANCIAL_E2E/LOAN.parquet'), + } + for path in copied: + assert path.read_bytes() == expected[path.name].read_bytes() diff --git a/test/rest/integ/test_gcs_e2e.py b/test/rest/integ/test_gcs_e2e.py index 56f9d284a..bb226cd1b 100644 --- a/test/rest/integ/test_gcs_e2e.py +++ b/test/rest/integ/test_gcs_e2e.py @@ -66,6 +66,8 @@ # This suite covers GCS-connector e2e, not storage-backend coverage — # only test_diskgraph_e2e.py needs the [s3] parametrize. pytestmark = [ + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), pytest.mark.parametrize("storage_backend", ["local"], indirect=True), ] diff --git a/test/rest/integ/test_gcs_public_api_e2e.py b/test/rest/integ/test_gcs_public_api_e2e.py index b62a32462..930798f81 100644 --- a/test/rest/integ/test_gcs_public_api_e2e.py +++ b/test/rest/integ/test_gcs_public_api_e2e.py @@ -50,6 +50,8 @@ # (routed via ``DiskgraphIngestWorkflow`` -> ``pool.wf_coordinator``) hangs on a # no-worker AutoRPC queue and the workflow times out. pytestmark = [ + pytest.mark.ci_profile('gcp'), + pytest.mark.ci_gate('gcp-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), pytest.mark.parametrize("storage_backend", ["local"], indirect=True), ] diff --git a/test/rest/integ/test_snowflake_diskgraph_e2e.py b/test/rest/integ/test_snowflake_diskgraph_e2e.py index 2f5a81f97..cac7b16a7 100644 --- a/test/rest/integ/test_snowflake_diskgraph_e2e.py +++ b/test/rest/integ/test_snowflake_diskgraph_e2e.py @@ -53,12 +53,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks up the -# diskgraph config even when the env var isn't set by CI. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) FIXTURE_PATH = Path( __file__).parent / "test_cases" / "test_snowflake_e2e_fixtures.json" @@ -69,8 +63,11 @@ # storage_backend is pinned to s3 to exercise Kumo-owned remote persistence # before the pool-local diskgraph download. pytestmark = [ + pytest.mark.ci_gate('snowflake-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py b/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py index 29bb03bf9..71fc8060e 100644 --- a/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py +++ b/test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py @@ -58,7 +58,7 @@ Prerequisites: - Temporal dev server: ``make up-local-temporal`` - ``diskgraph`` Python package installed (CI uses - ``install-diskgraph: 'true'`` in ``testing_snowflake.yml``) + ``install-diskgraph: 'true'`` in ``testing_private.yml``) - ``DEPLOY_ENV_CONFIG_FILE`` pointing at a diskgraph deploy config - Snowflake credentials available via ``withSnowflakeCredentials`` (AWS Secrets Manager ``xva19026.snowflakecomputing.com`` or @@ -77,7 +77,6 @@ import os import time from decimal import Decimal -from pathlib import Path from test.rest.integ.test_diskgraph_e2e import ( MAX_POLL_SECONDS, POLL_INTERVAL_SECONDS, @@ -130,13 +129,6 @@ log = logging.getLogger(__name__) -# Default DEPLOY_ENV_CONFIG_FILE so _diskgraph_deploy_config picks -# up the diskgraph config even when the env var isn't set by CI. -# Mirrors test_snowflake_diskgraph_e2e.py. -if 'DEPLOY_ENV_CONFIG_FILE' not in os.environ: - _default_deploy_config = (Path(__file__).resolve().parents[3] / 'rest' / - 'devserver' / 'configs' / 'diskgraph-test.json') - os.environ['DEPLOY_ENV_CONFIG_FILE'] = str(_default_deploy_config) _CONNECTOR_NAME = "financial_snowflake_diskgraph" @@ -162,8 +154,11 @@ # parametrization hits a kumo-side stats persistence issue # unrelated to the Snowflake connector path. pytestmark = [ + pytest.mark.ci_gate('snowflake-e2e'), pytest.mark.usefixtures("_pool_subprocesses"), - pytest.mark.parametrize("storage_backend", ["s3"], indirect=True), + pytest.mark.parametrize("storage_backend", [ + pytest.param("s3", marks=pytest.mark.private_resource('aws')), + ], indirect=True), ] diff --git a/test/scripts/test_ci_resources.py b/test/scripts/test_ci_resources.py new file mode 100644 index 000000000..232644214 --- /dev/null +++ b/test/scripts/test_ci_resources.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise real pytest collection/fixture ordering without cloud access.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + + +def run_pytest(tmp_path, source, group=None, strict=False, collect=False): + test = tmp_path / 'test_selection.py' + test.write_text(source) + manifest = tmp_path / f'{group or "all"}.json' + command = [ + sys.executable, '-m', 'pytest', '-c', '/dev/null', '--noconftest', '-p', + 'test.ci_resources', '-q', '--resource-manifest', + str(manifest), + str(test) + ] + if group: + command += ['--resource-group', group] + if strict: + command += ['--require-resource-credentials'] + if collect: + command += ['--collect-only'] + result = subprocess.run( + command, capture_output=True, text=True, timeout=45, env={ + **os.environ, 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': '1' + }) + return result, json.loads(manifest.read_text()) + + +def test_public_selection_never_resolves_private_fixture(tmp_path): + source = ''' +import pytest +import runpy +requires_resource = runpy.run_path('kumo/testing/_skip.py')['requires_resource'] + +@pytest.fixture +@requires_resource('gcp') +def private_input(): + raise AssertionError('Private fixture ran on a public runner') + +@pytest.fixture +def indirect_input(private_input): + return private_input + +def test_local(): + pass + +@pytest.mark.private_resource('aws') +@pytest.mark.ci_group('aws-public-api') +def test_balanced(): + pass + +@pytest.mark.parametrize('storage_backend', [ + pytest.param('s3', marks=pytest.mark.private_resource('aws'))]) +def test_live(indirect_input, storage_backend): + pass + +@pytest.mark.parametrize('storage_backend', [ + 'local', pytest.param('s3', marks=pytest.mark.private_resource('aws'))]) +def test_backend(storage_backend): + assert storage_backend == 'local' +''' + result, public = run_pytest(tmp_path, source, 'public') + assert result.returncode == 0, result.stdout + result.stderr + assert len(public) == 2 + assert all(not value['resources'] for value in public.values()) + _, complete = run_pytest(tmp_path, source, collect=True) + _, aws = run_pytest(tmp_path, source, 'aws', collect=True) + _, gcp = run_pytest(tmp_path, source, 'gcp', collect=True) + _, balanced = run_pytest(tmp_path, source, 'aws-public-api', collect=True) + assert len(balanced) == 1 + assert set(complete) == set(public) | set(aws) | set(gcp) | set(balanced) + assert sum(map(len, (public, aws, gcp, balanced))) == len(complete) + assert not (set(public) & (set(aws) | set(gcp))) + assert not (set(aws) & set(gcp)) + assert [row['resources'] for row in gcp.values()] == [['aws', 'gcp']] + + +def test_private_ci_rejects_missing_credentials_but_preserves_intentional_skips( + tmp_path): + source = ''' +import pytest +@pytest.mark.private_resource('aws') +def test_missing(): + pytest.skip('No AWS credentials found for S3') +''' + result, _ = run_pytest(tmp_path, source, 'aws', strict=True) + assert result.returncode == 1, result.stdout + result.stderr + assert 'Private CI skipped tests' in result.stdout + source = source.replace('No AWS credentials found for S3', + 'Only run in the full suite') + result, _ = run_pytest(tmp_path, source, 'aws', strict=True) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_discovery_runs_declared_profiles_in_fresh_processes(tmp_path): + component = tmp_path / 'test' / 'new_component' + component.mkdir(parents=True) + helper = repr(str(Path('kumo/testing/_skip.py').resolve())) + source = f''' +import os +import runpy +import pytest +from test.ci_resources import execution_profile +requires_resource = runpy.run_path({helper})['requires_resource'] +IMPORTED_AUTH = {{key: os.environ.get(key) for key in ( + 'DATABRICKS_TOKEN', 'DATABRICKS_CLIENT_ID', 'DATABRICKS_CLIENT_SECRET')}} + +@pytest.fixture(scope='session') +@execution_profile('diskgraph') +def process_state(): + assert not os.path.exists(os.environ['PROFILE_PROCESS_STATE']) + with open(os.environ['PROFILE_PROCESS_STATE'], 'w') as state: + state.write(str(os.getpid())) + yield + os.unlink(os.environ['PROFILE_PROCESS_STATE']) + +@requires_resource('databricks', profile='databricks-pat') +def test_pat(process_state): + assert IMPORTED_AUTH == {{'DATABRICKS_TOKEN': 'fake-pat', + 'DATABRICKS_CLIENT_ID': '', + 'DATABRICKS_CLIENT_SECRET': ''}} + +@requires_resource('databricks', profile='databricks-oauth') +def test_oauth(process_state): + assert IMPORTED_AUTH == {{'DATABRICKS_TOKEN': '', + 'DATABRICKS_CLIENT_ID': 'fake-client', + 'DATABRICKS_CLIENT_SECRET': 'fake-secret'}} +''' + for name in ('one', 'two'): + (component / f'test_{name}.py').write_text(source) + (component / 'test_gcp.py').write_text(''' +import os +import pytest +IMPORTED_GCP = os.environ.get('IS_GCP_DEPLOY') + +@pytest.mark.private_resource('gcp') +@pytest.mark.ci_profile('gcp') +@pytest.mark.parametrize('storage_backend', [ + pytest.param('s3', marks=pytest.mark.private_resource('aws'))]) +def test_combined_identity(storage_backend): + assert IMPORTED_GCP == '1' + +@pytest.mark.private_resource('gcp') +@pytest.mark.ci_gate('gcp-e2e') +def test_gated(): + raise AssertionError('Disabled provider suite ran') +''') + # Unsupported release/script entry points must not enter component discovery. + (tmp_path / 'test' / 'test_script.py').write_text('raise RuntimeError()') + release = tmp_path / 'test' / 'release_tests' + release.mkdir() + (release / 'test_release.py').write_text('raise RuntimeError()') + env = { + **os.environ, + 'PYTEST_DISABLE_PLUGIN_AUTOLOAD': + '1', + 'PYTHONPATH': + os.pathsep.join([ + str(Path(__file__).resolve().parents[2]), + os.environ.get('PYTHONPATH', '') + ]), + 'PYTEST_ADDOPTS': + '--noconftest -c /dev/null -p test.ci_resources -p pytest_timeout', + 'DATABRICKS_TOKEN': + 'fake-pat', + 'DATABRICKS_CLIENT_ID': + 'fake-client', + 'DATABRICKS_CLIENT_SECRET': + 'fake-secret', + 'IS_GCP_DEPLOY': + '0', + 'CI_RUN_GCP_E2E': + 'false', + 'PROFILE_PROCESS_STATE': + str(tmp_path / 'process-state'), + } + for group, sessions, cases in [('databricks', 4, 4), ('gcp', 1, 1)]: + output = tmp_path / group + result = subprocess.run( + [sys.executable, '-m', 'tools.run_private_tests', group], + cwd=tmp_path, env={ + **env, 'CI_TEST_RESULTS': str(output) + }, capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stdout + result.stderr + discovered = json.loads((output / 'discovery.json').read_text()) + selections = [ + json.loads(path.read_text()) + for path in output.glob('*-selection.json') + ] + assert len(selections) == sessions + assert len(discovered) == cases + assert discovered == { + key: value + for part in selections + for key, value in part.items() + } + assert len(list(output.glob('*.xml'))) == sessions + if group == 'gcp': + assert [row['resources'] + for row in discovered.values()] == [['aws', 'gcp']] + + +def test_phase_timings_keep_only_safe_fields(tmp_path): + source = """ +def test_phase(): + print('CI_PHASE public-api provision elapsed_seconds=1.234https://example.test/data?X-Amz-Signature=sensitive') +""" + result, _ = run_pytest(tmp_path, source, 'public') + assert result.returncode == 0, result.stdout + result.stderr + assert 'CI_PHASE public-api provision elapsed_seconds=1.234' in result.stdout + assert 'X-Amz-Signature' not in result.stdout + + +def test_private_output_redacts_download_queries(monkeypatch): + from tools.run_private_tests import redact_download_urls + monkeypatch.setenv('DATABRICKS_CLIENT_SECRET', 'synthetic-private-secret') + text = 'download https://example.test/data?sig=sensitive&token=value end' + assert redact_download_urls( + text) == 'download https://example.test/data?REDACTED end' + assert redact_download_urls( + 'secret synthetic-private-secret') == 'secret REDACTED' + + +def test_provider_fixture_changes_keep_e2e_enabled(): + from tools.ci_private_changes import selection + matrix, gcp, _ = selection( + ['test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json']) + assert gcp + assert {'group': 'gcp-e2e'} in matrix['include'] + matrix, _, _ = selection(['docs/connector-test-matrix.md']) + assert matrix == {'include': [{'group': 'none'}]} + matrix, gcp, snowflake = selection(['README.md'], force=True) + assert len(matrix['include']) == 7 + assert gcp and snowflake diff --git a/tools/ci_private_changes.py b/tools/ci_private_changes.py new file mode 100644 index 000000000..12f9b3702 --- /dev/null +++ b/tools/ci_private_changes.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Choose private jobs from the approved PR's complete changed-file list.""" + +import fnmatch +import json +import os + + +def matches(files, patterns): + return any( + fnmatch.fnmatchcase(path, pattern) for path in files + for pattern in patterns) + + +def selection(files, force=False): + code = force or any( + not matches([path], ('ui/**', 'docs/**', 'helm/**', 'tools/**', + 'benchmark/**', '*.md')) or path in + ('tools/run_private_tests.py', 'tools/ci_private_changes.py') + for path in files) + shared = ('.github/actions/setup/**', + '.github/workflows/testing_private.yml', 'test/ci_resources.py', + 'kumo/testing/**', 'setup.py', 'uv.lock', + 'tools/run_private_tests.py', 'tools/ci_private_changes.py') + gcp = force or matches( + files, (*shared, 'kumo/connector/**', 'io_utils/**', 'rest/**', + 'packages/kumo-core/**', 'test/rest/integ/test_gcs*', + 'test/rest/integ/test_bigquery*', + 'test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json')) + snowflake = force or matches( + files, (*shared, 'kumo/connector/**', + 'rest/backends/data_backend/activities/**', + 'test/rest/integ/test_snowflake*', + 'test/rest/integ/test_cases/test_snowflake_e2e_fixtures.json', + 'rest/devserver/configs/diskgraph-test.json')) + groups = [ + 'aws', 'aws-public-api', 'gcp', 'azure', 'snowflake', 'databricks' + ] if code else [] + if code and gcp: + groups.append('gcp-e2e') + # A sentinel avoids GitHub's invalid empty matrix; the test job skips it. + matrix = { + 'include': [{ + 'group': group + } for group in groups] or [{ + 'group': 'none' + }] + } + return matrix, gcp, snowflake + + +if __name__ == '__main__': + matrix, gcp, snowflake = selection( + json.loads(os.environ['CHANGED_FILES']), + os.environ.get('FORCE_PRIVATE_TESTS') == 'true') + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'matrix={json.dumps(matrix)}\n') + output.write( + f'run={str(matrix["include"][0]["group"] != "none").lower()}\n') + output.write(f'gcp-e2e={str(gcp).lower()}\n') + output.write(f'snowflake-e2e={str(snowflake).lower()}\n') diff --git a/tools/run_private_tests.py b/tools/run_private_tests.py new file mode 100644 index 000000000..9ce2b0d8c --- /dev/null +++ b/tools/run_private_tests.py @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Discover private cases and run their declared profiles in fresh processes.""" + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from test.ci_resources import GROUPS, PROFILE_ENV + + +def redact_download_urls(text): + for name, value in os.environ.items(): + if value and (name.endswith( + ('_TOKEN', '_SECRET', '_SECRET_ACCESS_KEY', '_PASSWORD', + '_PRIVATE_KEY')) or name == 'AWS_ACCESS_KEY_ID'): + text = text.replace(value, 'REDACTED') + return re.sub( + r"https?://[^\s'\"<>]+", + lambda match: match[0].split('?', 1)[0] + '?REDACTED' + if '?' in match[0] else match[0], text) + + +def run_pytest(args, env, output, name, *, coverage=False): + command = [sys.executable, '-X', 'faulthandler', '-m'] + if coverage: + command += [ + 'coverage', 'run', '--data-file', + str(output / f'{name}.coverage'), '-m' + ] + command += [ + 'pytest', '--rootdir=.', '--capture=fd', '--verbosity=0', + '--disable-warnings', *args + ] + with (output / f'{name}.log').open('w') as log: + process = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + for line in process.stdout: + line = redact_download_urls(line) + print(line, end='', flush=True) + log.write(line) + result = process.wait() + junit = output / f'{name}.xml' + if junit.exists(): + junit.write_text(redact_download_urls(junit.read_text())) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('group', choices=GROUPS[1:]) + parser.add_argument('paths', nargs='*', + help='Override component collection roots') + parser.add_argument('--collect-only', action='store_true') + args = parser.parse_args() + output = Path(os.environ.get('CI_TEST_RESULTS', 'ci-test-results')) + output.mkdir(parents=True, exist_ok=True) + # Hosted component suites are subdirectories of test/. Root-level script + # tests and release tooling have separate entry points and prerequisites. + roots = args.paths or [ + str(path) for path in sorted(Path('test').iterdir()) if path.is_dir() + and not path.name.startswith('_') and path.name != 'release_tests' + ] + discovery = output / 'discovery.json' + result = run_pytest([ + *roots, '--collect-only', '--resource-group', args.group, + '--private-discovery', '--resource-manifest', + str(discovery) + ], os.environ.copy(), output, 'discovery') + if result: + return result + sessions = {} + for nodeid, metadata in json.loads(discovery.read_text()).items(): + key = (tuple(metadata['profiles']), metadata['isolated_module']) + sessions.setdefault(key, {})[nodeid] = metadata + if not sessions: + raise RuntimeError( + f'No tests discovered for private group {args.group}') + failed = False + for index, ((profiles, module), + expected) in enumerate(sorted(sessions.items())): + name = f'{args.group}-{index}' + selection = output / f'{name}-selection.json' + selection.write_text(json.dumps(expected, indent=2) + '\n') + print(f'CI_SELECTION {name} profiles={profiles} module={module}', + flush=True) + if args.collect_only: + continue + env = os.environ.copy() + for profile in profiles: + env.update(PROFILE_ENV[profile]) + failed |= run_pytest([ + *expected, + '--resource-group', + args.group, + '--require-resource-credentials', + '--resource-manifest', + str(selection), + '--junitxml', + str(output / f'{name}.xml'), + '--durations=0', + '--timeout=1200', + '--timeout-method=signal', + ], env, output, name, coverage=True) != 0 + if json.loads(selection.read_text()) != expected: + print( + f'CI selection changed between discovery and execution: {name}', + flush=True) + failed = True + return int(failed) + + +if __name__ == '__main__': + raise SystemExit(main()) From 4941bd60fc5cdc09b0b12d1dff1685c697c291bc Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Thu, 29 Oct 2026 22:00:14 +0000 Subject: [PATCH 2/3] Skip the coverage upload when no artifact name was set The artifact name is set by a step that runs only if every earlier step passed. After an earlier failure, the upload ran with an empty name and failed with "Input required and not supplied: path", adding a second error to the job. Co-Authored-By: Claude (cherry picked from commit 3411cb24427d8793f2daa1ab9576c1820df8a292) --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d390f0582..976cad880 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -256,7 +256,7 @@ jobs: KUMO_TESTING: "True" - name: Upload coverage artifact - if: always() + if: always() && env.ARTIFACT_NAME_SAFE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_SAFE }} From d95c014e565755f350d9a429ada56c3318ce08f5 Mon Sep 17 00:00:00 2001 From: Aaron Braunstein Date: Fri, 30 Oct 2026 20:59:36 +0000 Subject: [PATCH 3/3] Run approved OpenAPI checks and compare complete PR changes Analyze OpenAPI schemas on approved copied PR branches, publish schema patches, and update PR comments after checking the current head. Share copied-PR validation and local merge-base selection with private CI, and document the required CI configuration. Adapt the merge-base approach from Vedaant Jain's commit 2c6ddfc9a456d679ff915b288edcd84748dcd68f. Co-authored-by: Vedaant Jain --- .github/scripts/approved-pr.test.cjs | 141 ++++++++++++++++++ .github/scripts/copied-pr.cjs | 34 +++++ .github/scripts/openapi-pr.cjs | 44 ++++++ .github/workflows/testing_private.yml | 20 +-- .github/workflows/ui-client-code-generate.yml | 140 +++++++---------- .github/workflows/workflow-security.yml | 2 +- tools/ci_private_changes.py | 2 +- 7 files changed, 286 insertions(+), 97 deletions(-) create mode 100644 .github/scripts/approved-pr.test.cjs create mode 100644 .github/scripts/copied-pr.cjs create mode 100644 .github/scripts/openapi-pr.cjs diff --git a/.github/scripts/approved-pr.test.cjs b/.github/scripts/approved-pr.test.cjs new file mode 100644 index 000000000..aa8a2c3ea --- /dev/null +++ b/.github/scripts/approved-pr.test.cjs @@ -0,0 +1,141 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { readCopiedPullRequest, setCopiedPullRequestOutputs } = require('./copied-pr.cjs'); +const { reportOpenAPI } = require('./openapi-pr.cjs'); + +function fixture({ changed = true, comments = [], fork = true } = {}) { + const context = { + eventName: 'push', ref: 'refs/heads/pull-request/42', sha: 'a'.repeat(40), + repo: { owner: 'example', repo: 'engine' }, serverUrl: 'https://github.com', runId: 123, + }; + const pr = { + number: 42, state: 'open', + head: { sha: context.sha, repo: { full_name: fork ? 'contributor/engine' : 'example/engine' } }, + base: { sha: 'b'.repeat(40), repo: { full_name: 'example/engine' } }, + }; + const writes = []; + const github = { + rest: { + pulls: { get: async (args) => { assert.equal(args.pull_number, 42); return { data: pr }; } }, + issues: { + listComments: () => {}, + createComment: async (args) => writes.push({ kind: 'create', ...args }), + updateComment: async (args) => writes.push({ kind: 'update', ...args }), + }, + }, + paginate: async () => comments, + }; + return { github, context, changed, pr, writes }; +} + +test('only current copied commits of open PRs targeting this repository are accepted', async () => { + const f = fixture(); + assert.equal(await readCopiedPullRequest(f), f.pr); + for (const [field, value] of [['eventName', 'pull_request'], ['ref', 'refs/heads/main'], ['ref', 'refs/heads/pull-request/42/extra']]) { + await assert.rejects(readCopiedPullRequest({ ...f, context: { ...f.context, [field]: value } }), /copy-pr-bot/); + } + f.pr.head.sha = 'c'.repeat(40); + await assert.rejects(readCopiedPullRequest(f), /stale/); + f.pr.head.sha = f.context.sha; + f.pr.state = 'closed'; + await assert.rejects(readCopiedPullRequest(f), /open pull request/); + f.pr.state = 'open'; + f.pr.base.repo.full_name = 'other/engine'; + await assert.rejects(readCopiedPullRequest(f), /this repository/); +}); + +test('merge-base selection retains earlier PR changes and excludes independent base changes', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'copied-pr-base-')); + const previous = process.cwd(); + const git = (...args) => execFileSync('git', ['-c', 'user.name=CI Test', '-c', 'user.email=ci@example.test', + '-c', 'commit.gpgsign=false', '-c', 'core.hooksPath=/dev/null', ...args], { cwd: dir, encoding: 'utf8' }).trim(); + const commit = (file) => { + fs.writeFileSync(path.join(dir, file), file); + git('add', file); + git('commit', '-qm', file); + return git('rev-parse', 'HEAD'); + }; + try { + git('init', '-qb', 'main'); + const common = commit('base.txt'); + git('checkout', '-qb', 'feature'); + commit('service.py'); + const head = commit('README.md'); + git('checkout', '-q', 'main'); + const base = commit('unrelated-gcp.py'); + git('checkout', '-q', 'feature'); + const f = fixture(); + f.context.sha = f.pr.head.sha = head; + f.pr.base.sha = base; + const outputs = {}; + process.chdir(dir); + await setCopiedPullRequestOutputs({ ...f, core: { setOutput: (key, value) => { outputs[key] = value; } } }); + assert.deepEqual(outputs, { number: '42', head, base: common }); + assert.deepEqual(git('diff', '--name-only', `${outputs.base}..${head}`).split('\n'), ['README.md', 'service.py']); + } finally { + process.chdir(previous); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('fork warning links the analyzed SHA, schema patch, and local generation command', async () => { + const f = fixture(); + await reportOpenAPI(f); + assert.equal(f.writes.length, 1); + assert.equal(f.writes[0].issue_number, 42); + assert.match(f.writes[0].body, //); + assert.match(f.writes[0].body, /make generate-ui-client-code/); + assert.match(f.writes[0].body, /push.*fork/); + assert.match(f.writes[0].body, /actions\/runs\/123/); + assert.match(f.writes[0].body, /View the schema patch in the workflow run/); + assert.ok(f.writes[0].body.includes(f.context.sha)); + assert.doesNotMatch(f.writes[0].body, /trigger-ui-codegen/); +}); + +test('same-repository warning preserves the authorized code-generation command', async () => { + const f = fixture({ fork: false }); + await reportOpenAPI(f); + assert.match(f.writes[0].body, /trigger-ui-codegen/); +}); + +test('an existing bot warning is updated when schemas become current', async () => { + const f = fixture({ changed: false, comments: [ + { id: 1, user: { login: 'contributor' }, body: ' spoof' }, + { id: 2, user: { login: 'github-actions[bot]' }, body: '⚠️ **OpenAPI spec changes detected in this PR**' }, + ] }); + await reportOpenAPI(f); + assert.equal(f.writes[0].kind, 'update'); + assert.equal(f.writes[0].comment_id, 2); + assert.match(f.writes[0].body, /up to date/); +}); + +test('a repeated warning updates the bot comment instead of creating duplicates', async () => { + const f = fixture({ comments: [{ id: 7, user: { login: 'github-actions[bot]' }, body: ' old' }] }); + await reportOpenAPI(f); + assert.equal(f.writes[0].kind, 'update'); + assert.equal(f.writes[0].comment_id, 7); +}); + +test('current schemas do not create an unsolicited success comment', async () => { + const f = fixture({ changed: false }); + await reportOpenAPI(f); + assert.equal(f.writes.length, 0); +}); + +test('a PR update during reporting prevents a stale write', async () => { + const f = fixture(); + f.github.paginate = async () => { + f.pr.head.sha = 'c'.repeat(40); + return []; + }; + await assert.rejects(reportOpenAPI(f), /stale/); + assert.equal(f.writes.length, 0); +}); diff --git a/.github/scripts/copied-pr.cjs b/.github/scripts/copied-pr.cjs new file mode 100644 index 000000000..3099660f7 --- /dev/null +++ b/.github/scripts/copied-pr.cjs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { execFileSync } = require('node:child_process'); + +async function readCopiedPullRequest({ github, context }) { + const match = /^refs\/heads\/pull-request\/([1-9][0-9]*)$/.exec(context.ref); + if (context.eventName !== 'push' || !match) { + throw new Error('Expected a copy-pr-bot branch push'); + } + const { data: pr } = await github.rest.pulls.get({ + ...context.repo, + pull_number: Number(match[1]), + }); + if (pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}` || pr.state !== 'open') { + throw new Error('Expected an open pull request targeting this repository'); + } + if (pr.head.sha !== context.sha) { + throw new Error('Copied PR is stale; approve the current SHA'); + } + return pr; +} + +async function setCopiedPullRequestOutputs({ github, context, core }) { + const pr = await readCopiedPullRequest({ github, context }); + // Diff the complete PR, excluding changes made only on the base branch. + const base = execFileSync('git', ['merge-base', pr.base.sha, context.sha], { encoding: 'utf8' }).trim(); + core.setOutput('number', String(pr.number)); + core.setOutput('head', pr.head.sha); + core.setOutput('base', base); +} + +module.exports = { readCopiedPullRequest, setCopiedPullRequestOutputs }; diff --git a/.github/scripts/openapi-pr.cjs b/.github/scripts/openapi-pr.cjs new file mode 100644 index 000000000..1689d5086 --- /dev/null +++ b/.github/scripts/openapi-pr.cjs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { readCopiedPullRequest } = require('./copied-pr.cjs'); + +const MARKER = ''; +const LEGACY_HEADING = '⚠️ **OpenAPI spec changes detected in this PR**'; + +async function reportOpenAPI({ github, context, changed }) { + const pr = await readCopiedPullRequest({ github, context }); + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number: pr.number, + per_page: 100, + }); + const existing = comments.find((comment) => comment.user?.login === 'github-actions[bot]' && + (comment.body?.startsWith(MARKER) || comment.body?.startsWith(LEGACY_HEADING))); + if (!changed && !existing) return; + + const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const instructions = pr.head.repo.full_name === pr.base.repo.full_name + ? 'Run `make generate-ui-client-code` locally, or comment `/trigger-ui-codegen` if you have repository write access.' + : 'Run `make generate-ui-client-code` locally and push the regenerated files to your fork.'; + const body = [ + MARKER, + changed ? '⚠️ **OpenAPI schema regeneration needed**' : '✅ **OpenAPI schemas are up to date**', + '', + `Analyzed commit: \`${context.sha}\`.`, + '', + changed ? `${instructions} [View the schema patch in the workflow run](${run}).` + : `[View the workflow run](${run}).`, + ].join('\n'); + + // The PR may have advanced while comments were being fetched. + await readCopiedPullRequest({ github, context }); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ ...context.repo, issue_number: pr.number, body }); + } +} + +module.exports = { reportOpenAPI }; diff --git a/.github/workflows/testing_private.yml b/.github/workflows/testing_private.yml index 4d6002580..47c35e36b 100644 --- a/.github/workflows/testing_private.yml +++ b/.github/workflows/testing_private.yml @@ -38,25 +38,19 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Read copied PR context + - name: Verify the copied PR and find its merge base id: pr if: startsWith(github.ref, 'refs/heads/pull-request/') - uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf - - name: Verify the copied commit - if: startsWith(github.ref, 'refs/heads/pull-request/') - env: - PR_INFO: ${{ steps.pr.outputs.pr-info }} - run: | - python3 - <<'PY' - import json, os - pr = json.loads(os.environ['PR_INFO']) - assert pr['head']['sha'] == os.environ['GITHUB_SHA'], 'Copied PR is stale; approve the current SHA' - PY + uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 + with: + script: | + const { setCopiedPullRequestOutputs } = require('./.github/scripts/copied-pr.cjs'); + await setCopiedPullRequestOutputs({ github, context, core }); - name: Read complete changed-file list id: files uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 with: - base_sha: ${{ steps.pr.outputs.pr-info && fromJSON(steps.pr.outputs.pr-info).base.sha || github.event.before }} + base_sha: ${{ steps.pr.outputs.base || github.event.before }} json: true escape_json: false - name: Select private jobs diff --git a/.github/workflows/ui-client-code-generate.yml b/.github/workflows/ui-client-code-generate.yml index e5a431c17..d11b51199 100644 --- a/.github/workflows/ui-client-code-generate.yml +++ b/.github/workflows/ui-client-code-generate.yml @@ -6,70 +6,54 @@ name: Auto UI client code gen using Orval on: # yamllint disable-line rule:truthy workflow_dispatch: - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'rest/app.py' - - 'rest/config/**' - - 'rest/main.py' - - 'rest/routers/**' - - 'rest/scripts/download_schema.py' - - 'rest/utils/routers.py' - - 'ui/src/gen/spec/**' + # Schema analysis waits for approval of the exact commit by copy-pr-bot. + push: + branches: ["pull-request/[0-9]+"] issue_comment: types: [created] concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}-${{ startsWith(github.ref, 'refs/pull/') || github.run_number }} - # Only cancel intermediate builds if on a PR: - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} - -env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} + group: ${{ github.workflow }}-${{ github.event.issue.number || github.ref }}-${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'approved' || github.run_number }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} permissions: - contents: write - pull-requests: write - issues: write - packages: read + contents: read jobs: check-openapi-changes: name: Check OpenAPI Changes runs-on: ubuntu-24.04 - if: github.event_name == 'pull_request' + timeout-minutes: 45 + if: github.event_name == 'push' permissions: contents: read pull-requests: write - issues: write - packages: read - outputs: - spec_changed: ${{steps.openapi-changes.outputs.changed}} steps: - - name: Checkout PR branch + - name: Checkout the approved commit uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: ref: ${{ github.sha }} - # Shallow checkout to avoid a full clone of git history - fetch-depth: 1 + fetch-depth: 0 + persist-credentials: false - - name: Set up Python - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Verify the copied PR and find its merge base + id: pr + uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 with: - python-version: "3.14" - activate-environment: true - enable-cache: false + script: | + const { setCopiedPullRequestOutputs } = require('./.github/scripts/copied-pr.cjs'); + await setCopiedPullRequestOutputs({ github, context, core }); - name: Check for backend/API changes id: backend-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 with: + base_sha: ${{ steps.pr.outputs.base }} files: | + .github/actions/setup/** + .github/scripts/copied-pr.cjs + .github/scripts/openapi-pr.cjs + .github/workflows/ui-client-code-generate.yml rest/app.py rest/config/** rest/main.py @@ -83,8 +67,8 @@ jobs: uses: ./.github/actions/setup with: python-version: "3.14" - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - ref: ${{ github.event.pull_request.head.sha }} + github_token: ${{ github.token }} + ref: ${{ steps.pr.outputs.head }} - name: Download openapi.json files if: steps.backend-changes.outputs.any_changed == 'true' @@ -113,56 +97,48 @@ jobs: cd ui npm run lint-format-codegen --subdir=spec - - name: Check for Spec changes + - name: Check for schema changes if: steps.backend-changes.outputs.any_changed == 'true' id: openapi-changes run: | - if git diff --quiet --exit-code ui/src/gen/spec/; then - echo "changed=false" >> $GITHUB_OUTPUT + if git diff --quiet --exit-code -- ui/src/gen/spec/; then + echo "changed=false" >> "$GITHUB_OUTPUT" else - echo "changed=true" >> $GITHUB_OUTPUT + echo "changed=true" >> "$GITHUB_OUTPUT" + git diff --binary -- ui/src/gen/spec/ > "$RUNNER_TEMP/openapi-schema.patch" fi - - name: Post warning comment if changes detected - if: steps.backend-changes.outputs.any_changed == 'true' && steps.openapi-changes.outputs.changed == 'true' + - name: Upload the schema patch + if: steps.openapi-changes.outputs.changed == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: openapi-schema-patch + path: ${{ runner.temp }}/openapi-schema.patch + if-no-files-found: error + retention-days: 7 + + - name: Summarize schema analysis + env: + SCHEMA_CHANGED: ${{ steps.openapi-changes.outputs.changed }} + API_CHANGED: ${{ steps.backend-changes.outputs.any_changed }} + run: | + if [[ "$SCHEMA_CHANGED" == true ]]; then + echo 'OpenAPI schemas need regeneration. Download the schema patch from this run.' >> "$GITHUB_STEP_SUMMARY" + elif [[ "$API_CHANGED" == true ]]; then + echo 'OpenAPI schemas are up to date.' >> "$GITHUB_STEP_SUMMARY" + else + echo 'No OpenAPI-related files changed.' >> "$GITHUB_STEP_SUMMARY" + fi + echo "Analyzed commit: $GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" + + - name: Update the OpenAPI PR comment + env: + SCHEMA_CHANGED: ${{ steps.openapi-changes.outputs.changed }} uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 with: - github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const warningMessage = `⚠️ **OpenAPI spec changes detected in this PR** - - This means the UI client code needs to be regenerated. - - To regenerate the UI client code in this PR, comment: - \`/trigger-ui-codegen\``; - - // Find existing warning comment - const comments = await github.rest.issues.listComments({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }); - - const existingComment = comments.data.find(comment => - comment.body.includes('⚠️ **OpenAPI spec changes detected in this PR**') - ); - - // Update existing comment or create new one - if (existingComment) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existingComment.id, - body: warningMessage - }); - } else { - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: warningMessage - }); - } + const { reportOpenAPI } = require('./.github/scripts/openapi-pr.cjs'); + await reportOpenAPI({ github, context, changed: process.env.SCHEMA_CHANGED === 'true' }); trigger-codegen: name: Trigger UI Codegen diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index 44c29f679..fbb629848 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -22,6 +22,6 @@ jobs: with: persist-credentials: false - name: Test workflow security policy - run: node --test .github/scripts/workflow-security.test.cjs + run: node --test .github/scripts/workflow-security.test.cjs .github/scripts/approved-pr.test.cjs - name: Reject new workflow security findings run: node .github/scripts/workflow-security.cjs . --baseline .github/workflow-security-baseline.json diff --git a/tools/ci_private_changes.py b/tools/ci_private_changes.py index 12f9b3702..aca158c1c 100644 --- a/tools/ci_private_changes.py +++ b/tools/ci_private_changes.py @@ -20,7 +20,7 @@ def selection(files, force=False): 'benchmark/**', '*.md')) or path in ('tools/run_private_tests.py', 'tools/ci_private_changes.py') for path in files) - shared = ('.github/actions/setup/**', + shared = ('.github/actions/setup/**', '.github/scripts/copied-pr.cjs', '.github/workflows/testing_private.yml', 'test/ci_resources.py', 'kumo/testing/**', 'setup.py', 'uv.lock', 'tools/run_private_tests.py', 'tools/ci_private_changes.py')