diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml index 1f578bfee..f90ddf0e5 100644 --- a/.github/workflows/integration-cassandra.yml +++ b/.github/workflows/integration-cassandra.yml @@ -99,9 +99,33 @@ jobs: env: EXTENDDB_TEST_CASSANDRA: required + # One release build shared by the pytest and rust jobs; see integration.yml. + build-cassandra: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Build release (cassandra backend) + run: cargo build --release --no-default-features --features cassandra + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-cassandra + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + cassandra-pytest: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 45 + needs: build-cassandra services: cassandra: image: cassandra:4.1 @@ -119,12 +143,6 @@ jobs: --health-start-period 60s steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Configure Cassandra authentication run: | docker exec ${{ job.services.cassandra.id }} sed -i \ @@ -147,14 +165,19 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (cassandra backend) - run: cargo build --release --no-default-features --features cassandra + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-cassandra + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Run Cassandra pytest suite run: devtools/run-cassandra-tests -- --pytest --comprehensive --parallel cassandra-rust: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 45 + needs: build-cassandra services: cassandra: image: cassandra:4.1 @@ -200,8 +223,13 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (cassandra backend) - run: cargo build --release --no-default-features --features cassandra + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-cassandra + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Run Cassandra rust integration suite run: devtools/run-cassandra-tests -- --rust-integration diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml index 01e28552f..aedef1205 100644 --- a/.github/workflows/integration-mongodb.yml +++ b/.github/workflows/integration-mongodb.yml @@ -29,10 +29,8 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - mongodb-pytest: - # Dedicated ExtendDB runner group (Ubuntu 24.04, 4 vCPU / 16 GB, capped at - # 20 concurrent). ubuntu-latest is an Amazon-wide shared pool where our jobs - # waited 10-90 minutes to start (2026-09-29/30); see integration.yml. + # One release build shared by the pytest and rust jobs; see integration.yml. + build-mongodb: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 steps: @@ -43,16 +41,59 @@ jobs: cache-on-failure: true # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Build release (mongodb backend) + run: cargo build --release --no-default-features --features mongodb-test-hooks + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-mongodb + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + + mongodb-pytest: + # Dedicated ExtendDB runner group (Ubuntu 24.04, 4 vCPU / 16 GB, capped at + # 20 concurrent). ubuntu-latest is an Amazon-wide shared pool where our jobs + # waited 10-90 minutes to start (2026-09-29/30); see integration.yml. + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + needs: build-mongodb + # Three shards by test file; mongodb-integration checks that together + # they collected every test of the whole suite. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3 + steps: + - uses: actions/checkout@v6 - uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (mongodb backend) - run: cargo build --release --no-default-features --features mongodb-test-hooks + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-mongodb + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Run MongoDB pytest suite run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel + # mongodb-integration checks these lists. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: shards-mongodb-pytest-${{ matrix.shard }} + path: discussions/shards/ + if-no-files-found: warn + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + # SnapshotEngine consistency check against a disposable replica set. The # tests build their own per-test scratch databases (snap_cat and # _data) and drop them; they run in this job because it is the one that @@ -107,6 +148,7 @@ jobs: mongodb-rust: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-mongodb steps: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable @@ -120,8 +162,13 @@ jobs: python-version: "3.11" - name: Install Python dependencies run: pip install -r requirements.txt - - name: Build release (mongodb backend) - run: cargo build --release --no-default-features --features mongodb-test-hooks + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-mongodb + path: target/release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Run MongoDB rust integration suite run: devtools/run-mongodb-tests -- --rust --rust-integration @@ -152,3 +199,12 @@ jobs: [ "${{ needs.mongodb-snapshot.result }}" != "success" ]; then exit 1 fi + # Fails unless the shards together collected every test of the whole + # suite exactly once, so a bad split cannot pass. + - uses: actions/checkout@v6 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: shards-mongodb-pytest-* + path: shards/mongodb-pytest + - name: Check the shards ran every test + run: python3 devtools/test-shards check shards/mongodb-pytest diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b1de36c0b..5339b373c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -20,6 +20,61 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # One release build per feature set. The jobs below download the binary + # instead of each compiling the same one, so the cargo flags and features of + # a test job live in the build job it needs. + build-postgres: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build release + run: cargo build --release + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-postgres + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + + build-sqlite: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build release (SQLite backend) + run: cargo build --release -p extenddb --no-default-features --features sqlite + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: extenddb-sqlite + path: target/release/extenddb + if-no-files-found: error + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + run-integration: # Dedicated ExtendDB runner group (4 vCPU / 16 GB, capped at 20 concurrent), # pinned to Ubuntu 24.04. ubuntu-latest is an Amazon-wide pool of 1,000 @@ -29,6 +84,15 @@ jobs: # 2026 and this compiles Rust; move to 26 deliberately. runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-postgres + # Three shards by test file (devtools/test-shards); the integration job + # checks that together they collected every test of the whole suite. + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3 services: postgres: # pgvector's own image, which is postgres:16 plus the extension. The @@ -49,16 +113,13 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@stable - - - uses: Swatinem/rust-cache@v2 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} + name: extenddb-postgres + path: target/release - - name: Build release - run: cargo build --release + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -107,22 +168,37 @@ jobs: if: failure() run: sudo journalctl -t extenddb --no-pager -n 500 || true + # The integration job checks these lists. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: shards-run-integration-${{ matrix.shard }} + path: discussions/shards/ + if-no-files-found: warn + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + run-integration-sqlite: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-sqlite + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3 steps: - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@stable - - - uses: Swatinem/rust-cache@v2 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - cache-on-failure: true - # Only pushes to main save, so other runs restore main's cache. - save-if: ${{ github.ref == 'refs/heads/main' }} + name: extenddb-sqlite + path: target/release - - name: Build release (SQLite backend) - run: cargo build --release -p extenddb --no-default-features --features sqlite + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -167,6 +243,17 @@ jobs: if: failure() run: sudo journalctl -t extenddb --no-pager -n 500 || true + # The integration job checks these lists. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: shards-run-integration-sqlite-${{ matrix.shard }} + path: discussions/shards/ + if-no-files-found: warn + retention-days: 1 + # A re-run of a failed job uploads the same name again. + overwrite: true + run-integration-dev-mode: # dev-mode was shipped with no CI coverage at all, which is how the batch and # transaction authorization regression reached main: the build compiled, so @@ -225,6 +312,7 @@ jobs: run-rust-integration: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-postgres services: postgres: image: pgvector/pgvector:pg16 @@ -249,8 +337,14 @@ jobs: # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Build release - run: cargo build --release + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-postgres + path: target/release + + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -303,6 +397,42 @@ jobs: EXTENDDB_EXPECT_VECTORS: "1" run: devtools/run-tests --extenddb --rust-integration --release + # The daemonized server logs to syslog; dump it so server-side failures + # are diagnosable from the job log. + - name: Dump server syslog on failure + if: failure() + run: sudo journalctl -t extenddb --no-pager -n 500 || true + + # The PostgreSQL storage-level tests drive the storage layer directly and + # need only the database, not a running server, so they run beside + # run-rust-integration instead of after it. + run-postgres-storage-tests: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_PASSWORD: devpass + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 5 + ports: + - 127.0.0.1:5432:5432 + + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + # The control plane for vector indexes is not reachable over the wire while # this backend declares no vector search capability, so its tests drive the # storage layer directly against this job's PostgreSQL. The PutItem create @@ -326,12 +456,6 @@ jobs: --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test backup_restore --test gsi_online_build --test gsi_stale_writer --test table_snapshot - # The daemonized server logs to syslog; dump it so server-side failures - # are diagnosable from the job log. - - name: Dump server syslog on failure - if: failure() - run: sudo journalctl -t extenddb --no-pager -n 500 || true - # The runtime-detection proof: the same binary, against a PostgreSQL with no # pgvector, must refuse vector indexes over the wire. Today the job above runs # on a plain image too, so this looks like a duplicate of it; it is not, because @@ -341,6 +465,7 @@ jobs: run-rust-integration-postgres-novector: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-postgres services: postgres: image: postgres:16 @@ -365,8 +490,14 @@ jobs: # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Build release - run: cargo build --release + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-postgres + path: target/release + + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -416,6 +547,7 @@ jobs: run-rust-integration-sqlite: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 + needs: build-sqlite steps: - uses: actions/checkout@v6 @@ -428,8 +560,14 @@ jobs: # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Build release (SQLite backend) - run: cargo build --release -p extenddb --no-default-features --features sqlite + # After the cache restore, so a cached target/ cannot replace it. + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: extenddb-sqlite + path: target/release + + # Artifacts do not keep file modes. + - run: chmod +x target/release/extenddb - name: Initialize ExtendDB id: init @@ -491,6 +629,7 @@ jobs: run-integration-sqlite, run-integration-dev-mode, run-rust-integration, + run-postgres-storage-tests, run-rust-integration-postgres-novector, run-rust-integration-sqlite, ] @@ -501,7 +640,22 @@ jobs: [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ + [ "${{ needs.run-postgres-storage-tests.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then exit 1 fi + + # Fails unless the shards of each sharded job together collected every + # test of the whole suite exactly once, so a bad split cannot pass. + - uses: actions/checkout@v6 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: shards-run-integration-[0-9]* + path: shards/run-integration + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: shards-run-integration-sqlite-* + path: shards/run-integration-sqlite + - name: Check the shards ran every test + run: python3 devtools/test-shards check shards/run-integration shards/run-integration-sqlite diff --git a/devtools/pytest_shard.py b/devtools/pytest_shard.py new file mode 100644 index 000000000..c11080514 --- /dev/null +++ b/devtools/pytest_shard.py @@ -0,0 +1,35 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""pytest plugin for `devtools/run-tests --shard K/N`. + +Loaded with `-p pytest_shard` (run-tests puts devtools/ on PYTHONPATH). It +keeps the collected tests whose file is listed in $EXTENDDB_SHARD_FILES, one +path per line relative to the working directory, as `devtools/test-shards +plan` writes them, and deselects the rest. The suite's own arguments are left +as they are, so filters, ignores and xdist options apply exactly as in an +unsharded run. Under xdist every worker reads the same list and keeps the +same tests. +""" + +import os + +import pytest + + +def _shard_files(): + path = os.environ.get("EXTENDDB_SHARD_FILES") + if not path: + raise pytest.UsageError( + "pytest_shard: EXTENDDB_SHARD_FILES is not set") + with open(path, encoding="utf-8") as f: + return {line.strip() for line in f if line.strip()} + + +def pytest_collection_modifyitems(config, items): + files = _shard_files() + keep, drop = [], [] + for item in items: + (keep if os.path.relpath(str(item.path)) in files else drop).append(item) + if drop: + config.hook.pytest_deselected(items=drop) + items[:] = keep diff --git a/devtools/run-tests b/devtools/run-tests index 421a97b3a..a0da2ff5c 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -36,6 +36,7 @@ # --release use release build for Rust tests # --filter PATTERN pass to cargo test / pytest -k # --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) +# --shard K/N run only shard K of N of each pytest suite (CI) # --config PATH config file path (default: extenddb.toml in repo root) # # REQ-TEST-001, D-1, D-2, D-3, D-4, D-5 @@ -72,6 +73,8 @@ RELEASE=false FILTER="" CONFIG_PATH="" PARALLEL="" +# --shard K/N; CI sets EXTENDDB_TEST_SHARD instead. +SHARD="${EXTENDDB_TEST_SHARD:-}" BACKEND="postgres" usage() { @@ -97,6 +100,7 @@ Options: --release release build for Rust tests --filter PATTERN filter for cargo test / pytest -k --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) + --shard K/N run only shard K of N of each pytest suite --config PATH config file path EOF exit 1 @@ -128,6 +132,7 @@ while [[ $# -gt 0 ]]; do --parallel=*) PARALLEL="${1#--parallel=}"; shift ;; + --shard) SHARD="$2"; shift 2 ;; -h|--help) usage ;; *) echo "error: unknown option: $1"; usage ;; esac @@ -456,6 +461,36 @@ if $RUN_RUST_INTEGRATION; then echo "" fi +# --- Shards (--shard K/N) --- +# CI splits the long pytest suites across jobs. Before each suite runs, +# devtools/test-shards prepare collects the whole suite with the suite's own +# arguments, picks this shard's files, and records both lists under +# $SHARD_DIR for the summary job's `devtools/test-shards check`. pytest then +# loads devtools/pytest_shard.py, which keeps only those files. A shard with +# no files of a suite runs nothing for it. shard_suite SUITE CMD... leaves +# the command to run in SHARD_CMD. +SHARD_DIR="${EXTENDDB_SHARD_DIR:-discussions/shards}" +shard_suite() { + local suite="$1" rc=0 + shift + # "${@:4}" is the suite's arguments after "python3 -m pytest". + "$SCRIPT_DIR/test-shards" prepare --shard "$SHARD" --out "$SHARD_DIR" \ + --backend "$BACKEND" --config "${CONFIG_PATH:-extenddb.toml}" \ + "$suite" -- "${@:4}" || rc=$? + SHARD_CMD=("$@") + if (( rc == 0 )); then + export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}" + export EXTENDDB_SHARD_FILES="$SHARD_DIR/files-$suite.txt" + SHARD_CMD+=(-p pytest_shard) + elif (( rc == 10 )); then + echo " shard $SHARD has no $suite files; nothing to run" + SHARD_CMD=(true) + else + echo " ✗ devtools/test-shards prepare failed for the $suite suite" + SHARD_CMD=(false) + fi +} + # --- Pytest --- if $RUN_PYTEST; then OUTFILE="discussions/test-pytest-${HASH}.txt" @@ -481,6 +516,7 @@ if $RUN_PYTEST; then if [[ -n "$FILTER" ]]; then PYTEST_ARGS+=(-k "$FILTER") fi + if [[ -n "$SHARD" ]]; then shard_suite main "${PYTEST_ARGS[@]}"; PYTEST_ARGS=("${SHARD_CMD[@]}"); fi if "${PYTEST_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then echo " ✓ Pytest passed" else @@ -543,6 +579,7 @@ if $RUN_COMPREHENSIVE; then if [[ -n "$FILTER" ]]; then COMP_ARGS+=(-k "$FILTER") fi + if [[ -n "$SHARD" ]]; then shard_suite comprehensive "${COMP_ARGS[@]}"; COMP_ARGS=("${SHARD_CMD[@]}"); fi if "${COMP_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then echo " ✓ Comprehensive tests passed" else @@ -563,6 +600,7 @@ if $RUN_PYTEST \ if [[ -n "$FILTER" ]]; then CLI_ARGS+=(-k "$FILTER") fi + if [[ -n "$SHARD" ]]; then shard_suite cli "${CLI_ARGS[@]}"; CLI_ARGS=("${SHARD_CMD[@]}"); fi if "${CLI_ARGS[@]}" 2>&1 | tee "$CLI_OUTFILE"; then echo " ✓ CLI lifecycle tests passed" else diff --git a/devtools/test-shard-weights.tsv b/devtools/test-shard-weights.tsv new file mode 100644 index 000000000..1c59cc741 --- /dev/null +++ b/devtools/test-shard-weights.tsv @@ -0,0 +1,174 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Seconds each test file (pytest suites) or tests/rust module (Rust +# integration suite) took per backend, read from the job logs of the merge +# group run for c5fe7818 on 2026-10-30. devtools/test-shards uses them only +# to balance shards: a file missing here is weighted by its test count, and +# a stale value can make one shard slower but cannot drop a test, because CI +# checks that the shards together collected every test of the whole suite +# (devtools/test-shards check). +# +# suite unit postgres sqlite mongodb +main tests/test_abac.py 304 239 286 +main tests/test_attribute_value_validation.py 4 3 5 +main tests/test_auth_error_fidelity.py 1 1 0 +main tests/test_auth_integration.py 115 90 124 +main tests/test_auth_permissions.py 97 74 108 +main tests/test_authz_arn_conditions.py 32 26 30 +main tests/test_authz_resource_arn.py 92 72 101 +main tests/test_backup_restore_fidelity.py 65 70 177 +main tests/test_batch_operations.py 11 12 3 +main tests/test_cache_coherence.py 15 12 16 +main tests/test_cli_settings_redaction.py 1 0 0 +main tests/test_concurrency.py 137 91 180 +main tests/test_condition_nested_paths.py 1 1 0 +main tests/test_conditional_atomicity.py 6 4 4 +main tests/test_conditional_operator.py 4 3 1 +main tests/test_conditional_writes.py 17 17 6 +main tests/test_config_fields.py 6 6 2 +main tests/test_console_cache_coherence.py 7 5 8 +main tests/test_console_integration.py 4 3 5 +main tests/test_console_settings_redaction.py 0 0 0 +main tests/test_consumed_capacity_shape.py 10 9 2 +main tests/test_cross_account_isolation.py 33 30 27 +main tests/test_dev_mode_authorization.py 0 0 0 +main tests/test_expr_operand_type_and_size.py 5 4 1 +main tests/test_expression_attribute_map_validation.py 2 2 0 +main tests/test_expression_error_prefix.py 0 1 0 +main tests/test_expression_param_usage.py 1 1 0 +main tests/test_expression_size_limit.py 1 1 0 +main tests/test_fgac_attributes.py 11 7 10 +main tests/test_fgac_leading_keys.py 115 96 107 +main tests/test_gsi_async.py 12 7 15 +main tests/test_gsi_async_queue_sqlite.py 0 272 0 +main tests/test_gsi_attribute_definitions.py 2 1 4 +main tests/test_gsi_online_backfill.py 10 4 25 +main tests/test_idempotency_account_scope.py 15 13 11 +main tests/test_import_export.py - - 2 +main tests/test_item_operations.py 23 28 7 +main tests/test_list_map_item_size.py 12 10 4 +main tests/test_management_settings_redaction.py 1 1 2 +main tests/test_metrics.py 5 4 5 +main tests/test_misc_operations.py 7 7 2 +main tests/test_mongodb_backup_restore.py 0 0 11 +main tests/test_mongodb_commit_outcome.py 0 0 1 +main tests/test_mongodb_physical_cleanup.py 0 0 19 +main tests/test_mongodb_ttl_cleanup.py 0 0 5 +main tests/test_multipart_gsi.py 8 8 2 +main tests/test_nested_projection.py 1 1 0 +main tests/test_projection_overlapping_paths.py 1 1 0 +main tests/test_put_item_create_race.py 7 5 4 +main tests/test_query_scan.py 18 21 6 +main tests/test_rate_limiting.py 5 4 6 +main tests/test_read_paging.py 6 8 7 +main tests/test_resource_tag_migration.py 0 10 0 +main tests/test_set_evaluation.py 3 3 1 +main tests/test_stream_table_reuse.py 10 8 6 +main tests/test_streams.py 15 16 4 +main tests/test_table_operations.py 22 23 6 +main tests/test_table_throughput_mode.py 11 10 5 +main tests/test_tag_request_allow_conditions.py 110 87 102 +main tests/test_tag_request_array_conditions.py 65 46 58 +main tests/test_tag_request_conditions.py 112 79 93 +main tests/test_tag_request_repeated_keys.py 8 5 7 +main tests/test_tag_request_variable_conditions.py 27 17 25 +main tests/test_transact_absent_item_isolation.py 4 2 4 +main tests/test_transact_expression_validation.py 3 4 1 +main tests/test_transact_get_snapshot.py 6 6 5 +main tests/test_transact_write_conflict.py 2 1 2 +main tests/test_transaction_key_size_validation.py 6 7 2 +main tests/test_transaction_operations.py 1 1 0 +main tests/test_ttl.py 130 115 83 +main tests/test_unused_expression_attribute_names.py 1 0 0 +main tests/test_update_item_dollar_values.py 28 41 10 +main tests/test_update_item_number_validation.py 5 5 1 +main tests/test_update_table_billing_validation.py 38 13 29 +main tests/test_validation_precedence.py 0 0 0 +main tests/test_vector_index_query_scan.py 1 1 0 +main tests/test_wording_parity_validation.py 7 7 2 +comprehensive tests/python/test_batch.py 25 17 6 +comprehensive tests/python/test_begins_with_type_check.py 2 1 0 +comprehensive tests/python/test_composite_keys.py 15 10 3 +comprehensive tests/python/test_control_plane.py 28 18 5 +comprehensive tests/python/test_data_types.py 49 32 10 +comprehensive tests/python/test_error_handling.py 6 4 1 +comprehensive tests/python/test_expressions.py 29 19 6 +comprehensive tests/python/test_gsi.py 8 6 3 +comprehensive tests/python/test_item_edge_cases.py 60 39 12 +comprehensive tests/python/test_items.py 61 40 11 +comprehensive tests/python/test_query_scan.py 3 2 1 +comprehensive tests/python/test_rfc0003_concurrency.py 8 5 4 +comprehensive tests/python/test_scan_edge_cases.py 36 24 8 +comprehensive tests/python/test_stream_shard_ownership.py 7 5 2 +comprehensive tests/python/test_streams.py 40 31 21 +comprehensive tests/python/test_streams_transactions.py 16 13 10 +comprehensive tests/python/test_tables.py 21 14 4 +comprehensive tests/python/test_tagging.py 11 6 2 +comprehensive tests/python/test_transactions.py 22 14 4 +comprehensive tests/python/test_ttl.py 6 5 1 +cli tests/test_cli_container_readiness.py 22 - - +cli tests/test_cli_lifecycle.py 23 - - +cli tests/test_cli_migrate_concurrency.py 2 - - +cli tests/test_cli_vector_catalog_migration.py 8 - - +rust-integration authorization 2 2 1 +rust-integration backup_arn_scoping 5 4 4 +rust-integration backup_restore 10 10 11 +rust-integration batch_get_item 8 6 5 +rust-integration batch_transact_authz 31 26 26 +rust-integration batch_write_item 0 0 0 +rust-integration binary_sort_key 5 5 5 +rust-integration capacity_throttling 97 104 98 +rust-integration composite_keys 0 0 0 +rust-integration concurrency 6 5 5 +rust-integration condition_expressions 0 0 0 +rust-integration condition_expressions_more 0 0 0 +rust-integration consumed_capacity_shape 0 0 0 +rust-integration create_table_validation 0 0 0 +rust-integration data_types 16 13 13 +rust-integration delete_item 0 0 0 +rust-integration delete_item_more 0 0 0 +rust-integration empty_values 11 11 11 +rust-integration error_handling 3 3 3 +rust-integration get_item 0 0 0 +rust-integration gsi 14 14 14 +rust-integration gsi_more 10 11 11 +rust-integration gsi_on_hash_only_table 3 1 11 +rust-integration gsi_update_table_attr_defs 8 4 20 +rust-integration index_key_validation 0 0 0 +rust-integration lsi 8 7 7 +rust-integration misc_control_plane 0 0 0 +rust-integration number_validation 0 0 0 +rust-integration put_item 1 1 1 +rust-integration put_item_edge 0 0 0 +rust-integration query 0 0 0 +rust-integration query_more 3 3 3 +rust-integration raw_http 0 0 0 +rust-integration restore_active_completeness 45 18 63 +rust-integration scan 13 14 12 +rust-integration select_projection_validation 0 0 0 +rust-integration table_operations 6 5 5 +rust-integration table_operations_more 5 4 4 +rust-integration tagging 10 11 11 +rust-integration transact_get_items 0 0 0 +rust-integration transact_key_validation 0 0 0 +rust-integration transact_write_concurrency 8 5 7 +rust-integration transact_write_items 0 0 0 +rust-integration transact_write_items_more 0 0 0 +rust-integration transaction_key_size_validation 0 0 0 +rust-integration transaction_validation 0 0 0 +rust-integration ttl 23 41 51 +rust-integration unicode 10 8 8 +rust-integration update_expressions 1 0 0 +rust-integration update_expressions_more 0 0 0 +rust-integration update_item 0 0 0 +rust-integration update_item_edge 0 0 0 +rust-integration update_item_more 0 0 0 +rust-integration update_item_number_validation 0 0 0 +rust-integration update_table_attribute_definitions 10 4 4 +rust-integration update_table_billing_validation 0 0 0 +rust-integration vector_index_search 67 59 0 +rust-integration vector_index_unsupported 0 0 0 +rust-integration vector_index_update_rules 6 6 0 +rust-integration vector_index_write_capacity 6 6 0 +rust-integration wording_parity_validation 1 1 1 diff --git a/devtools/test-shards b/devtools/test-shards new file mode 100755 index 000000000..c04b25552 --- /dev/null +++ b/devtools/test-shards @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Split the pytest suites into shards by test file, and check that the +shards of one CI run together collected every test of the whole suite. + + test-shards collect OUT -- PYTEST_ARGS... + Collect PYTEST_ARGS as `pytest --collect-only -q` does and write OUT + with one line per collected test: its file relative to the working + directory, a tab, and its id (that file, then the rest of the pytest + node id). The ids do not depend on which rootdir pytest picked, so a + shard's list and the whole suite's list compare line for line. + + test-shards plan --shard K/N --out DIR [--backend NAME] [--config PATH] + [--weights TSV] SUITE=COLLECTED... + Assign every file of the collected suites to one of N shards and write + DIR/files-SUITE.txt with the files of shard K, one per line, sorted. + Every shard computes the same assignment from the same collection, so + each file lands in exactly one shard. Files are spread by the seconds in + the weights file; a file missing from it counts its tests at the + suite's mean seconds per test. Weights only balance the shards: a wrong + weight makes one shard slower, it cannot drop or repeat a file. + + test-shards prepare --shard K/N --out DIR [--backend NAME] [--config PATH] + SUITE -- PYTEST_ARGS... + What `devtools/run-tests --shard K/N` runs before each pytest suite: + collect the whole SUITE with PYTEST_ARGS into DIR/whole-SUITE.tsv, plan + shard K's files into DIR/files-SUITE.txt, collect what the shard will + run (PYTEST_ARGS plus `-p pytest_shard`) into DIR/ran-SUITE.tsv, and + write K/N to DIR/shard.txt. Exits 0 when the shard has files of SUITE, + 10 when it has none (the caller runs nothing), 1 on any failure. + + test-shards check DIR... + Each DIR holds the lists written by the legs of one sharded job, one + subdirectory per leg (shard.txt, whole-SUITE.tsv, ran-SUITE.tsv), as + actions/download-artifact leaves them. Fails unless the legs are shards + 1 to N of the same N, agree on the whole suite, and their ran lists + together hold every test of it exactly once. +""" + +import argparse +import contextlib +import os +import subprocess +import sys + +# The main suite runs under pytest-xdist; on the CI runners (4 vCPU) that is +# two workers, so a main-suite file costs a shard about half its seconds. The +# other suites run one file after another. +PARALLEL_SUITES = {"main": 2} +DEFAULT_WEIGHTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "test-shard-weights.tsv") + + +def parse_shard(text): + try: + k, n = (int(part) for part in text.split("/")) + except ValueError: + raise SystemExit(f"test-shards: --shard must be K/N, got {text!r}") + if not 1 <= k <= n: + raise SystemExit(f"test-shards: --shard {text}: need 1 <= K <= N") + return k, n + + +def read_tsv(path): + with open(path, encoding="utf-8") as f: + return [line.rstrip("\n").split("\t") for line in f if line.strip()] + + +# --- collect --------------------------------------------------------------- + +def cmd_collect(out, pytest_args): + import pytest # noqa: PLC0415 (only collect needs pytest) + + # `python3 -m pytest` puts the working directory first on sys.path; do the + # same so the suites import exactly as they do when they run. + sys.path[0] = os.getcwd() + rows = [] + + class Recorder: + def pytest_collection_finish(self, session): + for item in session.items: + path = os.path.relpath(str(item.path)) + rest = item.nodeid.partition("::")[2] + rows.append((path, f"{path}::{rest}" if rest else path)) + + # Tabs and newlines would split a row; escape them in every list alike. + def field(text): + return (text.replace("\\", "\\\\").replace("\t", "\\t") + .replace("\n", "\\n")) + + # pytest prints the collected ids; keep stdout for callers that read it. + with contextlib.redirect_stdout(sys.stderr): + rc = pytest.main(["--collect-only", "-q", *pytest_args], + plugins=[Recorder()]) + # 5 is pytest's "no tests collected", which is a valid, empty selection. + if rc not in (0, 5): + print(f"test-shards: collection failed (pytest exit {int(rc)})", + file=sys.stderr) + return int(rc) + with open(out, "w", encoding="utf-8") as f: + for path, test_id in rows: + f.write(f"{field(path)}\t{field(test_id)}\n") + print(f"test-shards: {len(rows)} tests collected into {out}", + file=sys.stderr) + return 0 + + +# --- plan ------------------------------------------------------------------ + +def load_weights(path): + """{(suite, file): {backend: seconds}} from the weights file.""" + columns, weights = [], {} + with open(path, encoding="utf-8") as f: + for line in f: + line = line.rstrip("\n") + if line.startswith("# suite\t"): + columns = line[2:].split("\t")[2:] + if not line or line.startswith("#"): + continue + suite, unit, *secs = line.split("\t") + weights[(suite, unit)] = { + b: float(s) for b, s in zip(columns, secs) if s != "-"} + return columns, weights + + +def config_backend(path): + """The [storage] backend a config file names, or None.""" + try: + import tomllib # noqa: PLC0415 (Python 3.11+) + with open(path, "rb") as f: + return tomllib.load(f).get("storage", {}).get("backend") + except (ImportError, OSError, ValueError): + return None + + +def assign(units, n): + """Longest weight first, each to the least loaded shard; ties go to the + lower shard number, so every caller gets the same answer.""" + loads = [0.0] * n + shards = [[] for _ in range(n)] + for unit in sorted(units, key=lambda u: (-u[2], u[0], u[1])): + i = loads.index(min(loads)) + shards[i].append(unit) + loads[i] += unit[2] + return shards, loads + + +def cmd_plan(args): + k, n = parse_shard(args.shard) + backend = (config_backend(args.config) if args.config else None) \ + or args.backend + columns, weights = load_weights(args.weights) + column = backend if backend in columns else (columns[0] if columns else None) + + suites = [] + for spec in args.collected: + suite, sep, path = spec.partition("=") + if not sep: + raise SystemExit(f"test-shards: expected SUITE=FILE, got {spec!r}") + counts = {} + for row in read_tsv(path): + counts[row[0]] = counts.get(row[0], 0) + 1 + suites.append((suite, counts)) + + units = [] + for suite, counts in suites: + known = {f: weights[(suite, f)][column] for f in counts + if column in weights.get((suite, f), {})} + known_tests = sum(counts[f] for f in known) + rate = sum(known.values()) / known_tests if known_tests else 1.0 + divisor = PARALLEL_SUITES.get(suite, 1) + for f, count in counts.items(): + secs = known.get(f, count * rate) + units.append((suite, f, secs / divisor)) + + shards, loads = assign(units, n) + os.makedirs(args.out, exist_ok=True) + for suite, _ in suites: + files = sorted(u[1] for u in shards[k - 1] if u[0] == suite) + with open(os.path.join(args.out, f"files-{suite}.txt"), "w", + encoding="utf-8") as f: + f.writelines(f"{name}\n" for name in files) + print(f"test-shards: {len(units)} files, weights column {column}") + for i, (shard, load) in enumerate(zip(shards, loads), start=1): + per_suite = ", ".join( + f"{suite} {sum(1 for u in shard if u[0] == suite)}" + for suite, _ in suites) + mark = " <- this shard" if i == k else "" + print(f" shard {i}/{n}: {load / 60:5.1f} min modelled ({per_suite})" + f"{mark}") + return 0 + + +# --- check ----------------------------------------------------------------- + +def read_ids(path): + return [row[1] if len(row) > 1 else row[0] for row in read_tsv(path)] + + +def check_group(group): + """Problems found in one sharded job's lists; prints its summary.""" + legs = sorted(root for root, _, files in os.walk(group) + if "shard.txt" in files) + if not legs: + return [f"{group}: no shard lists found (did every leg upload?)"] + problems, shards = [], {} + for leg in legs: + with open(os.path.join(leg, "shard.txt"), encoding="utf-8") as f: + k, n = parse_shard(f.read().strip()) + if k in shards: + problems.append(f"{group}: shard {k} appears twice " + f"({shards[k][0]} and {leg})") + shards[k] = (leg, n) + totals = {n for _, n in shards.values()} + if len(totals) != 1: + return problems + [f"{group}: legs disagree on N: {sorted(totals)}"] + n = totals.pop() + missing = sorted(set(range(1, n + 1)) - set(shards)) + if missing: + problems.append(f"{group}: no lists from shard(s) {missing} of {n}") + + suites = sorted({name[len("whole-"):-len(".tsv")] + for leg, _ in shards.values() for name in os.listdir(leg) + if name.startswith("whole-") and name.endswith(".tsv")}) + if not suites: + problems.append(f"{group}: no whole-suite lists found") + for suite in suites: + whole, ran = None, {} + for k, (leg, _) in sorted(shards.items()): + whole_path = os.path.join(leg, f"whole-{suite}.tsv") + ran_path = os.path.join(leg, f"ran-{suite}.tsv") + if not os.path.exists(whole_path) or not os.path.exists(ran_path): + problems.append(f"{group}: shard {k} has no {suite} lists") + continue + ids = sorted(read_ids(whole_path)) + if whole is None: + whole = ids + elif ids != whole: + problems.append(f"{group}: shard {k} collected a different " + f"whole {suite} suite ({len(ids)} tests, " + f"others {len(whole)})") + ran[k] = read_ids(ran_path) + if whole is None: + continue + seen = {} + for k, ids in ran.items(): + for test_id in ids: + seen.setdefault(test_id, []).append(k) + repeated = sorted(t for t, ks in seen.items() if len(ks) > 1) + absent = sorted(set(whole) - set(seen)) + extra = sorted(set(seen) - set(whole)) + counts = " + ".join(str(len(ran.get(k, []))) for k in range(1, n + 1)) + total = sum(len(ids) for ids in ran.values()) + ok = bool(whole) and not (repeated or absent or extra) + print(f"{group} {suite}: whole {len(whole)}, shards {counts} = " + f"{total} {'OK' if ok else 'MISMATCH'}") + if not whole: + problems.append(f"{group}: the whole {suite} suite is empty") + for label, ids in (("not run by any shard", absent), + ("run by more than one shard", repeated), + ("not in the whole suite", extra)): + if ids: + problems.append(f"{group} {suite}: {len(ids)} test(s) {label}" + f", first: {ids[0]}") + return problems + + +def cmd_check(groups): + problems = [p for group in groups for p in check_group(group)] + for problem in problems: + print(f"error: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +# --- prepare --------------------------------------------------------------- + +EMPTY_SHARD = 10 + + +def cmd_prepare(argv): + if "--" not in argv: + raise SystemExit("usage: test-shards prepare --shard K/N --out DIR " + "[--backend NAME] [--config PATH] SUITE -- " + "PYTEST_ARGS...") + split = argv.index("--") + parser = argparse.ArgumentParser(prog="test-shards prepare") + parser.add_argument("--shard", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--backend", default="postgres") + parser.add_argument("--config") + parser.add_argument("--weights", default=DEFAULT_WEIGHTS) + parser.add_argument("suite") + args = parser.parse_args(argv[:split]) + pytest_args = argv[split + 1:] + k, n = parse_shard(args.shard) + os.makedirs(args.out, exist_ok=True) + whole, ran, files = (os.path.join(args.out, f"{kind}-{args.suite}{ext}") + for kind, ext in (("whole", ".tsv"), ("ran", ".tsv"), + ("files", ".txt"))) + + # Each collection runs in its own interpreter: pytest does not support + # two sessions importing the same test modules in one process. Its + # output is the whole collection tree, so it is shown only on failure. + def collect(out, extra, env=None): + proc = subprocess.run([sys.executable, os.path.abspath(__file__), + "collect", out, "--", *extra, *pytest_args], + env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + if proc.returncode: + sys.stderr.write(proc.stdout) + else: + sys.stderr.writelines(line for line in proc.stdout.splitlines(True) + if line.startswith("test-shards:")) + return proc.returncode + + if collect(whole, []): + return 1 + args.collected = [f"{args.suite}={whole}"] + if cmd_plan(args): + return 1 + devtools = os.path.dirname(os.path.abspath(__file__)) + path = os.environ.get("PYTHONPATH") + env = dict(os.environ, EXTENDDB_SHARD_FILES=files, + PYTHONPATH=f"{devtools}{os.pathsep}{path}" if path else devtools) + if collect(ran, ["-p", "pytest_shard"], env): + return 1 + with open(os.path.join(args.out, "shard.txt"), "w", encoding="utf-8") as f: + f.write(f"{k}/{n}\n") + with open(files, encoding="utf-8") as f: + return 0 if f.read().strip() else EMPTY_SHARD + + +def main(argv): + if len(argv) > 1 and argv[1] == "collect": + if len(argv) < 4 or argv[3] != "--": + raise SystemExit("usage: test-shards collect OUT -- PYTEST_ARGS...") + return cmd_collect(argv[2], argv[4:]) + if len(argv) > 1 and argv[1] == "prepare": + return cmd_prepare(argv[2:]) + parser = argparse.ArgumentParser(prog="test-shards") + sub = parser.add_subparsers(dest="command", required=True) + plan = sub.add_parser("plan") + plan.add_argument("--shard", required=True) + plan.add_argument("--out", required=True) + plan.add_argument("--backend", default="postgres") + plan.add_argument("--config") + plan.add_argument("--weights", default=DEFAULT_WEIGHTS) + plan.add_argument("collected", nargs="+", metavar="SUITE=COLLECTED") + check = sub.add_parser("check") + check.add_argument("groups", nargs="+", metavar="DIR") + args = parser.parse_args(argv[1:]) + if args.command == "plan": + return cmd_plan(args) + return cmd_check(args.groups) + + +if __name__ == "__main__": + sys.exit(main(sys.argv))