From 4737451765d193472d68d6969ed6284910f13e78 Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 01/10] devtools: add per-file test durations for shard
 balancing

devtools/test-shard-weights.tsv holds the seconds each pytest file and
each tests/rust module took on each backend in the merge group run for
c5fe7818 on 30 Oct, read from its job logs. devtools/test-shards uses
them to balance the shards.
---
 devtools/test-shard-weights.tsv | 173 ++++++++++++++++++++++++++++++++
 1 file changed, 173 insertions(+)
 create mode 100644 devtools/test-shard-weights.tsv

diff --git a/devtools/test-shard-weights.tsv b/devtools/test-shard-weights.tsv
new file mode 100644
index 000000000..d2f2d9e80
--- /dev/null
+++ b/devtools/test-shard-weights.tsv
@@ -0,0 +1,173 @@
+# Copyright 2026 ExtendDB contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Seconds each test file (pytest suites) or tests/rust module (Rust
+# integration suite) took per backend, read from the job logs of the merge
+# group run for c5fe7818 on 2026-10-30. devtools/test-shards uses them only
+# to balance shards: a unit missing here is weighted by its test count, and
+# a stale value can make one shard slower but cannot drop a test, because the
+# CI summary jobs compare every shard's collected tests with the whole suite.
+#
+# suite	unit	postgres	sqlite	mongodb
+main	tests/test_abac.py	304	239	286
+main	tests/test_attribute_value_validation.py	4	3	5
+main	tests/test_auth_error_fidelity.py	1	1	0
+main	tests/test_auth_integration.py	115	90	124
+main	tests/test_auth_permissions.py	97	74	108
+main	tests/test_authz_arn_conditions.py	32	26	30
+main	tests/test_authz_resource_arn.py	92	72	101
+main	tests/test_backup_restore_fidelity.py	65	70	177
+main	tests/test_batch_operations.py	11	12	3
+main	tests/test_cache_coherence.py	15	12	16
+main	tests/test_cli_settings_redaction.py	1	0	0
+main	tests/test_concurrency.py	137	91	180
+main	tests/test_condition_nested_paths.py	1	1	0
+main	tests/test_conditional_atomicity.py	6	4	4
+main	tests/test_conditional_operator.py	4	3	1
+main	tests/test_conditional_writes.py	17	17	6
+main	tests/test_config_fields.py	6	6	2
+main	tests/test_console_cache_coherence.py	7	5	8
+main	tests/test_console_integration.py	4	3	5
+main	tests/test_console_settings_redaction.py	0	0	0
+main	tests/test_consumed_capacity_shape.py	10	9	2
+main	tests/test_cross_account_isolation.py	33	30	27
+main	tests/test_dev_mode_authorization.py	0	0	0
+main	tests/test_expr_operand_type_and_size.py	5	4	1
+main	tests/test_expression_attribute_map_validation.py	2	2	0
+main	tests/test_expression_error_prefix.py	0	1	0
+main	tests/test_expression_param_usage.py	1	1	0
+main	tests/test_expression_size_limit.py	1	1	0
+main	tests/test_fgac_attributes.py	11	7	10
+main	tests/test_fgac_leading_keys.py	115	96	107
+main	tests/test_gsi_async.py	12	7	15
+main	tests/test_gsi_async_queue_sqlite.py	0	272	0
+main	tests/test_gsi_attribute_definitions.py	2	1	4
+main	tests/test_gsi_online_backfill.py	10	4	25
+main	tests/test_idempotency_account_scope.py	15	13	11
+main	tests/test_import_export.py	-	-	2
+main	tests/test_item_operations.py	23	28	7
+main	tests/test_list_map_item_size.py	12	10	4
+main	tests/test_management_settings_redaction.py	1	1	2
+main	tests/test_metrics.py	5	4	5
+main	tests/test_misc_operations.py	7	7	2
+main	tests/test_mongodb_backup_restore.py	0	0	11
+main	tests/test_mongodb_commit_outcome.py	0	0	1
+main	tests/test_mongodb_physical_cleanup.py	0	0	19
+main	tests/test_mongodb_ttl_cleanup.py	0	0	5
+main	tests/test_multipart_gsi.py	8	8	2
+main	tests/test_nested_projection.py	1	1	0
+main	tests/test_projection_overlapping_paths.py	1	1	0
+main	tests/test_put_item_create_race.py	7	5	4
+main	tests/test_query_scan.py	18	21	6
+main	tests/test_rate_limiting.py	5	4	6
+main	tests/test_read_paging.py	6	8	7
+main	tests/test_resource_tag_migration.py	0	10	0
+main	tests/test_set_evaluation.py	3	3	1
+main	tests/test_stream_table_reuse.py	10	8	6
+main	tests/test_streams.py	15	16	4
+main	tests/test_table_operations.py	22	23	6
+main	tests/test_table_throughput_mode.py	11	10	5
+main	tests/test_tag_request_allow_conditions.py	110	87	102
+main	tests/test_tag_request_array_conditions.py	65	46	58
+main	tests/test_tag_request_conditions.py	112	79	93
+main	tests/test_tag_request_repeated_keys.py	8	5	7
+main	tests/test_tag_request_variable_conditions.py	27	17	25
+main	tests/test_transact_absent_item_isolation.py	4	2	4
+main	tests/test_transact_expression_validation.py	3	4	1
+main	tests/test_transact_get_snapshot.py	6	6	5
+main	tests/test_transact_write_conflict.py	2	1	2
+main	tests/test_transaction_key_size_validation.py	6	7	2
+main	tests/test_transaction_operations.py	1	1	0
+main	tests/test_ttl.py	130	115	83
+main	tests/test_unused_expression_attribute_names.py	1	0	0
+main	tests/test_update_item_dollar_values.py	28	41	10
+main	tests/test_update_item_number_validation.py	5	5	1
+main	tests/test_update_table_billing_validation.py	38	13	29
+main	tests/test_validation_precedence.py	0	0	0
+main	tests/test_vector_index_query_scan.py	1	1	0
+main	tests/test_wording_parity_validation.py	7	7	2
+comprehensive	tests/python/test_batch.py	25	17	6
+comprehensive	tests/python/test_begins_with_type_check.py	2	1	0
+comprehensive	tests/python/test_composite_keys.py	15	10	3
+comprehensive	tests/python/test_control_plane.py	28	18	5
+comprehensive	tests/python/test_data_types.py	49	32	10
+comprehensive	tests/python/test_error_handling.py	6	4	1
+comprehensive	tests/python/test_expressions.py	29	19	6
+comprehensive	tests/python/test_gsi.py	8	6	3
+comprehensive	tests/python/test_item_edge_cases.py	60	39	12
+comprehensive	tests/python/test_items.py	61	40	11
+comprehensive	tests/python/test_query_scan.py	3	2	1
+comprehensive	tests/python/test_rfc0003_concurrency.py	8	5	4
+comprehensive	tests/python/test_scan_edge_cases.py	36	24	8
+comprehensive	tests/python/test_stream_shard_ownership.py	7	5	2
+comprehensive	tests/python/test_streams.py	40	31	21
+comprehensive	tests/python/test_streams_transactions.py	16	13	10
+comprehensive	tests/python/test_tables.py	21	14	4
+comprehensive	tests/python/test_tagging.py	11	6	2
+comprehensive	tests/python/test_transactions.py	22	14	4
+comprehensive	tests/python/test_ttl.py	6	5	1
+cli	tests/test_cli_container_readiness.py	22	-	-
+cli	tests/test_cli_lifecycle.py	23	-	-
+cli	tests/test_cli_migrate_concurrency.py	2	-	-
+cli	tests/test_cli_vector_catalog_migration.py	8	-	-
+rust-integration	authorization	2	2	1
+rust-integration	backup_arn_scoping	5	4	4
+rust-integration	backup_restore	10	10	11
+rust-integration	batch_get_item	8	6	5
+rust-integration	batch_transact_authz	31	26	26
+rust-integration	batch_write_item	0	0	0
+rust-integration	binary_sort_key	5	5	5
+rust-integration	capacity_throttling	97	104	98
+rust-integration	composite_keys	0	0	0
+rust-integration	concurrency	6	5	5
+rust-integration	condition_expressions	0	0	0
+rust-integration	condition_expressions_more	0	0	0
+rust-integration	consumed_capacity_shape	0	0	0
+rust-integration	create_table_validation	0	0	0
+rust-integration	data_types	16	13	13
+rust-integration	delete_item	0	0	0
+rust-integration	delete_item_more	0	0	0
+rust-integration	empty_values	11	11	11
+rust-integration	error_handling	3	3	3
+rust-integration	get_item	0	0	0
+rust-integration	gsi	14	14	14
+rust-integration	gsi_more	10	11	11
+rust-integration	gsi_on_hash_only_table	3	1	11
+rust-integration	gsi_update_table_attr_defs	8	4	20
+rust-integration	index_key_validation	0	0	0
+rust-integration	lsi	8	7	7
+rust-integration	misc_control_plane	0	0	0
+rust-integration	number_validation	0	0	0
+rust-integration	put_item	1	1	1
+rust-integration	put_item_edge	0	0	0
+rust-integration	query	0	0	0
+rust-integration	query_more	3	3	3
+rust-integration	raw_http	0	0	0
+rust-integration	restore_active_completeness	45	18	63
+rust-integration	scan	13	14	12
+rust-integration	select_projection_validation	0	0	0
+rust-integration	table_operations	6	5	5
+rust-integration	table_operations_more	5	4	4
+rust-integration	tagging	10	11	11
+rust-integration	transact_get_items	0	0	0
+rust-integration	transact_key_validation	0	0	0
+rust-integration	transact_write_concurrency	8	5	7
+rust-integration	transact_write_items	0	0	0
+rust-integration	transact_write_items_more	0	0	0
+rust-integration	transaction_key_size_validation	0	0	0
+rust-integration	transaction_validation	0	0	0
+rust-integration	ttl	23	41	51
+rust-integration	unicode	10	8	8
+rust-integration	update_expressions	1	0	0
+rust-integration	update_expressions_more	0	0	0
+rust-integration	update_item	0	0	0
+rust-integration	update_item_edge	0	0	0
+rust-integration	update_item_more	0	0	0
+rust-integration	update_item_number_validation	0	0	0
+rust-integration	update_table_attribute_definitions	10	4	4
+rust-integration	update_table_billing_validation	0	0	0
+rust-integration	vector_index_search	67	59	0
+rust-integration	vector_index_unsupported	0	0	0
+rust-integration	vector_index_update_rules	6	6	0
+rust-integration	vector_index_write_capacity	6	6	0
+rust-integration	wording_parity_validation	1	1	1

From 3648e28943f4dfa9b60c39adcc0074f1430c7bcb Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 02/10] ci: build each release binary once per feature set

Ten integration jobs each ran their own cargo build --release. Each
distinct feature set now has one build job that uploads
target/release/extenddb as an artifact, and the jobs that test it
download that binary instead of compiling it:

- build-postgres (cargo build --release): run-integration,
  run-rust-integration, run-rust-integration-postgres-novector
- build-sqlite (--no-default-features --features sqlite):
  run-integration-sqlite, run-rust-integration-sqlite
- build-mongodb (--features mongodb-test-hooks): mongodb-pytest,
  mongodb-rust
- build-cassandra (--features cassandra): cassandra-pytest,
  cassandra-rust

The cargo flags and features are unchanged. run-integration-dev-mode
keeps its own build because it is the only job using that feature set.

Jobs that only run pytest no longer install the Rust toolchain or
restore the cargo cache. Jobs that still run cargo test download the
binary after the cache restore, so a cached target/ cannot replace it.
Artifacts do not keep file modes, so each download is followed by
chmod +x.
---
 .github/workflows/integration-cassandra.yml |  44 ++++++--
 .github/workflows/integration-mongodb.yml   |  44 ++++++--
 .github/workflows/integration.yml           | 108 ++++++++++++++++----
 3 files changed, 154 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml
index 1f578bfee..8c037967e 100644
--- a/.github/workflows/integration-cassandra.yml
+++ b/.github/workflows/integration-cassandra.yml
@@ -99,9 +99,29 @@ jobs:
         env:
           EXTENDDB_TEST_CASSANDRA: required
 
+  # One release build shared by the pytest and rust jobs; see integration.yml.
+  build-cassandra:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v6
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - name: Build release (cassandra backend)
+        run: cargo build --release --no-default-features --features cassandra
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: extenddb-cassandra
+          path: target/release/extenddb
+          if-no-files-found: error
+          retention-days: 1
+
   cassandra-pytest:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 45
+    needs: build-cassandra
     services:
       cassandra:
         image: cassandra:4.1
@@ -119,12 +139,6 @@ jobs:
           --health-start-period 60s
     steps:
       - uses: actions/checkout@v6
-      - uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-on-failure: true
-          # Only pushes to main save, so other runs restore main's cache.
-          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Configure Cassandra authentication
         run: |
           docker exec ${{ job.services.cassandra.id }} sed -i \
@@ -147,14 +161,19 @@ jobs:
           python-version: "3.11"
       - name: Install Python dependencies
         run: pip install -r requirements.txt
-      - name: Build release (cassandra backend)
-        run: cargo build --release --no-default-features --features cassandra
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-cassandra
+          path: target/release
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
       - name: Run Cassandra pytest suite
         run: devtools/run-cassandra-tests -- --pytest --comprehensive --parallel
 
   cassandra-rust:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 45
+    needs: build-cassandra
     services:
       cassandra:
         image: cassandra:4.1
@@ -200,8 +219,13 @@ jobs:
           python-version: "3.11"
       - name: Install Python dependencies
         run: pip install -r requirements.txt
-      - name: Build release (cassandra backend)
-        run: cargo build --release --no-default-features --features cassandra
+      # After the cache restore, so a cached target/ cannot replace it.
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-cassandra
+          path: target/release
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
       - name: Run Cassandra rust integration suite
         run: devtools/run-cassandra-tests -- --rust-integration
 
diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml
index 01e28552f..73e8f9c69 100644
--- a/.github/workflows/integration-mongodb.yml
+++ b/.github/workflows/integration-mongodb.yml
@@ -29,10 +29,8 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  mongodb-pytest:
-    # Dedicated ExtendDB runner group (Ubuntu 24.04, 4 vCPU / 16 GB, capped at
-    # 20 concurrent). ubuntu-latest is an Amazon-wide shared pool where our jobs
-    # waited 10-90 minutes to start (2026-09-29/30); see integration.yml.
+  # One release build shared by the pytest and rust jobs; see integration.yml.
+  build-mongodb:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
     steps:
@@ -41,15 +39,35 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
-          # Only pushes to main save, so other runs restore main's cache.
-          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Build release (mongodb backend)
+        run: cargo build --release --no-default-features --features mongodb-test-hooks
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: extenddb-mongodb
+          path: target/release/extenddb
+          if-no-files-found: error
+          retention-days: 1
+
+  mongodb-pytest:
+    # Dedicated ExtendDB runner group (Ubuntu 24.04, 4 vCPU / 16 GB, capped at
+    # 20 concurrent). ubuntu-latest is an Amazon-wide shared pool where our jobs
+    # waited 10-90 minutes to start (2026-09-29/30); see integration.yml.
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    needs: build-mongodb
+    steps:
+      - uses: actions/checkout@v6
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
       - name: Install Python dependencies
         run: pip install -r requirements.txt
-      - name: Build release (mongodb backend)
-        run: cargo build --release --no-default-features --features mongodb-test-hooks
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-mongodb
+          path: target/release
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
       - name: Run MongoDB pytest suite
         run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel
 
@@ -107,6 +125,7 @@ jobs:
   mongodb-rust:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
+    needs: build-mongodb
     steps:
       - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
@@ -120,8 +139,13 @@ jobs:
           python-version: "3.11"
       - name: Install Python dependencies
         run: pip install -r requirements.txt
-      - name: Build release (mongodb backend)
-        run: cargo build --release --no-default-features --features mongodb-test-hooks
+      # After the cache restore, so a cached target/ cannot replace it.
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-mongodb
+          path: target/release
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
       - name: Run MongoDB rust integration suite
         run: devtools/run-mongodb-tests -- --rust --rust-integration
 
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index b1de36c0b..0b81bc2b7 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -20,6 +20,53 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  # One release build per feature set. The jobs below download the binary
+  # instead of each compiling the same one, so the cargo flags and features of
+  # a test job live in the build job it needs.
+  build-postgres:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Build release
+        run: cargo build --release
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: extenddb-postgres
+          path: target/release/extenddb
+          if-no-files-found: error
+          retention-days: 1
+
+  build-sqlite:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Build release (SQLite backend)
+        run: cargo build --release -p extenddb --no-default-features --features sqlite
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: extenddb-sqlite
+          path: target/release/extenddb
+          if-no-files-found: error
+          retention-days: 1
+
   run-integration:
     # Dedicated ExtendDB runner group (4 vCPU / 16 GB, capped at 20 concurrent),
     # pinned to Ubuntu 24.04. ubuntu-latest is an Amazon-wide pool of 1,000
@@ -29,6 +76,7 @@ jobs:
     # 2026 and this compiles Rust; move to 26 deliberately.
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
+    needs: build-postgres
     services:
       postgres:
         # pgvector's own image, which is postgres:16 plus the extension. The
@@ -49,16 +97,13 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - uses: dtolnay/rust-toolchain@stable
-
-      - uses: Swatinem/rust-cache@v2
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
         with:
-          cache-on-failure: true
-          # Only pushes to main save, so other runs restore main's cache.
-          save-if: ${{ github.ref == 'refs/heads/main' }}
+          name: extenddb-postgres
+          path: target/release
 
-      - name: Build release
-        run: cargo build --release
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
 
       - name: Initialize ExtendDB
         id: init
@@ -110,19 +155,17 @@ jobs:
   run-integration-sqlite:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
+    needs: build-sqlite
     steps:
       - uses: actions/checkout@v6
 
-      - uses: dtolnay/rust-toolchain@stable
-
-      - uses: Swatinem/rust-cache@v2
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
         with:
-          cache-on-failure: true
-          # Only pushes to main save, so other runs restore main's cache.
-          save-if: ${{ github.ref == 'refs/heads/main' }}
+          name: extenddb-sqlite
+          path: target/release
 
-      - name: Build release (SQLite backend)
-        run: cargo build --release -p extenddb --no-default-features --features sqlite
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
 
       - name: Initialize ExtendDB
         id: init
@@ -225,6 +268,7 @@ jobs:
   run-rust-integration:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
+    needs: build-postgres
     services:
       postgres:
         image: pgvector/pgvector:pg16
@@ -249,8 +293,14 @@ jobs:
           # Only pushes to main save, so other runs restore main's cache.
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Build release
-        run: cargo build --release
+      # After the cache restore, so a cached target/ cannot replace it.
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-postgres
+          path: target/release
+
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
 
       - name: Initialize ExtendDB
         id: init
@@ -341,6 +391,7 @@ jobs:
   run-rust-integration-postgres-novector:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
+    needs: build-postgres
     services:
       postgres:
         image: postgres:16
@@ -365,8 +416,14 @@ jobs:
           # Only pushes to main save, so other runs restore main's cache.
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Build release
-        run: cargo build --release
+      # After the cache restore, so a cached target/ cannot replace it.
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-postgres
+          path: target/release
+
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
 
       - name: Initialize ExtendDB
         id: init
@@ -416,6 +473,7 @@ jobs:
   run-rust-integration-sqlite:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
+    needs: build-sqlite
 
     steps:
       - uses: actions/checkout@v6
@@ -428,8 +486,14 @@ jobs:
           # Only pushes to main save, so other runs restore main's cache.
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Build release (SQLite backend)
-        run: cargo build --release -p extenddb --no-default-features --features sqlite
+      # After the cache restore, so a cached target/ cannot replace it.
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: extenddb-sqlite
+          path: target/release
+
+      # Artifacts do not keep file modes.
+      - run: chmod +x target/release/extenddb
 
       - name: Initialize ExtendDB
         id: init

From a4fb2353dc75d8bac9911565e82235f3cf8af25e Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 03/10] devtools: add test-shards to split pytest suites by
 file

devtools/test-shards has three commands:

- collect runs pytest --collect-only -q on the given arguments and
  writes each collected test's file and id, with paths relative to the
  working directory so lists from different rootdirs compare.
- plan assigns every collected file to one of N shards, longest first
  to the least loaded shard, using the seconds in
  devtools/test-shard-weights.tsv. Every shard computes the same
  assignment from the same collection, so each file is in exactly one
  shard.
- check reads the lists all legs of a sharded job wrote and fails
  unless they are shards 1 to N of one N, agree on the whole suite, and
  together ran every test of it exactly once.

The weights only balance the shards; a missing or stale weight cannot
drop a test because check compares the shards with the whole suite.
---
 devtools/test-shard-weights.tsv |   7 +-
 devtools/test-shards            | 284 ++++++++++++++++++++++++++++++++
 2 files changed, 288 insertions(+), 3 deletions(-)
 create mode 100755 devtools/test-shards

diff --git a/devtools/test-shard-weights.tsv b/devtools/test-shard-weights.tsv
index d2f2d9e80..1c59cc741 100644
--- a/devtools/test-shard-weights.tsv
+++ b/devtools/test-shard-weights.tsv
@@ -4,9 +4,10 @@
 # Seconds each test file (pytest suites) or tests/rust module (Rust
 # integration suite) took per backend, read from the job logs of the merge
 # group run for c5fe7818 on 2026-10-30. devtools/test-shards uses them only
-# to balance shards: a unit missing here is weighted by its test count, and
-# a stale value can make one shard slower but cannot drop a test, because the
-# CI summary jobs compare every shard's collected tests with the whole suite.
+# to balance shards: a file missing here is weighted by its test count, and
+# a stale value can make one shard slower but cannot drop a test, because CI
+# checks that the shards together collected every test of the whole suite
+# (devtools/test-shards check).
 #
 # suite	unit	postgres	sqlite	mongodb
 main	tests/test_abac.py	304	239	286
diff --git a/devtools/test-shards b/devtools/test-shards
new file mode 100755
index 000000000..1e81f8c2f
--- /dev/null
+++ b/devtools/test-shards
@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+# Copyright 2026 ExtendDB contributors
+# SPDX-License-Identifier: Apache-2.0
+"""Split the pytest suites into shards by test file, and check that the
+shards of one CI run together collected every test of the whole suite.
+
+  test-shards collect OUT -- PYTEST_ARGS...
+      Collect PYTEST_ARGS as `pytest --collect-only -q` does and write OUT
+      with one line per collected test: its file relative to the working
+      directory, a tab, and its id (that file, then the rest of the pytest
+      node id). The ids do not depend on which rootdir pytest picked, so a
+      shard's list and the whole suite's list compare line for line.
+
+  test-shards plan --shard K/N --out DIR [--backend NAME] [--config PATH]
+                   [--weights TSV] SUITE=COLLECTED...
+      Assign every file of the collected suites to one of N shards and write
+      DIR/files-SUITE.txt with the files of shard K, one per line, sorted.
+      Every shard computes the same assignment from the same collection, so
+      each file lands in exactly one shard. Files are spread by the seconds in
+      the weights file; a file missing from it counts its tests at the
+      suite's mean seconds per test. Weights only balance the shards: a wrong
+      weight makes one shard slower, it cannot drop or repeat a file.
+
+  test-shards check DIR...
+      Each DIR holds the lists written by the legs of one sharded job, one
+      subdirectory per leg (shard.txt, whole-SUITE.tsv, ran-SUITE.tsv), as
+      actions/download-artifact leaves them. Fails unless the legs are shards
+      1 to N of the same N, agree on the whole suite, and their ran lists
+      together hold every test of it exactly once.
+"""
+
+import argparse
+import contextlib
+import os
+import sys
+
+# The main suite runs under pytest-xdist; on the CI runners (4 vCPU) that is
+# two workers, so a main-suite file costs a shard about half its seconds. The
+# other suites run one file after another.
+PARALLEL_SUITES = {"main": 2}
+DEFAULT_WEIGHTS = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                               "test-shard-weights.tsv")
+
+
+def parse_shard(text):
+    try:
+        k, n = (int(part) for part in text.split("/"))
+    except ValueError:
+        raise SystemExit(f"test-shards: --shard must be K/N, got {text!r}")
+    if not 1 <= k <= n:
+        raise SystemExit(f"test-shards: --shard {text}: need 1 <= K <= N")
+    return k, n
+
+
+def read_tsv(path):
+    with open(path, encoding="utf-8") as f:
+        return [line.rstrip("\n").split("\t") for line in f if line.strip()]
+
+
+# --- collect ---------------------------------------------------------------
+
+def cmd_collect(out, pytest_args):
+    import pytest  # noqa: PLC0415 (only collect needs pytest)
+
+    # `python3 -m pytest` puts the working directory first on sys.path; do the
+    # same so the suites import exactly as they do when they run.
+    sys.path[0] = os.getcwd()
+    rows = []
+
+    class Recorder:
+        def pytest_collection_finish(self, session):
+            for item in session.items:
+                path = os.path.relpath(str(item.path))
+                rest = item.nodeid.partition("::")[2]
+                rows.append((path, f"{path}::{rest}" if rest else path))
+
+    # pytest prints the collected ids; keep stdout for callers that read it.
+    with contextlib.redirect_stdout(sys.stderr):
+        rc = pytest.main(["--collect-only", "-q", *pytest_args],
+                         plugins=[Recorder()])
+    # 5 is pytest's "no tests collected", which is a valid, empty selection.
+    if rc not in (0, 5):
+        print(f"test-shards: collection failed (pytest exit {int(rc)})",
+              file=sys.stderr)
+        return int(rc)
+    with open(out, "w", encoding="utf-8") as f:
+        for path, test_id in rows:
+            f.write(f"{path}\t{test_id}\n")
+    print(f"test-shards: {len(rows)} tests collected into {out}",
+          file=sys.stderr)
+    return 0
+
+
+# --- plan ------------------------------------------------------------------
+
+def load_weights(path):
+    """{(suite, file): {backend: seconds}} from the weights file."""
+    columns, weights = [], {}
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.rstrip("\n")
+            if line.startswith("# suite\t"):
+                columns = line[2:].split("\t")[2:]
+            if not line or line.startswith("#"):
+                continue
+            suite, unit, *secs = line.split("\t")
+            weights[(suite, unit)] = {
+                b: float(s) for b, s in zip(columns, secs) if s != "-"}
+    return columns, weights
+
+
+def config_backend(path):
+    """The [storage] backend a config file names, or None."""
+    try:
+        import tomllib  # noqa: PLC0415 (Python 3.11+)
+        with open(path, "rb") as f:
+            return tomllib.load(f).get("storage", {}).get("backend")
+    except (ImportError, OSError, ValueError):
+        return None
+
+
+def assign(units, n):
+    """Longest weight first, each to the least loaded shard; ties go to the
+    lower shard number, so every caller gets the same answer."""
+    loads = [0.0] * n
+    shards = [[] for _ in range(n)]
+    for unit in sorted(units, key=lambda u: (-u[2], u[0], u[1])):
+        i = loads.index(min(loads))
+        shards[i].append(unit)
+        loads[i] += unit[2]
+    return shards, loads
+
+
+def cmd_plan(args):
+    k, n = parse_shard(args.shard)
+    backend = (config_backend(args.config) if args.config else None) \
+        or args.backend
+    columns, weights = load_weights(args.weights)
+    column = backend if backend in columns else (columns[0] if columns else None)
+
+    suites = []
+    for spec in args.collected:
+        suite, sep, path = spec.partition("=")
+        if not sep:
+            raise SystemExit(f"test-shards: expected SUITE=FILE, got {spec!r}")
+        counts = {}
+        for row in read_tsv(path):
+            counts[row[0]] = counts.get(row[0], 0) + 1
+        suites.append((suite, counts))
+
+    units = []
+    for suite, counts in suites:
+        known = {f: weights[(suite, f)][column] for f in counts
+                 if column in weights.get((suite, f), {})}
+        known_tests = sum(counts[f] for f in known)
+        rate = sum(known.values()) / known_tests if known_tests else 1.0
+        divisor = PARALLEL_SUITES.get(suite, 1)
+        for f, count in counts.items():
+            secs = known.get(f, count * rate)
+            units.append((suite, f, secs / divisor))
+
+    shards, loads = assign(units, n)
+    os.makedirs(args.out, exist_ok=True)
+    for suite, _ in suites:
+        files = sorted(u[1] for u in shards[k - 1] if u[0] == suite)
+        with open(os.path.join(args.out, f"files-{suite}.txt"), "w",
+                  encoding="utf-8") as f:
+            f.writelines(f"{name}\n" for name in files)
+    print(f"test-shards: {len(units)} files, weights column {column}")
+    for i, (shard, load) in enumerate(zip(shards, loads), start=1):
+        per_suite = ", ".join(
+            f"{suite} {sum(1 for u in shard if u[0] == suite)}"
+            for suite, _ in suites)
+        mark = "  <- this shard" if i == k else ""
+        print(f"  shard {i}/{n}: {load / 60:5.1f} min modelled ({per_suite})"
+              f"{mark}")
+    return 0
+
+
+# --- check -----------------------------------------------------------------
+
+def read_ids(path):
+    return [row[1] if len(row) > 1 else row[0] for row in read_tsv(path)]
+
+
+def check_group(group):
+    """Problems found in one sharded job's lists; prints its summary."""
+    legs = sorted(root for root, _, files in os.walk(group)
+                  if "shard.txt" in files)
+    if not legs:
+        return [f"{group}: no shard lists found (did every leg upload?)"]
+    problems, shards = [], {}
+    for leg in legs:
+        with open(os.path.join(leg, "shard.txt"), encoding="utf-8") as f:
+            k, n = parse_shard(f.read().strip())
+        if k in shards:
+            problems.append(f"{group}: shard {k} appears twice "
+                            f"({shards[k][0]} and {leg})")
+        shards[k] = (leg, n)
+    totals = {n for _, n in shards.values()}
+    if len(totals) != 1:
+        return problems + [f"{group}: legs disagree on N: {sorted(totals)}"]
+    n = totals.pop()
+    missing = sorted(set(range(1, n + 1)) - set(shards))
+    if missing:
+        problems.append(f"{group}: no lists from shard(s) {missing} of {n}")
+
+    suites = sorted({name[len("whole-"):-len(".tsv")]
+                     for leg, _ in shards.values() for name in os.listdir(leg)
+                     if name.startswith("whole-") and name.endswith(".tsv")})
+    if not suites:
+        problems.append(f"{group}: no whole-suite lists found")
+    for suite in suites:
+        whole, ran = None, {}
+        for k, (leg, _) in sorted(shards.items()):
+            whole_path = os.path.join(leg, f"whole-{suite}.tsv")
+            ran_path = os.path.join(leg, f"ran-{suite}.tsv")
+            if not os.path.exists(whole_path) or not os.path.exists(ran_path):
+                problems.append(f"{group}: shard {k} has no {suite} lists")
+                continue
+            ids = sorted(read_ids(whole_path))
+            if whole is None:
+                whole = ids
+            elif ids != whole:
+                problems.append(f"{group}: shard {k} collected a different "
+                                f"whole {suite} suite ({len(ids)} tests, "
+                                f"others {len(whole)})")
+            ran[k] = read_ids(ran_path)
+        if whole is None:
+            continue
+        seen = {}
+        for k, ids in ran.items():
+            for test_id in ids:
+                seen.setdefault(test_id, []).append(k)
+        repeated = sorted(t for t, ks in seen.items() if len(ks) > 1)
+        absent = sorted(set(whole) - set(seen))
+        extra = sorted(set(seen) - set(whole))
+        counts = " + ".join(str(len(ran.get(k, []))) for k in range(1, n + 1))
+        total = sum(len(ids) for ids in ran.values())
+        ok = bool(whole) and not (repeated or absent or extra)
+        print(f"{group} {suite}: whole {len(whole)}, shards {counts} = "
+              f"{total} {'OK' if ok else 'MISMATCH'}")
+        if not whole:
+            problems.append(f"{group}: the whole {suite} suite is empty")
+        for label, ids in (("not run by any shard", absent),
+                           ("run by more than one shard", repeated),
+                           ("not in the whole suite", extra)):
+            if ids:
+                problems.append(f"{group} {suite}: {len(ids)} test(s) {label}"
+                                f", first: {ids[0]}")
+    return problems
+
+
+def cmd_check(groups):
+    problems = [p for group in groups for p in check_group(group)]
+    for problem in problems:
+        print(f"error: {problem}", file=sys.stderr)
+    return 1 if problems else 0
+
+
+def main(argv):
+    if len(argv) > 1 and argv[1] == "collect":
+        if len(argv) < 4 or argv[3] != "--":
+            raise SystemExit("usage: test-shards collect OUT -- PYTEST_ARGS...")
+        return cmd_collect(argv[2], argv[4:])
+    parser = argparse.ArgumentParser(prog="test-shards")
+    sub = parser.add_subparsers(dest="command", required=True)
+    plan = sub.add_parser("plan")
+    plan.add_argument("--shard", required=True)
+    plan.add_argument("--out", required=True)
+    plan.add_argument("--backend", default="postgres")
+    plan.add_argument("--config")
+    plan.add_argument("--weights", default=DEFAULT_WEIGHTS)
+    plan.add_argument("collected", nargs="+", metavar="SUITE=COLLECTED")
+    check = sub.add_parser("check")
+    check.add_argument("groups", nargs="+", metavar="DIR")
+    args = parser.parse_args(argv[1:])
+    if args.command == "plan":
+        return cmd_plan(args)
+    return cmd_check(args.groups)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

From 07253c2911251f68a7f1add86169eff37265015f Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 04/10] devtools: run one shard of each pytest suite with
 run-tests --shard

run-tests --shard K/N runs only shard K of the main, comprehensive and
CLI suites. Before each suite, devtools/test-shards prepare collects the
whole suite with the suite's own arguments, picks the shard's files, and
writes the whole and the shard's collected lists under
$EXTENDDB_SHARD_DIR (default discussions/shards) for the summary job.
pytest then loads devtools/pytest_shard.py, which deselects every test
outside the shard's files, so filters, ignores and xdist options apply as
in an unsharded run. A shard with no files of a suite runs nothing for
it rather than calling pytest with no paths.

The existing suite command lines are unchanged; each gains one line that
calls the hook when --shard is given.
---
 devtools/pytest_shard.py | 35 +++++++++++++++++++++++
 devtools/run-tests       | 35 +++++++++++++++++++++++
 devtools/test-shards     | 62 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 132 insertions(+)
 create mode 100644 devtools/pytest_shard.py

diff --git a/devtools/pytest_shard.py b/devtools/pytest_shard.py
new file mode 100644
index 000000000..c11080514
--- /dev/null
+++ b/devtools/pytest_shard.py
@@ -0,0 +1,35 @@
+# Copyright 2026 ExtendDB contributors
+# SPDX-License-Identifier: Apache-2.0
+"""pytest plugin for `devtools/run-tests --shard K/N`.
+
+Loaded with `-p pytest_shard` (run-tests puts devtools/ on PYTHONPATH). It
+keeps the collected tests whose file is listed in $EXTENDDB_SHARD_FILES, one
+path per line relative to the working directory, as `devtools/test-shards
+plan` writes them, and deselects the rest. The suite's own arguments are left
+as they are, so filters, ignores and xdist options apply exactly as in an
+unsharded run. Under xdist every worker reads the same list and keeps the
+same tests.
+"""
+
+import os
+
+import pytest
+
+
+def _shard_files():
+    path = os.environ.get("EXTENDDB_SHARD_FILES")
+    if not path:
+        raise pytest.UsageError(
+            "pytest_shard: EXTENDDB_SHARD_FILES is not set")
+    with open(path, encoding="utf-8") as f:
+        return {line.strip() for line in f if line.strip()}
+
+
+def pytest_collection_modifyitems(config, items):
+    files = _shard_files()
+    keep, drop = [], []
+    for item in items:
+        (keep if os.path.relpath(str(item.path)) in files else drop).append(item)
+    if drop:
+        config.hook.pytest_deselected(items=drop)
+        items[:] = keep
diff --git a/devtools/run-tests b/devtools/run-tests
index 421a97b3a..25ec8809d 100755
--- a/devtools/run-tests
+++ b/devtools/run-tests
@@ -36,6 +36,7 @@
 #   --release         use release build for Rust tests
 #   --filter PATTERN  pass to cargo test / pytest -k
 #   --parallel[=N]    run pytest in parallel (default: 1/3 of CPU cores)
+#   --shard K/N       run only shard K of N of each pytest suite (CI)
 #   --config PATH     config file path (default: extenddb.toml in repo root)
 #
 # REQ-TEST-001, D-1, D-2, D-3, D-4, D-5
@@ -72,6 +73,7 @@ RELEASE=false
 FILTER=""
 CONFIG_PATH=""
 PARALLEL=""
+SHARD=""
 BACKEND="postgres"
 
 usage() {
@@ -97,6 +99,7 @@ Options:
   --release         release build for Rust tests
   --filter PATTERN  filter for cargo test / pytest -k
   --parallel[=N]    run pytest in parallel (default: 1/3 of CPU cores)
+  --shard K/N       run only shard K of N of each pytest suite
   --config PATH     config file path
 EOF
     exit 1
@@ -128,6 +131,7 @@ while [[ $# -gt 0 ]]; do
         --parallel=*)
             PARALLEL="${1#--parallel=}"; shift
             ;;
+        --shard)          SHARD="$2"; shift 2 ;;
         -h|--help)        usage ;;
         *)                echo "error: unknown option: $1"; usage ;;
     esac
@@ -456,6 +460,34 @@ if $RUN_RUST_INTEGRATION; then
     echo ""
 fi
 
+# --- Shards (--shard K/N) ---
+# CI splits the long pytest suites across jobs. Before each suite runs,
+# devtools/test-shards prepare collects the whole suite with the suite's own
+# arguments, picks this shard's files, and records both lists under
+# $SHARD_DIR for the summary job's `devtools/test-shards check`. pytest then
+# loads devtools/pytest_shard.py, which keeps only those files. A shard with
+# no files of a suite runs nothing for it.
+SHARD_DIR="${EXTENDDB_SHARD_DIR:-discussions/shards}"
+shard_suite() {
+    local suite="$1"
+    local -n shard_cmd="$2"
+    local rc=0
+    "$SCRIPT_DIR/test-shards" prepare --shard "$SHARD" --out "$SHARD_DIR" \
+        --backend "$BACKEND" --config "${CONFIG_PATH:-extenddb.toml}" \
+        "$suite" -- "${shard_cmd[@]:3}" || rc=$?
+    if (( rc == 0 )); then
+        export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}"
+        export EXTENDDB_SHARD_FILES="$SHARD_DIR/files-$suite.txt"
+        shard_cmd+=(-p pytest_shard)
+    elif (( rc == 10 )); then
+        echo "  shard $SHARD has no $suite files; nothing to run"
+        shard_cmd=(true)
+    else
+        echo "  ✗ devtools/test-shards prepare failed for the $suite suite"
+        shard_cmd=(false)
+    fi
+}
+
 # --- Pytest ---
 if $RUN_PYTEST; then
     OUTFILE="discussions/test-pytest-${HASH}.txt"
@@ -481,6 +513,7 @@ if $RUN_PYTEST; then
     if [[ -n "$FILTER" ]]; then
         PYTEST_ARGS+=(-k "$FILTER")
     fi
+    if [[ -n "$SHARD" ]]; then shard_suite main PYTEST_ARGS; fi
     if "${PYTEST_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then
         echo "  ✓ Pytest passed"
     else
@@ -543,6 +576,7 @@ if $RUN_COMPREHENSIVE; then
     if [[ -n "$FILTER" ]]; then
         COMP_ARGS+=(-k "$FILTER")
     fi
+    if [[ -n "$SHARD" ]]; then shard_suite comprehensive COMP_ARGS; fi
     if "${COMP_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then
         echo "  ✓ Comprehensive tests passed"
     else
@@ -563,6 +597,7 @@ if $RUN_PYTEST \
     if [[ -n "$FILTER" ]]; then
         CLI_ARGS+=(-k "$FILTER")
     fi
+    if [[ -n "$SHARD" ]]; then shard_suite cli CLI_ARGS; fi
     if "${CLI_ARGS[@]}" 2>&1 | tee "$CLI_OUTFILE"; then
         echo "  ✓ CLI lifecycle tests passed"
     else
diff --git a/devtools/test-shards b/devtools/test-shards
index 1e81f8c2f..16fe844ec 100755
--- a/devtools/test-shards
+++ b/devtools/test-shards
@@ -21,6 +21,15 @@ shards of one CI run together collected every test of the whole suite.
       suite's mean seconds per test. Weights only balance the shards: a wrong
       weight makes one shard slower, it cannot drop or repeat a file.
 
+  test-shards prepare --shard K/N --out DIR [--backend NAME] [--config PATH]
+                      SUITE -- PYTEST_ARGS...
+      What `devtools/run-tests --shard K/N` runs before each pytest suite:
+      collect the whole SUITE with PYTEST_ARGS into DIR/whole-SUITE.tsv, plan
+      shard K's files into DIR/files-SUITE.txt, collect what the shard will
+      run (PYTEST_ARGS plus `-p pytest_shard`) into DIR/ran-SUITE.tsv, and
+      write K/N to DIR/shard.txt. Exits 0 when the shard has files of SUITE,
+      10 when it has none (the caller runs nothing), 1 on any failure.
+
   test-shards check DIR...
       Each DIR holds the lists written by the legs of one sharded job, one
       subdirectory per leg (shard.txt, whole-SUITE.tsv, ran-SUITE.tsv), as
@@ -32,6 +41,7 @@ shards of one CI run together collected every test of the whole suite.
 import argparse
 import contextlib
 import os
+import subprocess
 import sys
 
 # The main suite runs under pytest-xdist; on the CI runners (4 vCPU) that is
@@ -258,11 +268,63 @@ def cmd_check(groups):
     return 1 if problems else 0
 
 
+# --- prepare ---------------------------------------------------------------
+
+EMPTY_SHARD = 10
+
+
+def cmd_prepare(argv):
+    if "--" not in argv:
+        raise SystemExit("usage: test-shards prepare --shard K/N --out DIR "
+                         "[--backend NAME] [--config PATH] SUITE -- "
+                         "PYTEST_ARGS...")
+    split = argv.index("--")
+    parser = argparse.ArgumentParser(prog="test-shards prepare")
+    parser.add_argument("--shard", required=True)
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--backend", default="postgres")
+    parser.add_argument("--config")
+    parser.add_argument("--weights", default=DEFAULT_WEIGHTS)
+    parser.add_argument("suite")
+    args = parser.parse_args(argv[:split])
+    pytest_args = argv[split + 1:]
+    k, n = parse_shard(args.shard)
+    os.makedirs(args.out, exist_ok=True)
+    whole, ran, files = (os.path.join(args.out, f"{kind}-{args.suite}{ext}")
+                         for kind, ext in (("whole", ".tsv"), ("ran", ".tsv"),
+                                           ("files", ".txt")))
+
+    # Each collection runs in its own interpreter: pytest does not support
+    # two sessions importing the same test modules in one process.
+    def collect(out, extra, env=None):
+        return subprocess.call([sys.executable, os.path.abspath(__file__),
+                                "collect", out, "--", *extra, *pytest_args],
+                               env=env)
+
+    if collect(whole, []):
+        return 1
+    args.collected = [f"{args.suite}={whole}"]
+    if cmd_plan(args):
+        return 1
+    devtools = os.path.dirname(os.path.abspath(__file__))
+    path = os.environ.get("PYTHONPATH")
+    env = dict(os.environ, EXTENDDB_SHARD_FILES=files,
+               PYTHONPATH=f"{devtools}{os.pathsep}{path}" if path else devtools)
+    if collect(ran, ["-p", "pytest_shard"], env):
+        return 1
+    with open(os.path.join(args.out, "shard.txt"), "w", encoding="utf-8") as f:
+        f.write(f"{k}/{n}\n")
+    with open(files, encoding="utf-8") as f:
+        return 0 if f.read().strip() else EMPTY_SHARD
+
+
 def main(argv):
     if len(argv) > 1 and argv[1] == "collect":
         if len(argv) < 4 or argv[3] != "--":
             raise SystemExit("usage: test-shards collect OUT -- PYTEST_ARGS...")
         return cmd_collect(argv[2], argv[4:])
+    if len(argv) > 1 and argv[1] == "prepare":
+        return cmd_prepare(argv[2:])
     parser = argparse.ArgumentParser(prog="test-shards")
     sub = parser.add_subparsers(dest="command", required=True)
     plan = sub.add_parser("plan")

From be58f0ef629c59160a56a495b1dcf7f9d45cbb88 Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 05/10] devtools: return the shard command through SHARD_CMD

bash 4.2, which some development hosts still run, has no `local -n`, so
shard_suite now takes the suite's command as arguments and leaves the
command to run in SHARD_CMD.
---
 devtools/run-tests | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/devtools/run-tests b/devtools/run-tests
index 25ec8809d..4d67714f8 100755
--- a/devtools/run-tests
+++ b/devtools/run-tests
@@ -466,25 +466,27 @@ fi
 # arguments, picks this shard's files, and records both lists under
 # $SHARD_DIR for the summary job's `devtools/test-shards check`. pytest then
 # loads devtools/pytest_shard.py, which keeps only those files. A shard with
-# no files of a suite runs nothing for it.
+# no files of a suite runs nothing for it. shard_suite SUITE CMD... leaves
+# the command to run in SHARD_CMD.
 SHARD_DIR="${EXTENDDB_SHARD_DIR:-discussions/shards}"
 shard_suite() {
-    local suite="$1"
-    local -n shard_cmd="$2"
-    local rc=0
+    local suite="$1" rc=0
+    shift
+    # "${@:4}" is the suite's arguments after "python3 -m pytest".
     "$SCRIPT_DIR/test-shards" prepare --shard "$SHARD" --out "$SHARD_DIR" \
         --backend "$BACKEND" --config "${CONFIG_PATH:-extenddb.toml}" \
-        "$suite" -- "${shard_cmd[@]:3}" || rc=$?
+        "$suite" -- "${@:4}" || rc=$?
+    SHARD_CMD=("$@")
     if (( rc == 0 )); then
         export PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}"
         export EXTENDDB_SHARD_FILES="$SHARD_DIR/files-$suite.txt"
-        shard_cmd+=(-p pytest_shard)
+        SHARD_CMD+=(-p pytest_shard)
     elif (( rc == 10 )); then
         echo "  shard $SHARD has no $suite files; nothing to run"
-        shard_cmd=(true)
+        SHARD_CMD=(true)
     else
         echo "  ✗ devtools/test-shards prepare failed for the $suite suite"
-        shard_cmd=(false)
+        SHARD_CMD=(false)
     fi
 }
 
@@ -513,7 +515,7 @@ if $RUN_PYTEST; then
     if [[ -n "$FILTER" ]]; then
         PYTEST_ARGS+=(-k "$FILTER")
     fi
-    if [[ -n "$SHARD" ]]; then shard_suite main PYTEST_ARGS; fi
+    if [[ -n "$SHARD" ]]; then shard_suite main "${PYTEST_ARGS[@]}"; PYTEST_ARGS=("${SHARD_CMD[@]}"); fi
     if "${PYTEST_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then
         echo "  ✓ Pytest passed"
     else
@@ -576,7 +578,7 @@ if $RUN_COMPREHENSIVE; then
     if [[ -n "$FILTER" ]]; then
         COMP_ARGS+=(-k "$FILTER")
     fi
-    if [[ -n "$SHARD" ]]; then shard_suite comprehensive COMP_ARGS; fi
+    if [[ -n "$SHARD" ]]; then shard_suite comprehensive "${COMP_ARGS[@]}"; COMP_ARGS=("${SHARD_CMD[@]}"); fi
     if "${COMP_ARGS[@]}" 2>&1 | tee "$OUTFILE"; then
         echo "  ✓ Comprehensive tests passed"
     else
@@ -597,7 +599,7 @@ if $RUN_PYTEST \
     if [[ -n "$FILTER" ]]; then
         CLI_ARGS+=(-k "$FILTER")
     fi
-    if [[ -n "$SHARD" ]]; then shard_suite cli CLI_ARGS; fi
+    if [[ -n "$SHARD" ]]; then shard_suite cli "${CLI_ARGS[@]}"; CLI_ARGS=("${SHARD_CMD[@]}"); fi
     if "${CLI_ARGS[@]}" 2>&1 | tee "$CLI_OUTFILE"; then
         echo "  ✓ CLI lifecycle tests passed"
     else

From 0f11b75929eea9dae681314cddd8ebc1c6b4de3f Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 06/10] ci: run the long pytest jobs as three shards and check
 them

run-integration, run-integration-sqlite and mongodb-pytest each run as a
three-leg matrix, every leg calling run-tests with --shard K/3 against its
own server and database, as the single job did. Each leg uploads its
shard lists.

The integration and mongodb-integration summary jobs keep their names and
their result checks, then download the lists and run devtools/test-shards
check. It fails unless the legs of each job are shards 1 to 3 of one
split, agree on the whole suite, and together collected every test of it
exactly once, so a bad split turns the required check red.
---
 .github/workflows/integration-mongodb.yml | 24 ++++++++++++-
 .github/workflows/integration.yml         | 44 +++++++++++++++++++++--
 2 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml
index 73e8f9c69..5d4caefa3 100644
--- a/.github/workflows/integration-mongodb.yml
+++ b/.github/workflows/integration-mongodb.yml
@@ -55,6 +55,12 @@ jobs:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
     needs: build-mongodb
+    # Three shards by test file; mongodb-integration checks that together
+    # they collected every test of the whole suite.
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3]
     steps:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v5
@@ -69,7 +75,14 @@ jobs:
       # Artifacts do not keep file modes.
       - run: chmod +x target/release/extenddb
       - name: Run MongoDB pytest suite
-        run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel
+        run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel --shard ${{ matrix.shard }}/3
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        if: always()
+        with:
+          name: shards-mongodb-pytest-${{ matrix.shard }}
+          path: discussions/shards/
+          if-no-files-found: warn
+          retention-days: 1
 
   # SnapshotEngine consistency check against a disposable replica set. The
   # tests build their own per-test scratch databases (snap<uuid>_cat and
@@ -176,3 +189,12 @@ jobs:
              [ "${{ needs.mongodb-snapshot.result }}" != "success" ]; then
             exit 1
           fi
+      # Fails unless the shards together collected every test of the whole
+      # suite exactly once, so a bad split cannot pass.
+      - uses: actions/checkout@v6
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          pattern: shards-mongodb-pytest-*
+          path: shards/mongodb-pytest
+      - name: Check the shards ran every test
+        run: python3 devtools/test-shards check shards/mongodb-pytest
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 0b81bc2b7..5d6ad3555 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -77,6 +77,12 @@ jobs:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
     needs: build-postgres
+    # Three shards by test file (devtools/test-shards); the integration job
+    # checks that together they collected every test of the whole suite.
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3]
     services:
       postgres:
         # pgvector's own image, which is postgres:16 plus the extension. The
@@ -142,7 +148,15 @@ jobs:
           EXTENDDB_ADMIN_USER: admin
           EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
           EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
-        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
+        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" --shard ${{ matrix.shard }}/3
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        if: always()
+        with:
+          name: shards-run-integration-${{ matrix.shard }}
+          path: discussions/shards/
+          if-no-files-found: warn
+          retention-days: 1
 
       # run-tests restarts the server daemonized to apply the import/export
       # config, and a daemonized server logs to syslog, not the job log. Without
@@ -156,6 +170,10 @@ jobs:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
     needs: build-sqlite
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3]
     steps:
       - uses: actions/checkout@v6
 
@@ -202,7 +220,15 @@ jobs:
           EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
           EXTENDDB_ADMIN_USER: admin
           EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
-        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
+        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" --shard ${{ matrix.shard }}/3
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        if: always()
+        with:
+          name: shards-run-integration-sqlite-${{ matrix.shard }}
+          path: discussions/shards/
+          if-no-files-found: warn
+          retention-days: 1
 
       # Same rationale as the postgres job: the daemonized server logs to
       # syslog, so this dump is the only server-side evidence on failure.
@@ -569,3 +595,17 @@ jobs:
              [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then
             exit 1
           fi
+
+      # Fails unless the shards of each sharded job together collected every
+      # test of the whole suite exactly once, so a bad split cannot pass.
+      - uses: actions/checkout@v6
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          pattern: shards-run-integration-[0-9]*
+          path: shards/run-integration
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          pattern: shards-run-integration-sqlite-*
+          path: shards/run-integration-sqlite
+      - name: Check the shards ran every test
+        run: python3 devtools/test-shards check shards/run-integration shards/run-integration-sqlite

From 98629a7f79fa65607c4e4007485b1fea3659a434 Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 07/10] ci: run the PostgreSQL storage-level tests in their own
 job

run-rust-integration ran the storage-level tests after the wire-level
Rust suite, 7.1 of its 27.4 minutes in the merge group for #511 on
30 Oct. They drive the storage layer against the job's PostgreSQL and
need no server, so run-postgres-storage-tests runs them with the same
image, environment and cargo command, without waiting for the release
build. The integration summary job requires it.
---
 .github/workflows/integration.yml | 42 ++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 5d6ad3555..8871bfe5f 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -379,6 +379,40 @@ jobs:
           EXTENDDB_EXPECT_VECTORS: "1"
         run: devtools/run-tests --extenddb --rust-integration --release
 
+      # The daemonized server logs to syslog; dump it so server-side failures
+      # are diagnosable from the job log.
+      - name: Dump server syslog on failure
+        if: failure()
+        run: sudo journalctl -t extenddb --no-pager -n 500 || true
+
+  # The PostgreSQL storage-level tests drive the storage layer directly and
+  # need only the database, not a running server, so they run beside
+  # run-rust-integration instead of after it.
+  run-postgres-storage-tests:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_PASSWORD: devpass
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 127.0.0.1:5432:5432
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
       # The control plane for vector indexes is not reachable over the wire while
       # this backend declares no vector search capability, so its tests drive the
       # storage layer directly against this job's PostgreSQL. The PutItem create
@@ -402,12 +436,6 @@ jobs:
           --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test backup_restore
           --test gsi_online_build --test gsi_stale_writer --test table_snapshot
 
-      # The daemonized server logs to syslog; dump it so server-side failures
-      # are diagnosable from the job log.
-      - name: Dump server syslog on failure
-        if: failure()
-        run: sudo journalctl -t extenddb --no-pager -n 500 || true
-
   # The runtime-detection proof: the same binary, against a PostgreSQL with no
   # pgvector, must refuse vector indexes over the wire. Today the job above runs
   # on a plain image too, so this looks like a duplicate of it; it is not, because
@@ -581,6 +609,7 @@ jobs:
         run-integration-sqlite,
         run-integration-dev-mode,
         run-rust-integration,
+        run-postgres-storage-tests,
         run-rust-integration-postgres-novector,
         run-rust-integration-sqlite,
       ]
@@ -591,6 +620,7 @@ jobs:
              [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \
              [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration.result }}" != "success" ] || \
+             [ "${{ needs.run-postgres-storage-tests.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then
             exit 1

From b7fbcb76a6d671d3225697809c2401bda8bb5d56 Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 08/10] ci: pass the shard through EXTENDDB_TEST_SHARD

run-tests reads its --shard default from EXTENDDB_TEST_SHARD, and the
three sharded jobs set it at job level, so their run lines are unchanged
from main. Each upload of the shard lists moves to the end of its job.
Open pull requests that edit those run lines or add steps before them
merge without conflicts.
---
 .github/workflows/integration-mongodb.yml |  6 +++-
 .github/workflows/integration.yml         | 38 +++++++++++++----------
 devtools/run-tests                        |  3 +-
 3 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml
index 5d4caefa3..378451500 100644
--- a/.github/workflows/integration-mongodb.yml
+++ b/.github/workflows/integration-mongodb.yml
@@ -61,6 +61,8 @@ jobs:
       fail-fast: false
       matrix:
         shard: [1, 2, 3]
+    env:
+      EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3
     steps:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v5
@@ -75,7 +77,9 @@ jobs:
       # Artifacts do not keep file modes.
       - run: chmod +x target/release/extenddb
       - name: Run MongoDB pytest suite
-        run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel --shard ${{ matrix.shard }}/3
+        run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel
+
+      # mongodb-integration checks these lists.
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         if: always()
         with:
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 8871bfe5f..5a240377b 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -83,6 +83,8 @@ jobs:
       fail-fast: false
       matrix:
         shard: [1, 2, 3]
+    env:
+      EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3
     services:
       postgres:
         # pgvector's own image, which is postgres:16 plus the extension. The
@@ -148,15 +150,7 @@ jobs:
           EXTENDDB_ADMIN_USER: admin
           EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
           EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432
-        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" --shard ${{ matrix.shard }}/3
-
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
-        if: always()
-        with:
-          name: shards-run-integration-${{ matrix.shard }}
-          path: discussions/shards/
-          if-no-files-found: warn
-          retention-days: 1
+        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
 
       # run-tests restarts the server daemonized to apply the import/export
       # config, and a daemonized server logs to syslog, not the job log. Without
@@ -166,6 +160,15 @@ jobs:
         if: failure()
         run: sudo journalctl -t extenddb --no-pager -n 500 || true
 
+      # The integration job checks these lists.
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        if: always()
+        with:
+          name: shards-run-integration-${{ matrix.shard }}
+          path: discussions/shards/
+          if-no-files-found: warn
+          retention-days: 1
+
   run-integration-sqlite:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
@@ -174,6 +177,8 @@ jobs:
       fail-fast: false
       matrix:
         shard: [1, 2, 3]
+    env:
+      EXTENDDB_TEST_SHARD: ${{ matrix.shard }}/3
     steps:
       - uses: actions/checkout@v6
 
@@ -220,8 +225,15 @@ jobs:
           EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
           EXTENDDB_ADMIN_USER: admin
           EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
-        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" --shard ${{ matrix.shard }}/3
+        run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
+
+      # Same rationale as the postgres job: the daemonized server logs to
+      # syslog, so this dump is the only server-side evidence on failure.
+      - name: Dump server syslog on failure
+        if: failure()
+        run: sudo journalctl -t extenddb --no-pager -n 500 || true
 
+      # The integration job checks these lists.
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         if: always()
         with:
@@ -230,12 +242,6 @@ jobs:
           if-no-files-found: warn
           retention-days: 1
 
-      # Same rationale as the postgres job: the daemonized server logs to
-      # syslog, so this dump is the only server-side evidence on failure.
-      - name: Dump server syslog on failure
-        if: failure()
-        run: sudo journalctl -t extenddb --no-pager -n 500 || true
-
   run-integration-dev-mode:
     # dev-mode was shipped with no CI coverage at all, which is how the batch and
     # transaction authorization regression reached main: the build compiled, so
diff --git a/devtools/run-tests b/devtools/run-tests
index 4d67714f8..a0da2ff5c 100755
--- a/devtools/run-tests
+++ b/devtools/run-tests
@@ -73,7 +73,8 @@ RELEASE=false
 FILTER=""
 CONFIG_PATH=""
 PARALLEL=""
-SHARD=""
+# --shard K/N; CI sets EXTENDDB_TEST_SHARD instead.
+SHARD="${EXTENDDB_TEST_SHARD:-}"
 BACKEND="postgres"
 
 usage() {

From 4df7f810a63b6f4283aed073539cd77a7b309b7f Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 18:59:25 +0000
Subject: [PATCH 09/10] ci: escape list fields, quiet shard collection,
 overwrite artifacts

- devtools/test-shards collect escapes backslashes, tabs and newlines in
  each path and test id, so an id carrying a tab no longer splits a row
  and makes check report a test in two shards.
- prepare captures each collection's output and prints only its summary
  line, or the whole output when the collection fails. Each leg's log
  had carried the collection tree twice, about 4,900 lines for the main
  suite.
- Every upload-artifact step sets overwrite: true, so re-running one
  failed job can upload its artifact under the same name again.
---
 .github/workflows/integration-cassandra.yml |  2 ++
 .github/workflows/integration-mongodb.yml   |  4 ++++
 .github/workflows/integration.yml           |  8 +++++++
 devtools/test-shards                        | 23 ++++++++++++++++-----
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml
index 8c037967e..0d8e40fde 100644
--- a/.github/workflows/integration-cassandra.yml
+++ b/.github/workflows/integration-cassandra.yml
@@ -117,6 +117,8 @@ jobs:
           path: target/release/extenddb
           if-no-files-found: error
           retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
 
   cassandra-pytest:
     runs-on: extenddb_ubuntu-2404_4-core
diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml
index 378451500..0e766d54f 100644
--- a/.github/workflows/integration-mongodb.yml
+++ b/.github/workflows/integration-mongodb.yml
@@ -47,6 +47,8 @@ jobs:
           path: target/release/extenddb
           if-no-files-found: error
           retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
 
   mongodb-pytest:
     # Dedicated ExtendDB runner group (Ubuntu 24.04, 4 vCPU / 16 GB, capped at
@@ -87,6 +89,8 @@ jobs:
           path: discussions/shards/
           if-no-files-found: warn
           retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
 
   # SnapshotEngine consistency check against a disposable replica set. The
   # tests build their own per-test scratch databases (snap<uuid>_cat and
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 5a240377b..ac9c02aa3 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -44,6 +44,8 @@ jobs:
           path: target/release/extenddb
           if-no-files-found: error
           retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
 
   build-sqlite:
     runs-on: extenddb_ubuntu-2404_4-core
@@ -66,6 +68,8 @@ jobs:
           path: target/release/extenddb
           if-no-files-found: error
           retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
 
   run-integration:
     # Dedicated ExtendDB runner group (4 vCPU / 16 GB, capped at 20 concurrent),
@@ -168,6 +172,8 @@ jobs:
           path: discussions/shards/
           if-no-files-found: warn
           retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
 
   run-integration-sqlite:
     runs-on: extenddb_ubuntu-2404_4-core
@@ -241,6 +247,8 @@ jobs:
           path: discussions/shards/
           if-no-files-found: warn
           retention-days: 1
+          # A re-run of a failed job uploads the same name again.
+          overwrite: true
 
   run-integration-dev-mode:
     # dev-mode was shipped with no CI coverage at all, which is how the batch and
diff --git a/devtools/test-shards b/devtools/test-shards
index 16fe844ec..c04b25552 100755
--- a/devtools/test-shards
+++ b/devtools/test-shards
@@ -84,6 +84,11 @@ def cmd_collect(out, pytest_args):
                 rest = item.nodeid.partition("::")[2]
                 rows.append((path, f"{path}::{rest}" if rest else path))
 
+    # Tabs and newlines would split a row; escape them in every list alike.
+    def field(text):
+        return (text.replace("\\", "\\\\").replace("\t", "\\t")
+                .replace("\n", "\\n"))
+
     # pytest prints the collected ids; keep stdout for callers that read it.
     with contextlib.redirect_stdout(sys.stderr):
         rc = pytest.main(["--collect-only", "-q", *pytest_args],
@@ -95,7 +100,7 @@ def cmd_collect(out, pytest_args):
         return int(rc)
     with open(out, "w", encoding="utf-8") as f:
         for path, test_id in rows:
-            f.write(f"{path}\t{test_id}\n")
+            f.write(f"{field(path)}\t{field(test_id)}\n")
     print(f"test-shards: {len(rows)} tests collected into {out}",
           file=sys.stderr)
     return 0
@@ -295,11 +300,19 @@ def cmd_prepare(argv):
                                            ("files", ".txt")))
 
     # Each collection runs in its own interpreter: pytest does not support
-    # two sessions importing the same test modules in one process.
+    # two sessions importing the same test modules in one process. Its
+    # output is the whole collection tree, so it is shown only on failure.
     def collect(out, extra, env=None):
-        return subprocess.call([sys.executable, os.path.abspath(__file__),
-                                "collect", out, "--", *extra, *pytest_args],
-                               env=env)
+        proc = subprocess.run([sys.executable, os.path.abspath(__file__),
+                               "collect", out, "--", *extra, *pytest_args],
+                              env=env, stdout=subprocess.PIPE,
+                              stderr=subprocess.STDOUT, text=True)
+        if proc.returncode:
+            sys.stderr.write(proc.stdout)
+        else:
+            sys.stderr.writelines(line for line in proc.stdout.splitlines(True)
+                                  if line.startswith("test-shards:"))
+        return proc.returncode
 
     if collect(whole, []):
         return 1

From 4a45a51c44bc3bd513820099e0a15b9d89571362 Mon Sep 17 00:00:00 2001
From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com>
Date: Fri, 30 Oct 2026 21:35:48 +0000
Subject: [PATCH 10/10] ci: save Rust caches only from pushes to main in the
 new jobs

build-postgres, build-sqlite, run-postgres-storage-tests, build-mongodb
and build-cassandra are new in this branch, so #680 did not reach their
Swatinem/rust-cache steps. They now set save-if to
github.ref == 'refs/heads/main', as every other rust-cache step outside
launcher.yml does, so pull request and merge group runs of these jobs
restore main's caches and save nothing.
---
 .github/workflows/integration-cassandra.yml | 2 ++
 .github/workflows/integration-mongodb.yml   | 2 ++
 .github/workflows/integration.yml           | 6 ++++++
 3 files changed, 10 insertions(+)

diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml
index 0d8e40fde..f90ddf0e5 100644
--- a/.github/workflows/integration-cassandra.yml
+++ b/.github/workflows/integration-cassandra.yml
@@ -109,6 +109,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Build release (cassandra backend)
         run: cargo build --release --no-default-features --features cassandra
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml
index 0e766d54f..aedef1205 100644
--- a/.github/workflows/integration-mongodb.yml
+++ b/.github/workflows/integration-mongodb.yml
@@ -39,6 +39,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Build release (mongodb backend)
         run: cargo build --release --no-default-features --features mongodb-test-hooks
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index ac9c02aa3..5339b373c 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -34,6 +34,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Build release
         run: cargo build --release
@@ -58,6 +60,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Build release (SQLite backend)
         run: cargo build --release -p extenddb --no-default-features --features sqlite
@@ -426,6 +430,8 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       # The control plane for vector indexes is not reachable over the wire while
       # this backend declares no vector search capability, so its tests drive the