LeeroyHannigan · Nov 5, 2026 · Nov 5, 2026 · Nov 5, 2026 · Nov 5, 2026 · Nov 5, 2026
@@ -0,0 +1,144 @@
+# The bulk store on an S3-compatible service. integration.yml calls this
+# workflow, so its integration gate job requires the result.
+name: Integration (S3 bulk store)
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  # The bulk store on an S3-compatible service: the engine's store contract
+  # tests, then the Python bulk suite against a server whose [bulk] store is
+  # s3. The service is versitygw, which selects its backend by command
+  # (posix, on the container's /tmp); a services: container takes no
+  # command, so a step starts it. PostgreSQL is the backend, as in
+  # run-integration.
+  run-integration-s3:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_PASSWORD: devpass
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 127.0.0.1:5432:5432
+    env:
+      # The container's root keys. They exist only inside this job's
+      # container and grant nothing anywhere else.
+      S3_ACCESS_KEY: extenddb-ci
+      S3_SECRET_KEY: extenddb-ci-secret
+
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+
+      - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+        with:
+          toolchain: stable
+
+      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+        with:
+          cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Start the S3-compatible service
+        run: |
+          # versity/versitygw v1.0.18, pinned by digest.
+          docker run -d --name s3 -p 127.0.0.1:7070:7070 \
+            -e ROOT_ACCESS_KEY="$S3_ACCESS_KEY" -e ROOT_SECRET_KEY="$S3_SECRET_KEY" \
+            versity/versitygw@sha256:90f8280610a889d705ae93ae4b4064d74ffa9f3b565aacf2e3372b0614d7a796 \
+            posix /tmp
+          for i in $(seq 1 30); do
+            # Any HTTP answer means it is serving: an unsigned request is
+            # answered 403.
+            if curl -s -o /dev/null http://127.0.0.1:7070/; then
+              echo "S3-compatible service ready"
+              exit 0
+            fi
+            sleep 1
+          done
+          docker logs s3
+          echo "S3-compatible service failed to start"
+          exit 1
+
+      - name: Build release
+        run: cargo build --release
+
+      # Every contract case runs once per store; EXTENDDB_TEST_S3_REQUIRED
+      # makes a missing endpoint a failure rather than a skip. The smoke
+      # test reads, writes a multipart report and checks a dropped writer
+      # against the same service. The keys reach the store through the
+      # standard AWS credential variables, as they reach the server.
+      - name: Run the bulk store tests against the S3-compatible service
+        env:
+          EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:7070
+          EXTENDDB_TEST_S3_REQUIRED: "1"
+          EXTENDDB_TEST_S3_SMOKE_ENDPOINT: http://127.0.0.1:7070
+          AWS_ACCESS_KEY_ID: ${{ env.S3_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ env.S3_SECRET_KEY }}
+          AWS_REGION: us-east-1
+          AWS_EC2_METADATA_DISABLED: "true"
+        run: >-
+          cargo test --release -p extenddb-engine --lib --
+          bulk_put::source::contract bulk_put::s3::tests::smoke
+
+      - name: Initialize ExtendDB
+        id: init
+        run: |
+          output=$(./target/release/extenddb init --config extenddb.toml \
+            --pg-host 127.0.0.1 --pg-port 5432 --pg-user postgres --pg-pass devpass 2>&1)
+          echo "$output"
+          password=$(echo "$output" | grep -oP 'Password: \K\S+')
+          echo "admin_password=$password" >> "$GITHUB_OUTPUT"
+
+      - name: Start ExtendDB
+        run: |
+          # --write-pid-file so devtools/run-tests can restart the server
+          # with the S3 store configured and the S3 keys in its environment.
+          ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file &
+          for i in $(seq 1 30); do
+            if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then
+              echo "Server ready"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Server failed to start"
+          exit 1
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt
+
+      # run-tests writes the [bulk] s3 store into extenddb.toml and restarts
+      # the server with the S3 keys as its AWS credentials; the suite reaches
+      # the same service with the same keys to create its buckets and read
+      # the reports.
+      - name: Run the bulk suite on the S3 store
+        env:
+          EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443
+          EXTENDDB_ADMIN_USER: admin
+          EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }}
+          EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:7070
+          EXTENDDB_TEST_S3_ACCESS_KEY_ID: ${{ env.S3_ACCESS_KEY }}
+          EXTENDDB_TEST_S3_SECRET_ACCESS_KEY: ${{ env.S3_SECRET_KEY }}
+        run: devtools/run-tests --extenddb --pytest --bulk-store s3 --parallel --filter bulk_put
+
+      - name: Dump server syslog on failure
+        if: failure()
+        run: sudo journalctl -t extenddb --no-pager -n 500 || true
+
+      - name: Dump the S3-compatible service's log on failure
+        if: failure()
+        run: docker logs s3 || true
@@ -9,7 +9,7 @@ permissions:
   contents: read
 
 # Supersede an in-flight run of the SAME pull request when a new push arrives,
-# so two pushes do not each hold six slots on a capped pool. Everything that is
+# so two pushes do not each hold slots on a capped pool. Everything that is
 # not a pull request gets a group unique to the run (run_id): merge-group runs
 # can share a ref (gh-readonly-queue/main/pr-N-<sha> is reused across
 # re-queues of the same PR), and GitHub concurrency replaces a PENDING run in
@@ -157,9 +157,9 @@ jobs:
         run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export"
 
       # run-tests restarts the server daemonized to apply the import/export
-      # config, and a daemonized server logs to syslog, not the job log. Without
-      # this dump, the storage error behind a client-visible 500 leaves no
-      # evidence anywhere in CI.
+      # and bulk store config, and a daemonized server logs to syslog, not the
+      # job log. Without this dump, the storage error behind a client-visible
+      # 500 leaves no evidence anywhere in CI.
       - name: Dump server syslog on failure
         if: failure()
         run: sudo journalctl -t extenddb --no-pager -n 500 || true
@@ -593,6 +593,10 @@ jobs:
         if: failure()
         run: sudo journalctl -t extenddb --no-pager -n 500 || true
 
+  # The bulk store on an S3-compatible service (integration-s3.yml).
+  run-integration-s3:
+    uses: ./.github/workflows/integration-s3.yml
+
   integration:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
@@ -604,6 +608,7 @@ jobs:
         run-rust-integration,
         run-rust-integration-postgres-novector,
         run-rust-integration-sqlite,
+        run-integration-s3,
       ]
     if: always()
     steps:
@@ -613,6 +618,7 @@ jobs:
              [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration.result }}" != "success" ] || \
              [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \
-             [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ]; then
+             [ "${{ needs.run-rust-integration-sqlite.result }}" != "success" ] || \
+             [ "${{ needs.run-integration-s3.result }}" != "success" ]; then
             exit 1
           fi
@@ -474,6 +474,7 @@ Activate when the user asks about installing, configuring, running, or debugging
 | `crates/storage/src/lib.rs` | Storage trait definitions |
 | `crates/core/src/types.rs` | DynamoDB type definitions |
 | `crates/engine/src/` | Operation handlers |
+| `crates/s3/src/lib.rs` | S3 client building and the startup checks of the server's AWS credentials, for every S3 store |
 | `crates/bin/src/main.rs` | CLI entry point |
 
 ## Environment Variables

@@ -341,28 +341,28 @@ SQLite, PostgreSQL, Cassandra and MongoDB migrate through this framework.
 
 ### Example: a nullable column on `tables`
 
-Migration 8, `table_note` (8 being the next free id at the time of writing;
-migrations 6 and 7 are `backup_catalog_index` and `stream_pending`), adds a
-nullable `note TEXT` column to the catalog's `tables` table on SQLite and
-PostgreSQL.
+Migration 9, `table_note` (9 being the next free id at the time of writing;
+migrations 6, 7 and 8 are `backup_catalog_index`, `stream_pending` and
+`bulk_operations`), adds a nullable `note TEXT` column to the catalog's
+`tables` table on SQLite and PostgreSQL.
 
 In `crates/storage/src/migrations/registry.rs`, at the end of the
 `migration_ids!` list:
 
 ```rust
     /// Adds a nullable `note` column to the `tables` catalog table.
-    TableNote = 8, "table_note",
+    TableNote = 9, "table_note",
         "add a nullable note column to the tables catalog";
 ```
 
 PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file,
-`crates/storage-postgres/migrations/0008_table_note.sql`:
+`crates/storage-postgres/migrations/0009_table_note.sql`:
 
 ```sql
 -- Copyright 2026 ExtendDB contributors
 -- SPDX-License-Identifier: Apache-2.0
 --
--- Migration 8, table_note, PostgreSQL catalog database: a nullable note
+-- Migration 9, table_note, PostgreSQL catalog database: a nullable note
 -- column on tables. IF NOT EXISTS makes a second run change nothing.
 
 ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT;
@@ -377,7 +377,7 @@ In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`:
             bodies: vec![file_body!(
                 StoreKind::Catalog,
                 "migrations",
-                "0008_table_note.sql"
+                "0009_table_note.sql"
             )],
         },
 ```
@@ -391,19 +391,20 @@ and the id in the transactional arm of `runs_in()`:
         | MigrationId::DropContinuousBackups
         | MigrationId::BackupCatalogIndex
         | MigrationId::StreamPending
+        | MigrationId::BulkOperations
         | MigrationId::TableNote => RunsIn::Transaction,
 ```
 
 SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain
 `ADD COLUMN` fails when it runs a second time, so the SQLite body is code
 that checks for the column first.
-`crates/storage-sqlite/src/migrations/m0008_table_note.rs`:
+`crates/storage-sqlite/src/migrations/m0009_table_note.rs`:
 
 ```rust
 // Copyright 2026 ExtendDB contributors
 // SPDX-License-Identifier: Apache-2.0
 
-//! Migration 8, `table_note`, SQLite: a nullable `note` column on `tables`.
+//! Migration 9, `table_note`, SQLite: a nullable `note` column on `tables`.
 //!
 //! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's
 //! columns first and adds the column only when it is missing. A second run
@@ -415,10 +416,10 @@ use extenddb_storage::migrations::{CodeBody, Done, MigrationError};
 use super::SqliteStores;
 
 /// This file, whose SHA-256 the checksum test pins.
-pub(crate) const SOURCE: &str = include_str!("m0008_table_note.rs");
+pub(crate) const SOURCE: &str = include_str!("m0009_table_note.rs");
 
 /// Recorded as this migration's checksum. Changes only with a new migration.
-const FINGERPRINT: &str = "0008_table_note/sqlite/1";
+const FINGERPRINT: &str = "0009_table_note/sqlite/1";
 
 pub(crate) struct TableNote;
 
@@ -457,7 +458,7 @@ beside the others:
 ```rust
 #[rustfmt::skip]
 #[allow(clippy::all, deprecated)]
-mod m0008_table_note;
+mod m0009_table_note;
 ```
 
 and the arm in `migration()`:
@@ -468,8 +469,8 @@ and the arm in `migration()`:
             convergent: true,
             bodies: vec![StoreBody::code(
                 StoreKind::Catalog,
-                &m0008_table_note::TableNote,
-                m0008_table_note::SOURCE,
+                &m0009_table_note::TableNote,
+                m0009_table_note::SOURCE,
             )],
         },
 ```
@@ -481,8 +482,8 @@ rows of `tables`:
 
 ```rust
 #[tokio::test]
-async fn migration_8_adds_the_note_column_to_tables() {
-    let db = temp_db("m8");
+async fn migration_9_adds_the_note_column_to_tables() {
+    let db = temp_db("m9");
     load_dump(&db, DEV_0_1_14).await;
     run(&db).await.expect("migrate");
     let mut text = dump(
@@ -492,12 +493,12 @@ async fn migration_8_adds_the_note_column_to_tables() {
     )
     .await;
     text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await);
-    golden("0008_table_note", &text);
+    golden("0009_table_note", &text);
 }
 ```
 
-`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_8`
-writes `crates/storage-sqlite/tests/fixtures/golden/0008_table_note.txt`, one
+`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_9`
+writes `crates/storage-sqlite/tests/fixtures/golden/0009_table_note.txt`, one
 tab-separated line per row under each query:
 
 ```
@@ -512,15 +513,15 @@ The PostgreSQL body is a file, so it has no golden test.
 The example declares `Kind::Breaking`. A nullable column can be additive by
 the rule in step 4, but declaring it additive needs the old-binary job, which
 this example does not add. As a breaking migration it makes a 1.x binary
-without migration 8 refuse a catalog that has it, and a rollback restores the
+without migration 9 refuse a catalog that has it, and a rollback restores the
 backup.
 
 The same PR changes the code that reads and writes `tables.note` in each
 backend's catalog store, and commits the `docs/schema/sqlite.txt` and
 `docs/schema/postgres.txt` that the integration jobs regenerate. It changes
 no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED`
-entry. `extenddb migrate --list` then shows migration 8, and on a catalog at
-migration 7 `extenddb migrate` lists migration 8 as pending and exits 3.
+entry. `extenddb migrate --list` then shows migration 9, and on a catalog at
+migration 8 `extenddb migrate` lists migration 9 as pending and exits 3.
 
 ### Running the migration tests