feat(backup): backup sync, --adopt-store and --prune-missing [backup/restore 10]

.github/workflows/clippy.yml

@@ -32,4 +32,11 @@ jobs:
         with:
           # Only pushes to main save, so other runs restore main's cache.
           save-if: ${{ github.ref == 'refs/heads/main' }}
-      - run: cargo clippy --all-targets -- -D warnings
+      # With the Cassandra and MongoDB crates' `test-hooks` features, so the
+      # hook modules (`src/test_hooks.rs`) and their call sites, which only
+      # the integration test binaries build otherwise, are linted too.
+      - run: cargo clippy --all-targets --features extenddb-storage-cassandra/test-hooks,extenddb-storage-mongodb/test-hooks -- -D warnings
+      # The backup job's kill-point pauses (`EXTENDDB_BACKUP_PAUSE`), which
+      # `devtools/backup-kill-points.sh` runs by hand on a build with the
+      # feature; no other job builds it.
+      - run: cargo clippy --all-targets --features extenddb-engine/backup-test-hooks -- -D warnings

.github/workflows/integration-cassandra.yml

@@ -113,8 +113,10 @@ jobs:
       - name: Direct storage-trait integration tests
         # Ported from the plug-in repo's tests/rust suite. Parallel-safe:
         # each test provisions its own account and keyspace; the shared
-        # control-plane setting test restores what it mutates.
-        run: cargo test -p extenddb-storage-cassandra --test direct_integration
+        # control-plane setting test restores what it mutates. The
+        # test-hooks feature compiles the tests that pause an operation or
+        # shorten a heartbeat (crates/storage-cassandra/src/test_hooks.rs).
+        run: cargo test -p extenddb-storage-cassandra --features test-hooks --test direct_integration
         env:
           EXTENDDB_TEST_CASSANDRA: required
 
@@ -171,6 +173,10 @@ jobs:
       - name: Build release (cassandra backend)
         run: cargo build --release --no-default-features --features cassandra
       - name: Run Cassandra pytest suite
+        env:
+          # run-cassandra-tests initializes the server without [backup.store],
+          # and Cassandra takes the in-catalog create with one.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: devtools/run-cassandra-tests -- --pytest --comprehensive --parallel
 
   cassandra-rust:
@@ -226,6 +232,10 @@ jobs:
       - name: Build release (cassandra backend)
         run: cargo build --release --no-default-features --features cassandra
       - name: Run Cassandra rust integration suite
+        env:
+          # run-cassandra-tests initializes the server without [backup.store]:
+          # the Rust backup tests restore the in-catalog create's backups.
+          EXTENDDB_TEST_BACKUP_STORE: none
         run: devtools/run-cassandra-tests -- --rust-integration
 
   cassandra-production-build:

.github/workflows/integration-mongodb.yml

@@ -53,6 +53,12 @@ jobs:
       - name: Build release (mongodb backend)
         run: cargo build --release --no-default-features --features mongodb-test-hooks
       - name: Run MongoDB pytest suite
+        env:
+          # run-mongodb-tests gives the server a filesystem [backup.store]
+          # under its output directory: CreateBackup is the store-backed job
+          # and the restore cells (tests/test_mongodb_backup_restore.py)
+          # rewrite prefixes under that root.
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel
 
   # SnapshotEngine consistency check against a disposable replica set. The
@@ -129,8 +135,61 @@ jobs:
       - name: Build release (mongodb backend)
         run: cargo build --release --no-default-features --features mongodb-test-hooks
       - name: Run MongoDB rust integration suite
+        env:
+          # run-mongodb-tests gives the server a filesystem [backup.store]:
+          # the Rust backup tests restore the store-backed job's backups.
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
         run: devtools/run-mongodb-tests --rust-live -- --rust --rust-integration
 
+  # The store-backed CreateBackup on MongoDB: a server with a filesystem
+  # store, the backup selection of the pytest suite with
+  # EXTENDDB_TEST_BACKUP_STORE=filesystem (the fidelity suite's store branch,
+  # the ABAC and authorization backup cells), and the job suite
+  # (tests/test_backup_job.py), which starts servers of its own on a replica
+  # set of its own: the catalog's database names are fixed per replica set,
+  # and each of its tests destroys and initializes the catalog.
+  mongodb-backup-store:
+    runs-on: extenddb_ubuntu-2404_4-core
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+      - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
+        with:
+          cache-on-failure: true
+          # Only pushes to main save, so other runs restore main's cache.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.11"
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt
+      - name: Build release (mongodb backend)
+        run: cargo build --release --no-default-features --features mongodb-test-hooks
+      - name: Run the backup selection against a filesystem store
+        env:
+          EXTENDDB_TEST_BACKUP_STORE: filesystem
+        run: |
+          devtools/run-mongodb-tests --mongo-port 27022 --port 18453 \
+            --backup-store-root "$RUNNER_TEMP/backups" \
+            -- --pytest --parallel --filter "backup"
+      - name: Start the job suite's replica set
+        # No pipe: a bootstrap failure must fail this step.
+        run: devtools/run-mongodb-tests --mongo-only --mongo-port 27023
+      - name: Run the backup job suite
+        env:
+          EXTENDDB_BINARY: ./target/release/extenddb
+          EXTENDDB_TEST_MONGODB_URI: mongodb://127.0.0.1:27023/?replicaSet=rs0&directConnection=true
+        run: python3 -m pytest tests/test_backup_job.py -v -rP -p no:cacheprovider
+      - name: Stop the job suite's replica set
+        if: always()
+        run: docker rm -f extenddb-runtests-mongo-27023
+      - name: Dump server syslog on failure
+        if: failure()
+        run: sudo journalctl -t extenddb --no-pager -n 500 || true
+
   # The catalog migrations against released catalogs and released binaries,
   # each check on a mongo:7 replica set of its own that the script starts
   # with its ports on 127.0.0.1 and removes. The schema check uploads the
@@ -191,12 +250,13 @@ jobs:
   mongodb-integration:
     runs-on: extenddb_ubuntu-2404_4-core
     timeout-minutes: 40
-    needs: [mongodb-pytest, mongodb-rust, mongodb-catalog, mongodb-production-build, mongodb-snapshot]
+    needs: [mongodb-pytest, mongodb-rust, mongodb-backup-store, mongodb-catalog, mongodb-production-build, mongodb-snapshot]
     if: always()
     steps:
       - run: |
           if [ "${{ needs.mongodb-pytest.result }}" != "success" ] || \
              [ "${{ needs.mongodb-rust.result }}" != "success" ] || \
+             [ "${{ needs.mongodb-backup-store.result }}" != "success" ] || \
              [ "${{ needs.mongodb-catalog.result }}" != "success" ] || \
              [ "${{ needs.mongodb-production-build.result }}" != "success" ] || \
              [ "${{ needs.mongodb-snapshot.result }}" != "success" ]; then