fix(fsck): verify reachable pointer payloads

crab/docs/guides/fsck.md

@@ -13,6 +13,12 @@ crab fsck [OPTIONS]
 `crab fsck` cross-checks Crab manifests, pack/index presence, the shard/xorb
 data chain, and coordination state against the object store.
 
+For protocol-v2 repositories, the check pins an authenticated root, verifies
+Git connectivity, and follows every reachable Crab or Git LFS pointer to its
+origin payload. Crab files must reconstruct to the pointer's exact Blake3 hash
+and size; LFS objects must match their SHA-256 identity and size. Full checking
+reads the payload bytes, even when a local cache already contains them.
+
 With `--repair`, it can automatically fix certain categories of issues.
 
 ## Options
@@ -104,7 +110,7 @@ crab fsck: 1 error(s), 0 info, 1 repaired, 0 repair failure(s)
 
 ## Verification Boundaries
 
-- The Git-object check currently confirms that ref-journal records are
+- For legacy layouts, the Git-object check confirms that ref-journal records are
   readable; it does not reconstruct the current repository and run full Git
   reachability for missing commits, trees, or blobs. Use `crab recover history
   verify <generation>` for strict Git fsck of a selected historical root, and

crab/src/cmd/fsck_store.rs

@@ -91,13 +91,13 @@ pub struct StoreChecker {
     capsule: Option<CapsuleFsckState>,
 }
 
-#[derive(Clone)]
 struct CapsuleFsckState {
     view: crab_read::capsule_protocol::CapsuleRepositoryView,
     catalog: crab_metadata::capsule_protocol::PointerCatalog,
     source_lifetime: Arc<crab_metadata::capsule_protocol::RetainedCapsuleSources>,
     browse_lifetime:
         Arc<tokio::sync::OnceCell<crab_metadata::capsule_protocol::BrowseRetentionCapture>>,
+    git_workspace: tokio::sync::OnceCell<tempfile::TempDir>,
 }
 
 impl StoreChecker {
@@ -148,6 +148,7 @@ impl StoreChecker {
                 catalog,
                 source_lifetime: Arc::new(source_lifetime),
                 browse_lifetime: Arc::new(tokio::sync::OnceCell::new()),
+                git_workspace: tokio::sync::OnceCell::new(),
             }),
         })
     }
@@ -245,6 +246,25 @@ impl StoreChecker {
             }
         }
 
+        if issues.is_empty() {
+            // Keep the authenticated Git database across phases. Catalog bodies
+            // alone cannot prove reachable pointer coverage or whole-file/LFS bytes.
+            let workspace = state
+                .git_workspace
+                .get_or_try_init(|| {
+                    check_capsule_git_connectivity(&self.store, &self.router, state)
+                })
+                .await?;
+            crab_read::capsule_protocol::verify_installed_dependencies(
+                &layout,
+                workspace.path(),
+                state.view.refs(),
+                &state.catalog,
+                CAPSULE_GIT_SCAN_LIMITS,
+                &CancellationToken::new(),
+            )
+            .await?;
+        }
         self.check_capsule_root_stability(state).await?;
         Ok(issues)
     }
@@ -1214,10 +1234,10 @@ async fn install_capsule_git_packs(
 }
 
 async fn check_capsule_git_connectivity(
-    store: Store,
-    router: StoreLayout,
-    state: CapsuleFsckState,
-) -> Result<()> {
+    store: &Store,
+    router: &StoreLayout,
+    state: &CapsuleFsckState,
+) -> Result<tempfile::TempDir> {
     let workspace = tempfile::tempdir()?;
     crab_git::initialize_bare_git_dir(workspace.path())?;
     let storage = store.as_storage().clone();
@@ -1233,7 +1253,7 @@ async fn check_capsule_git_connectivity(
         .map(|(name, oid)| (name.clone(), oid.clone()))
         .collect::<Vec<_>>();
     install_capsule_git_packs(
-        state.view,
+        state.view.clone(),
         layout,
         workspace.path().to_owned(),
         MAX_FSCK_FRONTIER_BYTES,
@@ -1247,7 +1267,8 @@ async fn check_capsule_git_connectivity(
     .await
     .map_err(|error| CrabError::Internal(format!("Git connectivity scan failed: {error}")))??;
     crab_git::batch::verify_git_dir_blobs(workspace.path(), &scan.unchecked_blobs, &|| false)
-        .map_err(CrabError::Io)
+        .map_err(CrabError::Io)?;
+    Ok(workspace)
 }
 
 impl FsckChecker for StoreChecker {
@@ -1257,13 +1278,15 @@ impl FsckChecker for StoreChecker {
         let store = self.store.clone();
         let router = self.router.clone();
         let prefix = self.prefix.clone();
-        let capsule = self.capsule.clone();
         Box::pin(async move {
-            if let Some(state) = capsule {
+            if let Some(state) = &self.capsule {
                 // Pack installation validates every advertised pack. The reachable
                 // walker then proves each ref's commit/tree/blob closure instead of
                 // treating an authenticated ref OID as sufficient evidence.
-                check_capsule_git_connectivity(store, router, state).await?;
+                state
+                    .git_workspace
+                    .get_or_try_init(|| check_capsule_git_connectivity(&store, &router, state))
+                    .await?;
                 return Ok(Vec::new());
             }
             // Git-object connectivity requires a local git repo and gix-fsck.
@@ -1757,6 +1780,9 @@ mod verification_tests;
 #[cfg(test)]
 mod capsule_history_tests;
 
+#[cfg(test)]
+mod pointer_data_tests;
+
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 mod tests {

crab/src/cmd/fsck_store/pointer_data_tests.rs

@@ -0,0 +1,226 @@
+#![expect(clippy::unwrap_used, reason = "test assertions")]
+
+use super::*;
+use bytes::Bytes;
+use crab_metadata::capsule_protocol::{
+    Capsule, CapsuleGitPack, CapsuleRefEdit, CapsuleSection, CapsuleSectionKind,
+    CapsuleTransaction, CapsuleVisibilityDelta, FileCatalogEntry, PointerCatalog,
+    ShardCatalogEntry,
+};
+use crab_metadata::git_visibility::GitVisibilityEdit;
+use sha2::{Digest, Sha256};
+
+async fn publish_pointer(
+    layout: &crab_storage::StoreLayout<crab_storage::Store>,
+    root: crab_metadata::capsule_protocol::RootSnapshot,
+    body: &[u8],
+    delta: &PointerCatalog,
+) -> crab_metadata::capsule_protocol::RootSnapshot {
+    let kind = gix_object::Kind::Blob;
+    let oid = crab_remote::objects::object_id(kind, body)
+        .unwrap()
+        .to_string();
+    let mut bytes = Vec::new();
+    crab_git::pack_writer::write_pack(
+        &mut bytes,
+        std::iter::once(Ok((kind, body.len() as u64, body))),
+        1024 * 1024,
+        || false,
+    )
+    .unwrap();
+    let scratch = tempfile::tempdir().unwrap();
+    let path = scratch.path().join("source.pack");
+    std::fs::write(&path, &bytes).unwrap();
+    let indexed = crab_git::pack::install_pack_file_from_path(
+        &scratch.path().join("indexed"),
+        &path,
+        blake3::hash(&bytes).to_hex().as_ref(),
+        1024 * 1024,
+        true,
+    )
+    .unwrap();
+    let checksum = gix_hash::ObjectId::from_hex(indexed.git_sha1.as_bytes()).unwrap();
+    let kinds = crab_git::pack_locator::encode_pack_kind_metadata(checksum, &[kind]).unwrap();
+    let pack = CapsuleGitPack::new(
+        Bytes::from(bytes),
+        Bytes::from(std::fs::read(indexed.idx_path).unwrap()),
+        Bytes::from(std::fs::read(indexed.rev_path).unwrap()),
+        Bytes::from(kinds),
+        indexed.git_sha1,
+        1,
+    )
+    .unwrap();
+    let name = "refs/tags/payload";
+    let transaction = CapsuleTransaction::new(
+        root.record().digest(),
+        vec![CapsuleRefEdit::new(name, None, Some(oid.clone()), None)],
+    )
+    .unwrap();
+    let visibility = CapsuleVisibilityDelta::new(BTreeMap::from([(
+        name.to_owned(),
+        GitVisibilityEdit::from_replacement_objects(None, oid.clone(), vec![oid]),
+    )]))
+    .unwrap();
+    let capsule = Capsule::build(
+        &transaction,
+        vec![pack],
+        vec![
+            CapsuleSection::new(
+                CapsuleSectionKind::VisibilityDelta,
+                visibility.encode().unwrap(),
+            ),
+            CapsuleSection::new(
+                CapsuleSectionKind::CatalogDelta,
+                delta.encode_delta().unwrap(),
+            ),
+        ],
+    )
+    .unwrap();
+    crab_write::capsule_protocol::publish(layout, root, &transaction, &capsule)
+        .await
+        .unwrap()
+}
+
+async fn pointer_checker(case: &str) -> StoreChecker {
+    let (store, prefix, root, xorb_hash, shard_hash) =
+        super::tests::capsule_checker_fixture().await;
+    let router = StoreLayout::new(store.clone(), prefix.clone());
+    let layout = crab_storage::StoreLayout::with_global_prefix(
+        store.as_storage().clone(),
+        router.repo_prefix().to_owned(),
+        router.global_prefix().to_owned(),
+    );
+    let mut pointer = Pointer {
+        file_hash: blake3::hash(&[0x61; 1024]).into(),
+        size: 1024,
+        shard_hint: None,
+    };
+    let mut delta = PointerCatalog::new();
+    match case {
+        "missing" => pointer.file_hash = [42; 32],
+        "size" => pointer.size += 1,
+        "hash" => {
+            let (body, _) = store
+                .get_with_etag_bounded(&router.shard_path(&shard_hash), MAX_FSCK_SHARD_BYTES)
+                .await
+                .unwrap();
+            let reader = ShardReader::from_bytes(body, shard_hash);
+            let mut recipe = reader
+                .get_file_info(&MerkleHash::from(pointer.file_hash))
+                .unwrap()
+                .unwrap();
+            pointer.file_hash = [42; 32];
+            recipe.metadata.file_hash = MerkleHash::from(pointer.file_hash);
+            let mut shard = crab_xet::shard::ShardWriter::new();
+            shard
+                .add_xorb(Arc::new(reader.get_xorb_info(&xorb_hash).unwrap().unwrap()))
+                .unwrap();
+            shard.add_file(recipe).unwrap();
+            let (body, hash) = shard.finalize().unwrap();
+            delta
+                .insert_shard(
+                    hash.hex(),
+                    ShardCatalogEntry::new(body.len() as u64, vec![xorb_hash.hex()]),
+                )
+                .unwrap();
+            delta
+                .insert_file(
+                    MerkleHash::from(pointer.file_hash).hex(),
+                    FileCatalogEntry::new(pointer.size, hash.hex()),
+                )
+                .unwrap();
+            store
+                .put(&router.shard_path(&hash), Bytes::from(body))
+                .await
+                .unwrap();
+        }
+        _ => {}
+    }
+    let body = if case.starts_with("lfs-") {
+        let lfs = crab_git::lfs_pointer::LfsPointer {
+            oid: Sha256::digest(b"hello").into(),
+            size: 5,
+            extensions: Vec::new(),
+        };
+        if case != "lfs-missing" {
+            let path = crab_lfs::LfsObjectStore::new(layout.store().clone(), layout.repo_prefix())
+                .object_path_for(&lfs.oid);
+            let body = if case == "lfs-corrupt" {
+                b"wrong"
+            } else {
+                b"hello"
+            };
+            layout
+                .store()
+                .put(&path, Bytes::copy_from_slice(body))
+                .await
+                .unwrap();
+        }
+        lfs.serialize()
+    } else {
+        pointer.serialize()
+    };
+    let root = publish_pointer(&layout, root, &body, &delta).await;
+    StoreChecker::for_capsule_repository(store, prefix, root)
+        .await
+        .unwrap()
+}
+
+#[tokio::test]
+async fn full_fsck_proves_reachable_crab_and_lfs_payloads() {
+    let mut outcomes = Vec::new();
+    for case in [
+        "valid",
+        "missing",
+        "size",
+        "hash",
+        "lfs-valid",
+        "lfs-missing",
+        "lfs-corrupt",
+    ] {
+        let checker = Box::pin(pointer_checker(case)).await;
+        let (_, outcome) = crate::cmd::fsck::run_fsck(
+            &crate::cmd::fsck::FsckArgs::default(),
+            &checker,
+            &crate::cmd::fsck::NullRepairer,
+            &CancellationToken::new(),
+            Duration::from_hours(1),
+            None,
+        )
+        .await
+        .unwrap();
+        outcomes.push((case, outcome.errors == 0));
+    }
+    assert_eq!(
+        outcomes,
+        vec![
+            ("valid", true),
+            ("missing", false),
+            ("size", false),
+            ("hash", false),
+            ("lfs-valid", true),
+            ("lfs-missing", false),
+            ("lfs-corrupt", false)
+        ]
+    );
+}
+
+#[tokio::test]
+async fn fsck_retains_one_git_database_through_data_checks_then_releases_it() {
+    let checker = Box::pin(pointer_checker("valid")).await;
+    checker.check_git_objects().await.unwrap();
+    let state = checker.capsule.as_ref().unwrap();
+    let original = state.git_workspace.get().unwrap().path().to_owned();
+    let clean = checker.check_data_chain().await.unwrap().is_empty();
+    let reused = state.git_workspace.get().unwrap().path() == original;
+    drop(checker);
+    assert!(clean && reused && !original.exists());
+}
+
+#[tokio::test]
+async fn data_checks_prove_pointer_coverage_without_a_prior_git_phase() {
+    let checker = Box::pin(pointer_checker("missing")).await;
+    assert!(matches!(checker.check_data_chain().await,
+        Err(CrabError::CorruptObject { reason, .. }) if reason.contains("absent from the catalog")
+    ));
+}