fix(lexical): carry text artifact across added and removed files

crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs

@@ -3011,13 +3011,18 @@ impl LatestCodeTextGenerationV1 {
             }
             ancestor.clone_from(&ancestor_source.metadata().manifest().parent_generation);
         };
-        let changed = source.changed_files_since(&parent_source);
+        let replacements = source.file_replacements_since(&parent_source);
         drop(parent_source);
-        let Some(changed) = changed else {
+        let Some(replacements) = replacements else {
             return Ok(None);
         };
-        let changed_count = u64::try_from(changed.len()).unwrap_or(u64::MAX);
-        if changed_count.saturating_mul(8) > source.total_files() {
+        let replaced_files = replacements
+            .iter()
+            .map(|run| {
+                (run.files.end - run.files.start).max(run.parent_files.end - run.parent_files.start)
+            })
+            .fold(0u64, u64::saturating_add);
+        if replaced_files.saturating_mul(8) > source.total_files() {
             return Ok(None);
         }
         let parent_path = code_text_artifact_path(store.store_root(), &descriptor)
@@ -3044,7 +3049,7 @@ impl LatestCodeTextGenerationV1 {
             builder_budget,
             source_state_digest,
             source_format_revision,
-            &changed,
+            &replacements,
             &mut stage_file_pages,
             control,
         )

crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs

@@ -221,6 +221,87 @@ fn an_edit_carries_the_parent_text_artifact_byte_identical_to_a_cold_build() {
     }
 }
 
+/// One commit's written files and removed paths.
+type RosterChange<'a> = (Vec<(&'a str, &'a str)>, Vec<&'a str>);
+
+/// Files added and removed shift every later file's ordinal, yet the edit
+/// still carries: with every parent segment out of the store, the successor
+/// seals byte for byte what a cold build seals through an added file, a
+/// removed one, a file replaced by its neighbour beside a body edit, a
+/// rename across the roster, and a file added first and removed last.
+#[test]
+fn an_added_or_removed_file_carries_the_parent_text_artifact_byte_identical_to_a_cold_build() {
+    let fixture = corpus_fixture();
+    let store = TempDir::new().expect("store");
+    let mut scheduler = scheduler(
+        &fixture,
+        store.path().to_path_buf(),
+        Arc::new(SharedCodeIndexBytePoolV1::default()),
+    );
+    published(scheduler.reconcile_now().expect("publish the parent"));
+    drain_text(&scheduler);
+
+    let added = long_module(9, 150);
+    let body_edit = module(6, "").replace("wrapping_mul(31)", "wrapping_mul(37)");
+    let replacement = module(10, "");
+    let renamed = module(1, "").replace("wrapping_mul(31)", "wrapping_mul(43)");
+    let first = module(11, "");
+    let rounds: [RosterChange<'_>; 6] = [
+        (vec![("src/m03a.rs", added.as_str())], vec![]),
+        (vec![], vec!["src/m05.rs"]),
+        (
+            vec![
+                ("src/m03b.rs", replacement.as_str()),
+                ("src/m06.rs", body_edit.as_str()),
+            ],
+            vec!["src/m03a.rs"],
+        ),
+        (vec![("src/n01.rs", renamed.as_str())], vec!["src/m01.rs"]),
+        (vec![("a_first.rs", first.as_str())], vec![]),
+        (vec![], vec!["src/untouched_6.rs"]),
+    ];
+    let hidden = TempDir::new().expect("hidden segments");
+    for (round, (edits, removals)) in rounds.iter().enumerate() {
+        let parent_segments = segment_files(store.path());
+        for (path, source) in edits {
+            fixture.edit(path, source);
+        }
+        for path in removals {
+            fixture.remove(path);
+        }
+        fixture.commit_all(&format!("roster change {round}"));
+        published(scheduler.reconcile_now().expect("publish the successor"));
+        let successor = scheduler.latest_complete().expect("successor generation");
+        for segment in &parent_segments {
+            std::fs::rename(
+                segment,
+                hidden.path().join(segment.file_name().expect("name")),
+            )
+            .expect("hide a parent segment");
+        }
+        drain_latest_text(&successor);
+        drop(successor);
+        for segment in &parent_segments {
+            std::fs::rename(
+                hidden.path().join(segment.file_name().expect("name")),
+                segment,
+            )
+            .expect("restore a parent segment");
+        }
+        let carried = active_artifact(store.path());
+        let cold = cold_artifact(&fixture);
+        assert_eq!(
+            carried.len(),
+            cold.len(),
+            "round {round}: the carried artifact's size differs from a cold build's"
+        );
+        assert!(
+            carried == cold,
+            "round {round}: the carried artifact's bytes differ from a cold build's"
+        );
+    }
+}
+
 /// A parent artifact the carry cannot trust is not carried: the edit's build
 /// falls back to a cold build of its own source instead of failing or sealing
 /// the parent's damage into the successor.

crates/tracedecay-code-index/src/production/lexical_page_source.rs

@@ -849,6 +849,71 @@ impl VerifiedSealedLexicalPageV1 {
     }
 }
 
+/// One run of a successor's files that replaces a run of its parent's:
+/// parent file ordinals `parent_files` give way to successor ordinals
+/// `files`. Either run can be empty, for files added or removed.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SealedLexicalFileReplacementV1 {
+    pub parent_files: std::ops::Range<u64>,
+    pub files: std::ops::Range<u64>,
+}
+
+impl SealedLexicalFileReplacementV1 {
+    fn new(parent_files: (usize, usize), files: (usize, usize)) -> Option<Self> {
+        let ordinal = |ordinal: usize| u64::try_from(ordinal).ok();
+        Some(Self {
+            parent_files: ordinal(parent_files.0)?..ordinal(parent_files.1)?,
+            files: ordinal(files.0)?..ordinal(files.1)?,
+        })
+    }
+}
+
+/// Merge-walk two strictly ascending path rosters into the runs of files
+/// that differ: a successor file is carried only when its parent holds the
+/// same path and `same_segment(file, parent_file)` holds.
+fn file_replacement_runs(
+    paths: &[&str],
+    parent_paths: &[&str],
+    same_segment: impl Fn(usize, usize) -> bool,
+) -> Option<Vec<SealedLexicalFileReplacementV1>> {
+    let mut replacements = Vec::new();
+    let mut run: Option<(usize, usize)> = None;
+    let (mut file, mut parent_file) = (0, 0);
+    loop {
+        let (path, parent_path) = (paths.get(file), parent_paths.get(parent_file));
+        if path.is_none() && parent_path.is_none() {
+            break;
+        }
+        if path == parent_path && same_segment(file, parent_file) {
+            if let Some((parent_start, start)) = run.take() {
+                replacements.push(SealedLexicalFileReplacementV1::new(
+                    (parent_start, parent_file),
+                    (start, file),
+                )?);
+            }
+        } else {
+            run.get_or_insert((parent_file, file));
+        }
+        match (path, parent_path) {
+            (Some(path), Some(parent_path)) if path < parent_path => file += 1,
+            (Some(path), Some(parent_path)) if path > parent_path => parent_file += 1,
+            (Some(_), None) => file += 1,
+            (None, Some(_)) => parent_file += 1,
+            _ => {
+                file += 1;
+                parent_file += 1;
+            }
+        }
+    }
+    if let Some((parent_start, start)) = run {
+        replacements.push(SealedLexicalFileReplacementV1::new(
+            (parent_start, parent_file),
+            (start, file),
+        )?);
+    }
+    Some(replacements)
+}
+
 /// Final proof that all file ranges in one verified seal were exhausted.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct VerifiedSealedLexicalSourceReceiptV1 {
@@ -1382,11 +1447,16 @@ impl VerifiedSealedLexicalPageSourceV1 {
         &self.source_state_digest
     }
 
-    /// File ordinals whose pages differ from `parent`'s, or `None` when the
-    /// two sources cannot share pages: a different file roster, or route
-    /// identities of a different width, which the serialized rows carry and
-    /// page cuts therefore follow.
-    pub fn changed_files_since(&self, parent: &Self) -> Option<Vec<u64>> {
+    /// The runs of files whose pages differ from `parent`'s, aligned by
+    /// path: a run replaces parent files with successor files, either side
+    /// possibly empty, so an added or removed file is a run too. `None`
+    /// when the two sources cannot share pages: route identities of a
+    /// different width, which the serialized rows carry and page cuts
+    /// therefore follow, or a roster that does not name each file once.
+    pub fn file_replacements_since(
+        &self,
+        parent: &Self,
+    ) -> Option<Vec<SealedLexicalFileReplacementV1>> {
         if self.format_revision != parent.format_revision
             || self.maximum_page_chunks != parent.maximum_page_chunks
             || self.maximum_page_bytes != parent.maximum_page_bytes
@@ -1397,7 +1467,31 @@ impl VerifiedSealedLexicalPageSourceV1 {
         {
             return None;
         }
-        self.file_source.changed_files_since(&parent.file_source)
+        let paths = self.file_paths()?;
+        let parent_paths = parent.file_paths()?;
+        file_replacement_runs(&paths, &parent_paths, |file, parent_file| {
+            self.file_source
+                .same_segment(file, &parent.file_source, parent_file)
+        })
+    }
+
+    /// Each file's logical path in file ordinal order, or `None` unless the
+    /// paths strictly ascend, which is the order a build visits files in.
+    fn file_paths(&self) -> Option<Vec<&str>> {
+        let files = &self.metadata.snapshot().files;
+        let paths = self
+            .file_source
+            .file_keys()
+            .map(|key| {
+                files
+                    .get(usize::try_from(key).ok()?)
+                    .map(|file| file.logical_path.as_str())
+            })
+            .collect::<Option<Vec<_>>>()?;
+        paths
+            .windows(2)
+            .all(|pair| pair[0] < pair[1])
+            .then_some(paths)
     }
 
     /// Stage every page of one file as a build that reached it at

crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs

@@ -1067,3 +1067,69 @@ fn cancellation_during_staging_keeps_the_exact_pre_batch_cursor() {
         cursor_before,
     );
 }
+
+/// Runs from [`file_replacement_runs`] as `(parent_files, files)` pairs,
+/// where a file is carried when its path matches and it is not in `edited`.
+fn replacement_runs(
+    paths: &[&str],
+    parent_paths: &[&str],
+    edited: &[&str],
+) -> Vec<(std::ops::Range<u64>, std::ops::Range<u64>)> {
+    file_replacement_runs(paths, parent_paths, |file, _| {
+        !edited.contains(&paths[file])
+    })
+    .expect("ordinals fit")
+    .into_iter()
+    .map(|run| (run.parent_files, run.files))
+    .collect()
+}
+
+#[test]
+fn file_replacement_runs_align_rosters_by_path() {
+    let parent = ["b", "d", "f"];
+    assert_eq!(replacement_runs(&parent, &parent, &[]), vec![]);
+    assert_eq!(
+        replacement_runs(&parent, &parent, &["d"]),
+        vec![(1..2, 1..2)]
+    );
+    assert_eq!(
+        replacement_runs(&["a", "b", "d", "f"], &parent, &[]),
+        vec![(0..0, 0..1)],
+        "a file added first"
+    );
+    assert_eq!(
+        replacement_runs(&["b", "d", "f", "g"], &parent, &[]),
+        vec![(3..3, 3..4)],
+        "a file added last"
+    );
+    assert_eq!(
+        replacement_runs(&["d", "f"], &parent, &[]),
+        vec![(0..1, 0..0)],
+        "the first file removed"
+    );
+    assert_eq!(
+        replacement_runs(&["b", "d"], &parent, &[]),
+        vec![(2..3, 2..2)],
+        "the last file removed"
+    );
+    assert_eq!(
+        replacement_runs(&["b", "c", "f"], &parent, &[]),
+        vec![(1..2, 1..2)],
+        "a removal beside an addition is one run"
+    );
+    assert_eq!(
+        replacement_runs(&["a", "b", "e", "f"], &parent, &["f"]),
+        vec![(0..0, 0..1), (1..3, 2..4)],
+        "an edit after a replacement extends that run"
+    );
+    assert_eq!(
+        replacement_runs(&["x", "y"], &parent, &[]),
+        vec![(0..3, 0..2)],
+        "every file replaced"
+    );
+    assert_eq!(
+        replacement_runs(&[], &parent, &[]),
+        vec![(0..3, 0..0)],
+        "every file removed"
+    );
+}

crates/tracedecay-code-index/src/production/mod.rs

@@ -89,12 +89,12 @@ pub use generation_statistics::CodeIndexGenerationStatisticsV1;
 mod lexical_page_source;
 pub use lexical_page_source::VerifiedSealedLexicalCursorRestoreErrorV1;
 pub use lexical_page_source::{
-    VerifiedSealedLexicalCursorV1, VerifiedSealedLexicalPageBatchBoundsV1,
-    VerifiedSealedLexicalPageBatchReadV1, VerifiedSealedLexicalPageReadV1,
-    VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalPageV1,
-    VerifiedSealedLexicalSourceReceiptV1, VerifiedSealedLexicalSymbolDisplayV1,
-    VerifiedSealedTextGenerationMetadataV1, advance_import_dictionary_digest,
-    initial_import_dictionary_digest,
+    SealedLexicalFileReplacementV1, VerifiedSealedLexicalCursorV1,
+    VerifiedSealedLexicalPageBatchBoundsV1, VerifiedSealedLexicalPageBatchReadV1,
+    VerifiedSealedLexicalPageReadV1, VerifiedSealedLexicalPageSourceV1,
+    VerifiedSealedLexicalPageV1, VerifiedSealedLexicalSourceReceiptV1,
+    VerifiedSealedLexicalSymbolDisplayV1, VerifiedSealedTextGenerationMetadataV1,
+    advance_import_dictionary_digest, initial_import_dictionary_digest,
 };
 mod decoded_content;
 pub use decoded_content::{DecodedGenerationContentV1, SharedDecodedContentPoolV1};

crates/tracedecay-code-index/src/production/partitioned_codec.rs

@@ -1918,27 +1918,32 @@ impl PartitionedLexicalFileSourceV1 {
         }
     }
 
-    /// Ordinals of the files whose segment differs from `parent`'s at the same
-    /// position, or `None` when the two rosters name different files.
-    pub(super) fn changed_files_since(&self, parent: &Self) -> Option<Vec<u64>> {
-        if self.descriptors.len() != parent.descriptors.len() {
-            return None;
-        }
-        let mut changed = Vec::new();
-        for (ordinal, (child, parent)) in
-            self.descriptors.iter().zip(&parent.descriptors).enumerate()
-        {
-            if child.file_key != parent.file_key {
-                return None;
-            }
-            if child.segment_digest != parent.segment_digest
-                || child.file_occurrence_id != parent.file_occurrence_id
-                || child.symbol_identities_digest != parent.symbol_identities_digest
-            {
-                changed.push(u64::try_from(ordinal).ok()?);
+    /// Each file's snapshot key, in file ordinal order.
+    pub(super) fn file_keys(&self) -> impl Iterator<Item = u32> + '_ {
+        self.descriptors
+            .iter()
+            .map(|descriptor| descriptor.file_key)
+    }
+
+    /// Whether file `ordinal` seals the same segment as `parent`'s file
+    /// `parent_ordinal`, so both emit the same pages.
+    pub(super) fn same_segment(
+        &self,
+        ordinal: usize,
+        parent: &Self,
+        parent_ordinal: usize,
+    ) -> bool {
+        match (
+            self.descriptors.get(ordinal),
+            parent.descriptors.get(parent_ordinal),
+        ) {
+            (Some(child), Some(parent)) => {
+                child.segment_digest == parent.segment_digest
+                    && child.file_occurrence_id == parent.file_occurrence_id
+                    && child.symbol_identities_digest == parent.symbol_identities_digest
             }
+            _ => false,
         }
-        Some(changed)
     }
 
     pub(super) fn lexical_byte_offsets(&self) -> Result<Vec<u64>, CodeIndexProductionErrorV1> {