diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index 8f0fab57d9..26851b58b6 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -3011,13 +3011,18 @@ impl LatestCodeTextGenerationV1 { } ancestor.clone_from(&ancestor_source.metadata().manifest().parent_generation); }; - let changed = source.changed_files_since(&parent_source); + let replacements = source.file_replacements_since(&parent_source); drop(parent_source); - let Some(changed) = changed else { + let Some(replacements) = replacements else { return Ok(None); }; - let changed_count = u64::try_from(changed.len()).unwrap_or(u64::MAX); - if changed_count.saturating_mul(8) > source.total_files() { + let replaced_files = replacements + .iter() + .map(|run| { + (run.files.end - run.files.start).max(run.parent_files.end - run.parent_files.start) + }) + .fold(0u64, u64::saturating_add); + if replaced_files.saturating_mul(8) > source.total_files() { return Ok(None); } let parent_path = code_text_artifact_path(store.store_root(), &descriptor) @@ -3044,7 +3049,7 @@ impl LatestCodeTextGenerationV1 { builder_budget, source_state_digest, source_format_revision, - &changed, + &replacements, &mut stage_file_pages, control, ) diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs index c30f17d458..9f75d31ec5 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/tests/text_artifact_carry_tests.rs @@ -221,6 +221,87 @@ fn an_edit_carries_the_parent_text_artifact_byte_identical_to_a_cold_build() { } } +/// One commit's written files and removed paths. +type RosterChange<'a> = (Vec<(&'a str, &'a str)>, Vec<&'a str>); + +/// Files added and removed shift every later file's ordinal, yet the edit +/// still carries: with every parent segment out of the store, the successor +/// seals byte for byte what a cold build seals through an added file, a +/// removed one, a file replaced by its neighbour beside a body edit, a +/// rename across the roster, and a file added first and removed last. +#[test] +fn an_added_or_removed_file_carries_the_parent_text_artifact_byte_identical_to_a_cold_build() { + let fixture = corpus_fixture(); + let store = TempDir::new().expect("store"); + let mut scheduler = scheduler( + &fixture, + store.path().to_path_buf(), + Arc::new(SharedCodeIndexBytePoolV1::default()), + ); + published(scheduler.reconcile_now().expect("publish the parent")); + drain_text(&scheduler); + + let added = long_module(9, 150); + let body_edit = module(6, "").replace("wrapping_mul(31)", "wrapping_mul(37)"); + let replacement = module(10, ""); + let renamed = module(1, "").replace("wrapping_mul(31)", "wrapping_mul(43)"); + let first = module(11, ""); + let rounds: [RosterChange<'_>; 6] = [ + (vec![("src/m03a.rs", added.as_str())], vec![]), + (vec![], vec!["src/m05.rs"]), + ( + vec![ + ("src/m03b.rs", replacement.as_str()), + ("src/m06.rs", body_edit.as_str()), + ], + vec!["src/m03a.rs"], + ), + (vec![("src/n01.rs", renamed.as_str())], vec!["src/m01.rs"]), + (vec![("a_first.rs", first.as_str())], vec![]), + (vec![], vec!["src/untouched_6.rs"]), + ]; + let hidden = TempDir::new().expect("hidden segments"); + for (round, (edits, removals)) in rounds.iter().enumerate() { + let parent_segments = segment_files(store.path()); + for (path, source) in edits { + fixture.edit(path, source); + } + for path in removals { + fixture.remove(path); + } + fixture.commit_all(&format!("roster change {round}")); + published(scheduler.reconcile_now().expect("publish the successor")); + let successor = scheduler.latest_complete().expect("successor generation"); + for segment in &parent_segments { + std::fs::rename( + segment, + hidden.path().join(segment.file_name().expect("name")), + ) + .expect("hide a parent segment"); + } + drain_latest_text(&successor); + drop(successor); + for segment in &parent_segments { + std::fs::rename( + hidden.path().join(segment.file_name().expect("name")), + segment, + ) + .expect("restore a parent segment"); + } + let carried = active_artifact(store.path()); + let cold = cold_artifact(&fixture); + assert_eq!( + carried.len(), + cold.len(), + "round {round}: the carried artifact's size differs from a cold build's" + ); + assert!( + carried == cold, + "round {round}: the carried artifact's bytes differ from a cold build's" + ); + } +} + /// A parent artifact the carry cannot trust is not carried: the edit's build /// falls back to a cold build of its own source instead of failing or sealing /// the parent's damage into the successor. diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source.rs b/crates/tracedecay-code-index/src/production/lexical_page_source.rs index ba3c4d41d5..8b4a7a4d81 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source.rs @@ -849,6 +849,71 @@ impl VerifiedSealedLexicalPageV1 { } } +/// One run of a successor's files that replaces a run of its parent's: +/// parent file ordinals `parent_files` give way to successor ordinals +/// `files`. Either run can be empty, for files added or removed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SealedLexicalFileReplacementV1 { + pub parent_files: std::ops::Range, + pub files: std::ops::Range, +} + +impl SealedLexicalFileReplacementV1 { + fn new(parent_files: (usize, usize), files: (usize, usize)) -> Option { + let ordinal = |ordinal: usize| u64::try_from(ordinal).ok(); + Some(Self { + parent_files: ordinal(parent_files.0)?..ordinal(parent_files.1)?, + files: ordinal(files.0)?..ordinal(files.1)?, + }) + } +} + +/// Merge-walk two strictly ascending path rosters into the runs of files +/// that differ: a successor file is carried only when its parent holds the +/// same path and `same_segment(file, parent_file)` holds. +fn file_replacement_runs( + paths: &[&str], + parent_paths: &[&str], + same_segment: impl Fn(usize, usize) -> bool, +) -> Option> { + let mut replacements = Vec::new(); + let mut run: Option<(usize, usize)> = None; + let (mut file, mut parent_file) = (0, 0); + loop { + let (path, parent_path) = (paths.get(file), parent_paths.get(parent_file)); + if path.is_none() && parent_path.is_none() { + break; + } + if path == parent_path && same_segment(file, parent_file) { + if let Some((parent_start, start)) = run.take() { + replacements.push(SealedLexicalFileReplacementV1::new( + (parent_start, parent_file), + (start, file), + )?); + } + } else { + run.get_or_insert((parent_file, file)); + } + match (path, parent_path) { + (Some(path), Some(parent_path)) if path < parent_path => file += 1, + (Some(path), Some(parent_path)) if path > parent_path => parent_file += 1, + (Some(_), None) => file += 1, + (None, Some(_)) => parent_file += 1, + _ => { + file += 1; + parent_file += 1; + } + } + } + if let Some((parent_start, start)) = run { + replacements.push(SealedLexicalFileReplacementV1::new( + (parent_start, parent_file), + (start, file), + )?); + } + Some(replacements) +} + /// Final proof that all file ranges in one verified seal were exhausted. #[derive(Clone, Debug, PartialEq, Eq)] pub struct VerifiedSealedLexicalSourceReceiptV1 { @@ -1382,11 +1447,16 @@ impl VerifiedSealedLexicalPageSourceV1 { &self.source_state_digest } - /// File ordinals whose pages differ from `parent`'s, or `None` when the - /// two sources cannot share pages: a different file roster, or route - /// identities of a different width, which the serialized rows carry and - /// page cuts therefore follow. - pub fn changed_files_since(&self, parent: &Self) -> Option> { + /// The runs of files whose pages differ from `parent`'s, aligned by + /// path: a run replaces parent files with successor files, either side + /// possibly empty, so an added or removed file is a run too. `None` + /// when the two sources cannot share pages: route identities of a + /// different width, which the serialized rows carry and page cuts + /// therefore follow, or a roster that does not name each file once. + pub fn file_replacements_since( + &self, + parent: &Self, + ) -> Option> { if self.format_revision != parent.format_revision || self.maximum_page_chunks != parent.maximum_page_chunks || self.maximum_page_bytes != parent.maximum_page_bytes @@ -1397,7 +1467,31 @@ impl VerifiedSealedLexicalPageSourceV1 { { return None; } - self.file_source.changed_files_since(&parent.file_source) + let paths = self.file_paths()?; + let parent_paths = parent.file_paths()?; + file_replacement_runs(&paths, &parent_paths, |file, parent_file| { + self.file_source + .same_segment(file, &parent.file_source, parent_file) + }) + } + + /// Each file's logical path in file ordinal order, or `None` unless the + /// paths strictly ascend, which is the order a build visits files in. + fn file_paths(&self) -> Option> { + let files = &self.metadata.snapshot().files; + let paths = self + .file_source + .file_keys() + .map(|key| { + files + .get(usize::try_from(key).ok()?) + .map(|file| file.logical_path.as_str()) + }) + .collect::>>()?; + paths + .windows(2) + .all(|pair| pair[0] < pair[1]) + .then_some(paths) } /// Stage every page of one file as a build that reached it at diff --git a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs index d8c4c30a0b..a357dd0361 100644 --- a/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs +++ b/crates/tracedecay-code-index/src/production/lexical_page_source_tests.rs @@ -1067,3 +1067,69 @@ fn cancellation_during_staging_keeps_the_exact_pre_batch_cursor() { cursor_before, ); } + +/// Runs from [`file_replacement_runs`] as `(parent_files, files)` pairs, +/// where a file is carried when its path matches and it is not in `edited`. +fn replacement_runs( + paths: &[&str], + parent_paths: &[&str], + edited: &[&str], +) -> Vec<(std::ops::Range, std::ops::Range)> { + file_replacement_runs(paths, parent_paths, |file, _| { + !edited.contains(&paths[file]) + }) + .expect("ordinals fit") + .into_iter() + .map(|run| (run.parent_files, run.files)) + .collect() +} + +#[test] +fn file_replacement_runs_align_rosters_by_path() { + let parent = ["b", "d", "f"]; + assert_eq!(replacement_runs(&parent, &parent, &[]), vec![]); + assert_eq!( + replacement_runs(&parent, &parent, &["d"]), + vec![(1..2, 1..2)] + ); + assert_eq!( + replacement_runs(&["a", "b", "d", "f"], &parent, &[]), + vec![(0..0, 0..1)], + "a file added first" + ); + assert_eq!( + replacement_runs(&["b", "d", "f", "g"], &parent, &[]), + vec![(3..3, 3..4)], + "a file added last" + ); + assert_eq!( + replacement_runs(&["d", "f"], &parent, &[]), + vec![(0..1, 0..0)], + "the first file removed" + ); + assert_eq!( + replacement_runs(&["b", "d"], &parent, &[]), + vec![(2..3, 2..2)], + "the last file removed" + ); + assert_eq!( + replacement_runs(&["b", "c", "f"], &parent, &[]), + vec![(1..2, 1..2)], + "a removal beside an addition is one run" + ); + assert_eq!( + replacement_runs(&["a", "b", "e", "f"], &parent, &["f"]), + vec![(0..0, 0..1), (1..3, 2..4)], + "an edit after a replacement extends that run" + ); + assert_eq!( + replacement_runs(&["x", "y"], &parent, &[]), + vec![(0..3, 0..2)], + "every file replaced" + ); + assert_eq!( + replacement_runs(&[], &parent, &[]), + vec![(0..3, 0..0)], + "every file removed" + ); +} diff --git a/crates/tracedecay-code-index/src/production/mod.rs b/crates/tracedecay-code-index/src/production/mod.rs index c88824dec6..e4948475ad 100644 --- a/crates/tracedecay-code-index/src/production/mod.rs +++ b/crates/tracedecay-code-index/src/production/mod.rs @@ -89,12 +89,12 @@ pub use generation_statistics::CodeIndexGenerationStatisticsV1; mod lexical_page_source; pub use lexical_page_source::VerifiedSealedLexicalCursorRestoreErrorV1; pub use lexical_page_source::{ - VerifiedSealedLexicalCursorV1, VerifiedSealedLexicalPageBatchBoundsV1, - VerifiedSealedLexicalPageBatchReadV1, VerifiedSealedLexicalPageReadV1, - VerifiedSealedLexicalPageSourceV1, VerifiedSealedLexicalPageV1, - VerifiedSealedLexicalSourceReceiptV1, VerifiedSealedLexicalSymbolDisplayV1, - VerifiedSealedTextGenerationMetadataV1, advance_import_dictionary_digest, - initial_import_dictionary_digest, + SealedLexicalFileReplacementV1, VerifiedSealedLexicalCursorV1, + VerifiedSealedLexicalPageBatchBoundsV1, VerifiedSealedLexicalPageBatchReadV1, + VerifiedSealedLexicalPageReadV1, VerifiedSealedLexicalPageSourceV1, + VerifiedSealedLexicalPageV1, VerifiedSealedLexicalSourceReceiptV1, + VerifiedSealedLexicalSymbolDisplayV1, VerifiedSealedTextGenerationMetadataV1, + advance_import_dictionary_digest, initial_import_dictionary_digest, }; mod decoded_content; pub use decoded_content::{DecodedGenerationContentV1, SharedDecodedContentPoolV1}; diff --git a/crates/tracedecay-code-index/src/production/partitioned_codec.rs b/crates/tracedecay-code-index/src/production/partitioned_codec.rs index baa7577c4e..7dc8c920d0 100644 --- a/crates/tracedecay-code-index/src/production/partitioned_codec.rs +++ b/crates/tracedecay-code-index/src/production/partitioned_codec.rs @@ -1918,27 +1918,32 @@ impl PartitionedLexicalFileSourceV1 { } } - /// Ordinals of the files whose segment differs from `parent`'s at the same - /// position, or `None` when the two rosters name different files. - pub(super) fn changed_files_since(&self, parent: &Self) -> Option> { - if self.descriptors.len() != parent.descriptors.len() { - return None; - } - let mut changed = Vec::new(); - for (ordinal, (child, parent)) in - self.descriptors.iter().zip(&parent.descriptors).enumerate() - { - if child.file_key != parent.file_key { - return None; - } - if child.segment_digest != parent.segment_digest - || child.file_occurrence_id != parent.file_occurrence_id - || child.symbol_identities_digest != parent.symbol_identities_digest - { - changed.push(u64::try_from(ordinal).ok()?); + /// Each file's snapshot key, in file ordinal order. + pub(super) fn file_keys(&self) -> impl Iterator + '_ { + self.descriptors + .iter() + .map(|descriptor| descriptor.file_key) + } + + /// Whether file `ordinal` seals the same segment as `parent`'s file + /// `parent_ordinal`, so both emit the same pages. + pub(super) fn same_segment( + &self, + ordinal: usize, + parent: &Self, + parent_ordinal: usize, + ) -> bool { + match ( + self.descriptors.get(ordinal), + parent.descriptors.get(parent_ordinal), + ) { + (Some(child), Some(parent)) => { + child.segment_digest == parent.segment_digest + && child.file_occurrence_id == parent.file_occurrence_id + && child.symbol_identities_digest == parent.symbol_identities_digest } + _ => false, } - Some(changed) } pub(super) fn lexical_byte_offsets(&self) -> Result, CodeIndexProductionErrorV1> { diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs index d8d994fa8e..85dcbd8dd7 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/builder.rs @@ -18,10 +18,10 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tracedecay_code_index::chunks::ExtractionAdmittedCodeSearchChunkV1; use tracedecay_code_index::production::{ - CodeIndexExecutionControlV1, UninterruptibleCodeIndexControlV1, VerifiedSealedLexicalCursorV1, - VerifiedSealedLexicalPageV1, VerifiedSealedLexicalSourceReceiptV1, - VerifiedSealedLexicalSymbolDisplayV1, advance_import_dictionary_digest, - initial_import_dictionary_digest, + CodeIndexExecutionControlV1, SealedLexicalFileReplacementV1, UninterruptibleCodeIndexControlV1, + VerifiedSealedLexicalCursorV1, VerifiedSealedLexicalPageV1, + VerifiedSealedLexicalSourceReceiptV1, VerifiedSealedLexicalSymbolDisplayV1, + advance_import_dictionary_digest, initial_import_dictionary_digest, }; use tracedecay_domain::{ CodeGenerationId, CodeSearchChunkAnchorV1, CodeSearchChunkV1, ExactTechnicalTermV1, @@ -1037,7 +1037,7 @@ impl CodeLexicalArtifactBuilderV1 { /// Stage a successor over a byte copy of the sealed `parent`, whose bytes /// must hash to `parent_digest` and number `parent_size_bytes`, re-encoding - /// only `changed_files` (ascending file ordinals) through + /// only the successor files of `replacements` (ascending runs) through /// `stage_file_pages`. The result stands where a cold build of the same /// generation stands on entering digest verification, so /// `advance_started_finalization` seals the bytes a cold build seals. @@ -1058,7 +1058,7 @@ impl CodeLexicalArtifactBuilderV1 { memory_budget_bytes: usize, source_state_digest: ManifestDigest, source_format_revision: u32, - changed_files: &[u64], + replacements: &[SealedLexicalFileReplacementV1], stage_file_pages: &mut CarriedFilePagesV1<'_>, control: &dyn CodeIndexExecutionControlV1, ) -> Result { @@ -1100,7 +1100,7 @@ impl CodeLexicalArtifactBuilderV1 { let carried = carry_parent_rows( &transaction, &metadata, - changed_files, + replacements, stage_file_pages, control, )?; diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment.rs index 70a82f671a..fc1b39374a 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment.rs @@ -1,22 +1,24 @@ //! Carry a sealed parent artifact into a successor that re-encodes only the -//! pages of the files that changed. +//! pages of the files that changed, were added, or were removed. //! //! Pages never span files and their stored receipts are position-free, so //! every unchanged file's rows, postings, receipts, and clone rows are -//! already the rows a cold build of the successor writes, at document, -//! page, and occurrence positions shifted by the changed files' growth. -//! The carry replaces each changed file's pages in place at the cold -//! position, shifts what follows, and leaves the staging file exactly where -//! a cold build stands when it enters digest verification. The digests, -//! layout rewrite, and receipt are then the cold build's own steps, which -//! is what makes the sealed bytes equal. +//! already the rows a cold build of the successor writes, at file, +//! document, page, and occurrence positions shifted by the replaced files' +//! growth. The carry replaces each run of replaced files' pages in place at +//! the cold position, shifts what follows, and leaves the staging file +//! exactly where a cold build stands when it enters digest verification. +//! The digests, layout rewrite, and receipt are then the cold build's own +//! steps, which is what makes the sealed bytes equal. use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use rusqlite::{OptionalExtension, Transaction, params}; -use tracedecay_code_index::production::{CodeIndexExecutionControlV1, VerifiedSealedLexicalPageV1}; +use tracedecay_code_index::production::{ + CodeIndexExecutionControlV1, SealedLexicalFileReplacementV1, VerifiedSealedLexicalPageV1, +}; use tracedecay_domain::CodeGenerationId; use super::builder::page_transient_peak_bytes; @@ -66,6 +68,22 @@ struct PageRowV1 { } impl PageRowV1 { + fn shifted_by(&self, file_growth: i64) -> Result { + let file_ordinal = self + .file_ordinal + .checked_add(file_growth) + .filter(|ordinal| *ordinal >= 0) + .ok_or_else(|| { + CodeLexicalArtifactErrorV1::Corrupt( + "carried lexical file ordinal underflowed".to_owned(), + ) + })?; + Ok(Self { + file_ordinal, + ..self.clone() + }) + } + fn of_prepared( page: &PreparedCodeLexicalArtifactPageV1, ) -> Result { @@ -84,11 +102,13 @@ impl PageRowV1 { } } -/// One changed file: the parent positions it held and the pages it now has. -struct ChangedFileV1 { +/// One run of replaced files: the parent positions its files held, the +/// pages its successor files have, and how many files it adds net. +struct ReplacedFilesV1 { old_pages: (usize, usize), old_documents: (u64, u64), old_clones: (u64, u64), + file_growth: i64, pages: Vec, } @@ -259,21 +279,21 @@ impl RowDictionaryV1 for RecordingRowDictionaryV1<'_> { } } -/// Replace the changed files' pages of the sealed parent copy `transaction` +/// Replace the replaced files' pages of the sealed parent copy `transaction` /// holds with the successor's, in place, and shift everything after them. #[tracing::instrument(name = "query.artifact.carry.patch", level = "trace", skip_all)] pub(super) fn carry_parent_rows( transaction: &Transaction<'_>, metadata: &CodeLexicalProjectionMetadataV1, - changed_files: &[u64], + replacements: &[SealedLexicalFileReplacementV1], stage_file_pages: &mut CarriedFilePagesV1<'_>, control: &dyn CodeIndexExecutionControlV1, ) -> Result { let parent_pages = read_page_rows(transaction, control)?; - let changed = stage_changed_files( + let changed = stage_replaced_files( metadata, &parent_pages, - changed_files, + replacements, stage_file_pages, control, )?; @@ -384,15 +404,15 @@ fn read_page_rows( Ok(pages) } -/// Locate each changed file's parent pages and stage its successor pages at -/// the position a cold build reaches it. -fn stage_changed_files( +/// Locate each run's parent pages and stage its successor files' pages at +/// the position a cold build reaches them. +fn stage_replaced_files( metadata: &CodeLexicalProjectionMetadataV1, parent_pages: &[PageRowV1], - changed_files: &[u64], + replacements: &[SealedLexicalFileReplacementV1], stage_file_pages: &mut CarriedFilePagesV1<'_>, control: &dyn CodeIndexExecutionControlV1, -) -> Result, CodeLexicalArtifactErrorV1> { +) -> Result, CodeLexicalArtifactErrorV1> { let mut document_starts = Vec::with_capacity(parent_pages.len() + 1); let mut clone_starts = Vec::with_capacity(parent_pages.len() + 1); let (mut documents, mut clones) = (0u64, 0u64); @@ -406,42 +426,59 @@ fn stage_changed_files( clone_starts.push(clones); let mut page_growth = 0i64; let mut document_growth = 0i64; - let mut previous = None; - let mut changed = Vec::with_capacity(changed_files.len()); - for &file in changed_files { + let mut file_growth = 0i64; + let mut previous_end = 0u64; + let mut changed = Vec::with_capacity(replacements.len()); + for replacement in replacements { checkpoint(control)?; - if previous.is_some_and(|previous| previous >= file) { + let (parent_files, files) = (&replacement.parent_files, &replacement.files); + if parent_files.start < previous_end + || parent_files.end < parent_files.start + || files.end < files.start + || shifted(parent_files.start, file_growth)? != files.start + { return Err(CodeLexicalArtifactErrorV1::Contract( - "carried lexical changed files must ascend".to_owned(), + "carried lexical file replacements must ascend in step".to_owned(), )); } - previous = Some(file); - let file_ordinal = i64::try_from(file).map_err(contract_number)?; - let start = parent_pages.partition_point(|page| page.file_ordinal < file_ordinal); - let end = parent_pages.partition_point(|page| page.file_ordinal <= file_ordinal); - let first_page = shifted(start as u64, page_growth)?; - let first_chunk = shifted(document_starts[start], document_growth)?; - let staged = stage_file_pages(file, first_page, first_chunk)?; - let mut pages = Vec::with_capacity(staged.len()); - for (offset, page) in staged.iter().enumerate() { - if page.file_ordinal() != file || page.page_ordinal() != first_page + offset as u64 { - return Err(CodeLexicalArtifactErrorV1::Contract( - "carried lexical file pages are not that file's contiguous pages".to_owned(), - )); + previous_end = parent_files.end; + let first_file = i64::try_from(parent_files.start).map_err(contract_number)?; + let end_file = i64::try_from(parent_files.end).map_err(contract_number)?; + let start = parent_pages.partition_point(|page| page.file_ordinal < first_file); + let end = parent_pages.partition_point(|page| page.file_ordinal < end_file); + let mut next_page = shifted(start as u64, page_growth)?; + let mut next_chunk = shifted(document_starts[start], document_growth)?; + let mut pages = Vec::new(); + for file in files.clone() { + let staged = stage_file_pages(file, next_page, next_chunk)?; + for (offset, page) in staged.iter().enumerate() { + if page.file_ordinal() != file || page.page_ordinal() != next_page + offset as u64 { + return Err(CodeLexicalArtifactErrorV1::Contract( + "carried lexical file pages are not that file's contiguous pages" + .to_owned(), + )); + } + let scratch = page_transient_peak_bytes(metadata, page, usize::MAX)?; + let prepared = prepare_page(metadata, page, None, scratch, control)?; + next_chunk += prepared.chunk_count; + pages.push(prepared); } - let scratch = page_transient_peak_bytes(metadata, page, usize::MAX)?; - pages.push(prepare_page(metadata, page, None, scratch, control)?); + next_page += staged.len() as u64; } - page_growth += staged.len() as i64 - (end - start) as i64; + page_growth += pages.len() as i64 - (end - start) as i64; document_growth += pages .iter() .map(|page| page.chunk_count as i64) .sum::() - (document_starts[end] - document_starts[start]) as i64; - changed.push(ChangedFileV1 { + let run_growth = i64::try_from(files.end - files.start).map_err(contract_number)? + - i64::try_from(parent_files.end - parent_files.start).map_err(contract_number)?; + file_growth += run_growth; + changed.push(ReplacedFilesV1 { old_pages: (start, end), old_documents: (document_starts[start], document_starts[end]), old_clones: (clone_starts[start], clone_starts[end]), + file_growth: run_growth, pages, }); } @@ -465,7 +502,7 @@ fn carry_row_dictionary( transaction: &Transaction<'_>, metadata: &CodeLexicalProjectionMetadataV1, parent_pages: &[PageRowV1], - changed: &[ChangedFileV1], + changed: &[ReplacedFilesV1], control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { let mut net = BTreeMap::::new(); @@ -738,18 +775,24 @@ fn apply_field_totals( fn write_page_rows( transaction: &Transaction<'_>, parent_pages: &[PageRowV1], - changed: &[ChangedFileV1], + changed: &[ReplacedFilesV1], ) -> Result { let mut rows = Vec::with_capacity(parent_pages.len()); let mut next_parent = 0usize; - for file in changed { - rows.extend_from_slice(&parent_pages[next_parent..file.old_pages.0]); - for page in &file.pages { + let mut file_growth = 0i64; + for run in changed { + for row in &parent_pages[next_parent..run.old_pages.0] { + rows.push(row.shifted_by(file_growth)?); + } + for page in &run.pages { rows.push(PageRowV1::of_prepared(page)?); } - next_parent = file.old_pages.1; + file_growth += run.file_growth; + next_parent = run.old_pages.1; + } + for row in &parent_pages[next_parent..] { + rows.push(row.shifted_by(file_growth)?); } - rows.extend_from_slice(&parent_pages[next_parent..]); transaction .execute("DELETE FROM source_pages", []) .map_err(sqlite_error)?; diff --git a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment/clones.rs b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment/clones.rs index 44a6a1ac93..c1e4d0861e 100644 --- a/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment/clones.rs +++ b/crates/tracedecay-query/src/retrieval/lexical/projection/artifact/increment/clones.rs @@ -1,4 +1,4 @@ -//! Carry the parent's clone rows: replace the changed files' occurrences in +//! Carry the parent's clone rows: replace the replaced files' occurrences in //! place and renumber payloads by first use, as a cold build's insert order //! numbers them. @@ -17,7 +17,7 @@ use super::super::prepared::PreparedCloneBodyV1; use super::super::row_codec::stored_symbol_key; use super::super::{CodeLexicalArtifactErrorV1, checkpoint, sqlite_error}; use super::postings::{scan_window, transform_window}; -use super::{ChangedFileV1, ShiftV1, shift_rowid_keys, shifted}; +use super::{ReplacedFilesV1, ShiftV1, shift_rowid_keys, shifted}; /// One successor clone body with the occurrence ordinal it takes. struct CarriedCloneV1<'a> { @@ -25,13 +25,13 @@ struct CarriedCloneV1<'a> { body: &'a PreparedCloneBodyV1, } -/// Replace the changed files' clone occurrences in place, then renumber the +/// Replace the replaced files' clone occurrences in place, then renumber the /// payloads by first use over the whole occurrence sequence, which is how a /// cold build's insert order numbers them. pub(super) fn carry_clone_rows( transaction: &Transaction<'_>, clones: &ShiftV1, - changed: &[ChangedFileV1], + changed: &[ReplacedFilesV1], control: &dyn CodeIndexExecutionControlV1, ) -> Result<(), CodeLexicalArtifactErrorV1> { let mut bodies = Vec::new(); @@ -145,14 +145,14 @@ pub(super) fn carry_clone_rows( renumber_clone_payloads(transaction, &parent_payloads, &successor_payloads, control) } -/// Where a changed file's first successor occurrence lands: its parent -/// start moved by the growth of the changed files before it. `index` pairs -/// the file with its replaced range, since two files without clone bodies -/// can replace the same empty range. +/// Where a run's first successor occurrence lands: its parent start moved +/// by the growth of the runs before it. `index` pairs the run with its +/// replaced range, since two runs without clone bodies can replace the same +/// empty range. fn clone_range_image( clones: &ShiftV1, index: usize, - file: &ChangedFileV1, + file: &ReplacedFilesV1, ) -> Result { let start = file.old_clones.0 + 1; if clones.replaced.get(index) != Some(&(start, file.old_clones.1 + 1)) {