diff --git a/CHANGELOG.md b/CHANGELOG.md index d6342494a..5d140aad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,19 @@ released versions carry their date on the heading. ### Features +- 2026-10-05: **The newer backup decides when a message changed between two + backups of one phone.** Every message file now says when its backup was + made: an iPhone backup's own date, the date an SMS Backup & Restore file + records, an iMazing export's date, or when the backup's files were last + written for a source that records none. Importing two backups of the same + phone, in one import or two and in either order, gives each message the + newer backup's text, earlier versions and Deleted in the source app or + Unsent mark, so a message recovered after it was deleted loses its mark, + and a message unsent after the older backup reads as unsent. An older + backup imported after a newer one changes nothing. Import details under + Settings → Storage show the backup each import read and when it was made. + Message files exported before they said when their backup was made are + refused, and the backup must be exported again with this build. - 2026-10-05: **A WhatsApp reply now names the message it quotes.** When the quoted message is in the same chat of the same backup, the reply is linked to it, as Apple Messages replies already were: a mail export threads @@ -164,6 +177,10 @@ released versions carry their date on the heading. - If you have a program that reads messages from the HTTP API, a message's `timestamp` and an earlier version's `edited_at` now carry three digits of milliseconds, such as `2015-03-12T18:04:22.250Z`, where they had none. +- Message files exported before they said when their backup was made are + refused when you import or convert them. Export the backup again with + this build. A program that reads the HTTP API finds the backup's date in + a message's `backup_taken_at` and an Import Run's `backup_taken_at`. ## [0.10.1] - 2026-10-05 diff --git a/Cargo.lock b/Cargo.lock index 94388b61c..71b2f2dde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1920,6 +1920,7 @@ dependencies = [ "message-csv", "message-ir", "obfuscate", + "serde_json", "sha2 0.11.0", "tempfile", "thiserror 2.0.21", diff --git a/crates/core/message-crate-core/Cargo.toml b/crates/core/message-crate-core/Cargo.toml index e69ea83f4..b3096ad55 100644 --- a/crates/core/message-crate-core/Cargo.toml +++ b/crates/core/message-crate-core/Cargo.toml @@ -12,16 +12,19 @@ message-csv = { path = "../../libs/csv", optional = true } message-ir = { path = "../../libs/ir" } media = { path = "../../libs/media" } obfuscate = { path = "../../libs/obfuscate" } +serde_json = { workspace = true, optional = true } sha2 = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } [features] -# The CSV reader is only for `testutil`, which reads exports back in tests. -testutil = ["dep:message-csv"] +# The CSV and JSON readers are only for `testutil`, which reads exports back +# in tests. +testutil = ["dep:message-csv", "dep:serde_json"] [dev-dependencies] message-csv = { path = "../../libs/csv" } +serde_json = { workspace = true } media = { path = "../../libs/media", features = ["testutil"] } [lints] diff --git a/crates/core/message-crate-core/src/attachment_jobs/tests.rs b/crates/core/message-crate-core/src/attachment_jobs/tests.rs index c10625fd6..67b69de56 100644 --- a/crates/core/message-crate-core/src/attachment_jobs/tests.rs +++ b/crates/core/message-crate-core/src/attachment_jobs/tests.rs @@ -468,6 +468,7 @@ fn staging_a_conversation_writes_the_files_counts_them_and_frees_the_bytes() { tool_version: "0.1.0".into(), owner_identity: Some("+15555550100".into()), owner_display_name: None, + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "+15555550101".into(), @@ -710,6 +711,7 @@ fn staging_frees_the_bytes_the_documents_were_carrying() { tool_version: "0.1.0".into(), owner_identity: None, owner_display_name: None, + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "+15555550101".into(), diff --git a/crates/core/message-crate-core/src/lib.rs b/crates/core/message-crate-core/src/lib.rs index 5d2f5fef9..a79615111 100644 --- a/crates/core/message-crate-core/src/lib.rs +++ b/crates/core/message-crate-core/src/lib.rs @@ -41,8 +41,8 @@ pub use exporters::{ }; pub use pipeline::{ CSV_NOT_READ, ExportReport, IssueSink, NAME_ONLY_CHAT_NOTE, NOTE, RunIssue, RunResult, - discover_files, emit_issue, export_meta, prepare_outputs, project_conversation, - unreadable_parts_note, + discover_files, emit_issue, export_meta, file_modified_unix_ms, newest_file_modified_unix_ms, + prepare_outputs, project_conversation, unreadable_parts_note, }; pub use process::{ CancelFlag, Cancelled, LogSink, check_cancel, emit_log, is_cancelled, parallel_for_each, diff --git a/crates/core/message-crate-core/src/pipeline.rs b/crates/core/message-crate-core/src/pipeline.rs index c7c3fa343..3ac98d84b 100644 --- a/crates/core/message-crate-core/src/pipeline.rs +++ b/crates/core/message-crate-core/src/pipeline.rs @@ -467,13 +467,15 @@ pub fn prepare_outputs( Ok((resolved, output)) } -/// Standard export metadata: source / tool / version plus the owner identity. +/// Standard export metadata: source / tool / version, the owner identity, +/// and when the backup was made. pub fn export_meta( source: &str, tool: &str, tool_version: &str, owner_identity: Option, owner_display_name: Option, + backup_taken_at_unix_ms: Option, ) -> message_ir::ExportMeta { message_ir::ExportMeta { source: source.to_string(), @@ -481,13 +483,59 @@ pub fn export_meta( tool_version: tool_version.to_string(), owner_identity, owner_display_name, + backup_taken_at_unix_ms, } } +/// When a file was last changed, in Unix milliseconds, or `None` when it +/// cannot be read or its time is before 1970. The backup date of a source +/// whose files record none of their own: the file was written when the +/// backup was made, or copied with its time kept. +pub fn file_modified_unix_ms(path: &std::path::Path) -> Option { + let modified = fs::metadata(path).ok()?.modified().ok()?; + let since = modified.duration_since(std::time::UNIX_EPOCH).ok()?; + i64::try_from(since.as_millis()).ok() +} + +/// The latest [`file_modified_unix_ms`] of `paths`, or `None` when none has +/// one: the backup date of a source read from several files, which is as +/// new as the newest of them. +pub fn newest_file_modified_unix_ms<'a>( + paths: impl IntoIterator, +) -> Option { + paths.into_iter().filter_map(file_modified_unix_ms).max() +} + #[cfg(test)] mod tests { use super::*; + /// A source with no date of its own is as new as its newest file, and + /// a file that is not there adds nothing. + #[test] + fn a_backup_read_from_files_is_as_new_as_the_newest() { + use crate::testutil::{TEST_BACKUP_TAKEN_AT_UNIX_MS, set_modified_unix_ms}; + let dir = tempfile::tempdir().unwrap(); + let older = dir.path().join("older.csv"); + let newer = dir.path().join("newer.csv"); + fs::write(&older, "a").unwrap(); + fs::write(&newer, "b").unwrap(); + set_modified_unix_ms(&older, TEST_BACKUP_TAKEN_AT_UNIX_MS - 86_400_000); + set_modified_unix_ms(&newer, TEST_BACKUP_TAKEN_AT_UNIX_MS); + + assert_eq!( + file_modified_unix_ms(&older), + Some(TEST_BACKUP_TAKEN_AT_UNIX_MS - 86_400_000) + ); + let missing = dir.path().join("missing.csv"); + assert_eq!(file_modified_unix_ms(&missing), None); + assert_eq!( + newest_file_modified_unix_ms([older.as_path(), newer.as_path(), missing.as_path()]), + Some(TEST_BACKUP_TAKEN_AT_UNIX_MS) + ); + assert_eq!(newest_file_modified_unix_ms([missing.as_path()]), None); + } + #[test] fn the_not_sms_or_mms_line_names_the_format_and_the_count() { let mut report = ExportReport::default(); diff --git a/crates/core/message-crate-core/src/testutil.rs b/crates/core/message-crate-core/src/testutil.rs index 65d9fb0ae..bdcc23038 100644 --- a/crates/core/message-crate-core/src/testutil.rs +++ b/crates/core/message-crate-core/src/testutil.rs @@ -370,3 +370,44 @@ pub fn assert_run_wrote_jsonl( .collect::>() .join("\n") } + +/// The backup date the exporters' tests give a source: 2026-09-30T18:45:12Z, +/// in Unix milliseconds. A date of the test's own, so a test that reads it +/// back cannot pass by reading the time it ran at. +pub const TEST_BACKUP_TAKEN_AT_UNIX_MS: i64 = 1_790_793_912_000; + +/// Set `path`'s modification time to `unix_ms`, the way a backup file keeps +/// the time it was written. +/// +/// # Panics +/// +/// Panics when the file cannot be opened or its time cannot be set. +pub fn set_modified_unix_ms(path: &Path, unix_ms: i64) { + let at = std::time::UNIX_EPOCH + + std::time::Duration::from_millis(u64::try_from(unix_ms).expect("a time after 1970")); + fs::File::options() + .write(true) + .open(path) + .expect("open the file to date") + .set_modified(at) + .expect("set the file's modification time"); +} + +/// Every JSON Lines file's `export.backup_taken_at_unix_ms` under `dir`, in +/// file name order. +/// +/// # Panics +/// +/// Panics when a file cannot be read or its first line is not JSON. +pub fn jsonl_backup_dates(dir: &Path) -> Vec> { + jsonl_names(dir) + .iter() + .map(|name| { + let text = fs::read_to_string(dir.join(name)).expect("read jsonl"); + let header: serde_json::Value = + serde_json::from_str(text.lines().next().expect("a header line")) + .expect("the header is JSON"); + header["export"]["backup_taken_at_unix_ms"].as_i64() + }) + .collect() +} diff --git a/crates/exporters/go-sms-pro-exporter/src/emit.rs b/crates/exporters/go-sms-pro-exporter/src/emit.rs index a8951ba04..c674ae0b3 100644 --- a/crates/exporters/go-sms-pro-exporter/src/emit.rs +++ b/crates/exporters/go-sms-pro-exporter/src/emit.rs @@ -455,14 +455,21 @@ pub(crate) fn convert_export(args: ConvertExportArgs<'_>) -> Result) -> Result) -> Result) -> Result (record.sender_identity, record.sender_display_name), }; @@ -678,6 +679,7 @@ fn write_conversations( options.emit_log(""); options.emit_log(message_crate_core::CONVERSATION_FILES_PREPARING.line(total as u64)); options.emit_progress(ProgressEvent::Prepare { done: 0, total }); + let backup_taken_at_unix_ms = options.backup_taken_at_unix_ms(); let mut written = 0usize; let mut kept = 0u64; for (chat_identifier, convo) in conversations { @@ -687,7 +689,12 @@ fn write_conversations( continue; } kept += 1; - let doc = pending_to_document(chat_identifier, convo, options.use_caller_id); + let doc = pending_to_document( + chat_identifier, + convo, + options.use_caller_id, + backup_taken_at_unix_ms, + ); let document_id = doc.conversation.chat_identifier.clone(); sink.write_document(doc) .map_err(|e| anyhow!("write {} for {}: {e:#}", format.as_str(), document_id))?; @@ -702,11 +709,14 @@ fn write_conversations( Ok(kept) } -/// Project one accumulated conversation into the shared document shape. +/// Project one accumulated conversation into the shared document shape, +/// read from a backup made at `backup_taken_at_unix_ms` +/// ([`ExportOptions::backup_taken_at_unix_ms`]). fn pending_to_document( chat_identifier: String, convo: PendingConversation, use_caller_id: bool, + backup_taken_at_unix_ms: Option, ) -> ConversationDocument { let export = ExportMeta { source: EXPORT_SOURCE.into(), @@ -716,6 +726,7 @@ fn pending_to_document( owner_display_name: convo .owner_display_name .or_else(|| use_caller_id.then(|| "Me".to_string())), + backup_taken_at_unix_ms, }; // Each message keeps the address it was sent from; the conversation's // owner fills in only where the database recorded none. @@ -751,9 +762,15 @@ fn pending_to_unit( chat_identifier: String, mut convo: PendingConversation, use_caller_id: bool, + backup_taken_at_unix_ms: Option, ) -> ConversationUnit { let loads = std::mem::take(&mut convo.attachment_loads); - let doc = pending_to_document(chat_identifier, convo, use_caller_id); + let doc = pending_to_document( + chat_identifier, + convo, + use_caller_id, + backup_taken_at_unix_ms, + ); let mut loads = loads.into_iter(); ConversationUnit::from_doc(doc, |_, att| attachment_source(loads.next(), att)) } @@ -777,11 +794,19 @@ fn drain_conversations( not_decrypted: &mut NotDecrypted, ) -> Result { let use_caller_id = options.use_caller_id; + let backup_taken_at_unix_ms = options.backup_taken_at_unix_ms(); let units: Vec = collected .conversations .into_iter() .filter(|(_, convo)| !convo.messages.is_empty()) - .map(|(chat_identifier, convo)| pending_to_unit(chat_identifier, convo, use_caller_id)) + .map(|(chat_identifier, convo)| { + pending_to_unit( + chat_identifier, + convo, + use_caller_id, + backup_taken_at_unix_ms, + ) + }) .collect(); let queue = WriteQueueOptions { @@ -1214,7 +1239,7 @@ mod tests { attachment_loads: Vec::new(), }; - let doc = pending_to_document("+15555550122".into(), convo, false); + let doc = pending_to_document("+15555550122".into(), convo, false, None); let senders: Vec<_> = doc .messages @@ -1345,7 +1370,7 @@ mod tests { ], }; - let unit = pending_to_unit("+15555550101".into(), convo, false); + let unit = pending_to_unit("+15555550101".into(), convo, false, None); assert_eq!(unit.attachments.len(), 2); assert_eq!(unit.attachments[0].message_index, 0); diff --git a/crates/exporters/imessage-ir-exporter/src/run.rs b/crates/exporters/imessage-ir-exporter/src/run.rs index 2cbb26750..75cfa4a03 100644 --- a/crates/exporters/imessage-ir-exporter/src/run.rs +++ b/crates/exporters/imessage-ir-exporter/src/run.rs @@ -134,6 +134,36 @@ impl ExportOptions { pub fn check_cancel(&self) -> Result<()> { message_crate_core::check_cancel(self.cancel.as_ref()).map_err(|e| anyhow!(e)) } + + /// When the Messages data was backed up, in Unix milliseconds: an + /// iPhone backup's `Manifest.plist` date, or the last time Messages + /// wrote a Mac's `chat.db`: the newest modification time of `chat.db` + /// and its `-wal` and `-shm` files, because Messages keeps the database + /// in write-ahead mode and new rows reach `chat.db` itself only at a + /// checkpoint. `None` when neither can be read. + pub fn backup_taken_at_unix_ms(&self) -> Option { + backup_taken_at_unix_ms(&self.source) + } +} + +/// [`ExportOptions::backup_taken_at_unix_ms`] for `source`. +pub(crate) fn backup_taken_at_unix_ms(source: &Source) -> Option { + match source.platform { + Platform::Ios => ios_backup::ios_backup_date_unix_ms(&source.db_path), + Platform::MacOs => { + let sidecar = |suffix: &str| { + let mut name = source.db_path.clone().into_os_string(); + name.push(suffix); + std::path::PathBuf::from(name) + }; + let (wal, shm) = (sidecar("-wal"), sidecar("-shm")); + message_crate_core::newest_file_modified_unix_ms([ + source.db_path.as_path(), + wal.as_path(), + shm.as_path(), + ]) + } + } } /// Build options from [`ExporterConfig`], start the `imessage-reader` diff --git a/crates/exporters/imessage-ir-exporter/src/run/tests.rs b/crates/exporters/imessage-ir-exporter/src/run/tests.rs index db86cd746..a72649396 100644 --- a/crates/exporters/imessage-ir-exporter/src/run/tests.rs +++ b/crates/exporters/imessage-ir-exporter/src/run/tests.rs @@ -968,3 +968,57 @@ fn every_format_refuses_attachments_the_staging_disk_cannot_hold() { ); } } + +/// An iPhone backup is dated by its `Manifest.plist`, which is readable +/// whether or not the backup is encrypted. +#[test] +fn an_iphone_backup_is_dated_by_its_manifest() { + let backup = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/dated-backup"); + let source = Source { + db_path: backup, + platform: Platform::Ios, + backup_password: None, + }; + // 2026-09-30T18:45:12Z + assert_eq!(backup_taken_at_unix_ms(&source), Some(1_790_793_912_000)); +} + +/// A Mac's `chat.db` records no backup date, so it is dated by when +/// Messages last wrote it, to the database or to its write-ahead log. +#[test] +fn a_mac_chat_db_is_dated_by_its_modification_time() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("chat.db"); + fs::write(&db, b"not read here").unwrap(); + message_crate_core::testutil::set_modified_unix_ms( + &db, + message_crate_core::testutil::TEST_BACKUP_TAKEN_AT_UNIX_MS, + ); + let source = Source { + db_path: db, + platform: Platform::MacOs, + backup_password: None, + }; + assert_eq!( + backup_taken_at_unix_ms(&source), + Some(message_crate_core::testutil::TEST_BACKUP_TAKEN_AT_UNIX_MS) + ); + // Messages keeps chat.db in write-ahead mode: a row written since the + // last checkpoint moves only chat.db-wal's time. + let wal = dir.path().join("chat.db-wal"); + fs::write(&wal, b"not read here").unwrap(); + message_crate_core::testutil::set_modified_unix_ms( + &wal, + message_crate_core::testutil::TEST_BACKUP_TAKEN_AT_UNIX_MS + 3_600_000, + ); + assert_eq!( + backup_taken_at_unix_ms(&source), + Some(message_crate_core::testutil::TEST_BACKUP_TAKEN_AT_UNIX_MS + 3_600_000) + ); + let undated = Source { + db_path: dir.path().join("missing.db"), + platform: Platform::Ios, + backup_password: None, + }; + assert_eq!(backup_taken_at_unix_ms(&undated), None); +} diff --git a/crates/exporters/imessage-ir-exporter/tests/fixtures/dated-backup/Manifest.plist b/crates/exporters/imessage-ir-exporter/tests/fixtures/dated-backup/Manifest.plist new file mode 100644 index 000000000..64224a8e8 --- /dev/null +++ b/crates/exporters/imessage-ir-exporter/tests/fixtures/dated-backup/Manifest.plist @@ -0,0 +1,12 @@ + + + + + Date + 2026-09-30T18:45:12Z + IsEncrypted + + Version + 10.0 + + diff --git a/crates/exporters/imessage-ir-exporter/tests/helper_process.rs b/crates/exporters/imessage-ir-exporter/tests/helper_process.rs index 2a929e485..46dc3942c 100644 --- a/crates/exporters/imessage-ir-exporter/tests/helper_process.rs +++ b/crates/exporters/imessage-ir-exporter/tests/helper_process.rs @@ -84,6 +84,30 @@ fn exports_a_mac_chat_db_through_the_helper_process() { assert!(all.contains("\"attachments/"), "{all}"); } +/// Every conversation file of a Mac `chat.db` export says the database's +/// modification time as when the backup was made. +#[test] +fn every_conversation_file_says_when_the_chat_db_was_last_written() { + use message_crate_core::testutil::{ + TEST_BACKUP_TAKEN_AT_UNIX_MS, jsonl_backup_dates, set_modified_unix_ms, + }; + helper_binary(); + let dir = tempfile::tempdir().unwrap(); + let db_path = write_chat_db(dir.path()); + set_modified_unix_ms(&db_path, TEST_BACKUP_TAKEN_AT_UNIX_MS); + let output = dir.path().join("out"); + + imessage_ir_exporter::run(&config(&db_path, &output, None)).unwrap(); + let dates = jsonl_backup_dates(&output); + assert_eq!(dates.len(), 7, "{dates:?}"); + assert!( + dates + .iter() + .all(|d| *d == Some(TEST_BACKUP_TAKEN_AT_UNIX_MS)), + "{dates:?}" + ); +} + /// Every file below `dir`, recursively. fn walk(dir: &Path) -> Vec { let mut out = Vec::new(); diff --git a/crates/exporters/openextract-exporter/src/emit.rs b/crates/exporters/openextract-exporter/src/emit.rs index 2fdbdd380..f2ade380c 100644 --- a/crates/exporters/openextract-exporter/src/emit.rs +++ b/crates/exporters/openextract-exporter/src/emit.rs @@ -18,7 +18,7 @@ use phone::Handle; use serde_json::{Map, json}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, HashMap, HashSet}; -use std::path::Path; +use std::path::{Path, PathBuf}; const EXPORT_SOURCE: &str = "openextract"; const EXPORT_TOOL: &str = "OpenExtract"; @@ -69,9 +69,10 @@ pub(crate) fn convert_export(args: ConvertExportArgs<'_>) -> Result) -> Result>( .. } = ingest; + // SMS Backup+ writes no backup date of its own, so the backup is as new + // as the newest mail file read. let hooks = SbpProjection { export: message_crate_core::export_meta( EXPORT_SOURCE, @@ -528,6 +530,9 @@ pub(crate) fn convert_export>( EXPORT_TOOL_VERSION, Some(owner_identity), None, + message_crate_core::newest_file_modified_unix_ms( + eml_paths.iter().map(PathBuf::as_path), + ), ), }; let mut documents = Vec::new(); @@ -834,6 +839,7 @@ mod tests { tool_version: String::new(), owner_identity: None, owner_display_name: None, + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "test".into(), @@ -907,6 +913,7 @@ mod tests { EXPORT_TOOL_VERSION, Some("+15555550100".into()), None, + None, ), }; let mut report = ingest.report; diff --git a/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs b/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs index 8a07f13c8..bb7a7a7f7 100644 --- a/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs +++ b/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs @@ -386,3 +386,51 @@ fn a_run_that_copies_no_attachments_still_records_their_size() { assert_eq!(picture.path, None, "nothing was copied"); assert_eq!(picture.size_bytes, Some(22)); } + +/// SMS Backup+ records no backup date, so the conversation file says the +/// newest modification time of the mail files read. +#[test] +fn the_backup_date_is_the_newest_mail_files_modification_time() { + use message_crate_core::testutil::{ + TEST_BACKUP_TAKEN_AT_UNIX_MS, jsonl_backup_dates, set_modified_unix_ms, + }; + let tmp = tempfile::tempdir().expect("tempdir"); + let input = tmp.path().join("in"); + fs::create_dir_all(&input).unwrap(); + for (name, modified) in [ + ("flat_received.eml", TEST_BACKUP_TAKEN_AT_UNIX_MS), + ( + "flat_smssync_276_sam.eml", + TEST_BACKUP_TAKEN_AT_UNIX_MS - 60_000, + ), + ] { + let path = input.join(name); + fs::copy(fixtures().join(name), &path).unwrap(); + set_modified_unix_ms(&path, modified); + } + let output = tmp.path().join("out"); + let cache = tempfile::tempdir().unwrap(); + convert_export(ConvertExportArgs { + inputs: &[input.as_path()], + output_dir: &output, + scratch_dir: cache.path(), + owner_phones: &["+15555550100".into()], + owner_emails: &["owner@example.com".into()], + verbose: false, + transforms: ExportTransforms::none(), + output_format: OutputFormat::Jsonl, + cancel: None, + log: None, + issues: None, + resume: false, + }) + .expect("convert"); + let dates = jsonl_backup_dates(&output); + assert!(!dates.is_empty()); + assert!( + dates + .iter() + .all(|d| *d == Some(TEST_BACKUP_TAKEN_AT_UNIX_MS)), + "{dates:?}" + ); +} diff --git a/crates/exporters/sms-backup-restore-exporter/src/read.rs b/crates/exporters/sms-backup-restore-exporter/src/read.rs index 2aec79878..9c4f8c715 100644 --- a/crates/exporters/sms-backup-restore-exporter/src/read.rs +++ b/crates/exporters/sms-backup-restore-exporter/src/read.rs @@ -535,6 +535,7 @@ fn to_document( id: &str, conversation: &PendingConversation, owner_identity: Option<&str>, + backup_taken_at_unix_ms: Option, report: &mut ReadReport, ) -> ConversationDocument { let export = ExportMeta { @@ -543,6 +544,7 @@ fn to_document( tool_version: EXPORT_TOOL_VERSION.into(), owner_identity: owner_identity.map(str::to_string), owner_display_name: None, + backup_taken_at_unix_ms, }; let owner = owner_sender(&export); let messages = conversation @@ -747,6 +749,9 @@ pub fn read_backup( .and_then(OwnerHandleSet::primary_owner_handle); let mut report = ReadReport::default(); let mut conversations = BTreeMap::new(); + // When each file's backup was made: its `backup_date`, or the file's + // modification time when it has none. + let mut backup_dates: HashMap, i64> = HashMap::new(); for path in paths { check_cancel(options.cancel)?; let file: Arc = path.display().to_string().into(); @@ -779,6 +784,12 @@ pub fn read_backup( } }); merge_stats(&mut report, stats); + if let Some(date) = stats + .backup_date_unix_ms + .or_else(|| message_crate_core::file_modified_unix_ms(&path)) + { + backup_dates.insert(file.clone(), date); + } if let Some(error) = spool_error { return Err(error); } @@ -803,10 +814,17 @@ pub fn read_backup( if conversation.messages.is_empty() { continue; } + // A conversation read from two backups is as new as the newer one. + let backup_taken_at_unix_ms = conversation + .messages + .iter() + .filter_map(|message| backup_dates.get(&message.file).copied()) + .max(); documents.push(to_document( &id, &conversation, owner_identity.as_deref(), + backup_taken_at_unix_ms, &mut report, )); report.conversations += 1; diff --git a/crates/exporters/sms-backup-restore-exporter/src/read/tests.rs b/crates/exporters/sms-backup-restore-exporter/src/read/tests.rs index 2a14e7438..aea1ea767 100644 --- a/crates/exporters/sms-backup-restore-exporter/src/read/tests.rs +++ b/crates/exporters/sms-backup-restore-exporter/src/read/tests.rs @@ -556,3 +556,76 @@ fn a_message_kept_with_something_left_out_is_named() { assert_eq!(report.skipped_unreadable_part(), 1); assert_eq!(report.dropped_character_references(), 2); } + +/// The fixture's root says when SMS Backup & Restore made the backup. +fn dated_fixture() -> std::path::PathBuf { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/dated_backup.xml") +} + +/// The backup date of each conversation read from `input`, by chat id. +fn backup_dates(input: &Path) -> Vec<(String, Option)> { + let owners = ["+15555550100".to_string()]; + let (docs, _) = read_backup(input, opts(&owners, None, None)).unwrap(); + docs.into_iter() + .map(|doc| { + ( + doc.conversation.chat_identifier, + doc.export.backup_taken_at_unix_ms, + ) + }) + .collect() +} + +/// The root element's `backup_date` decides, not when the file was copied. +#[test] +fn the_backup_date_is_the_files_backup_date_attribute() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("dated_backup.xml"); + fs::copy(dated_fixture(), &input).unwrap(); + message_crate_core::testutil::set_modified_unix_ms(&input, 1_000_000_000_000); + assert_eq!( + backup_dates(&input), + [ + ("+15555550101".to_string(), Some(1_790_793_912_000)), + ("+15555550102".to_string(), Some(1_790_793_912_000)), + ] + ); +} + +/// A file without `backup_date` is dated by when it was last changed. +#[test] +fn a_file_without_a_backup_date_is_dated_by_its_modification_time() { + let dir = tempfile::tempdir().unwrap(); + let input = dir.path().join("undated.xml"); + fs::write( + &input, + r#""#, + ) + .unwrap(); + message_crate_core::testutil::set_modified_unix_ms(&input, 1_500_000_000_000); + assert_eq!( + backup_dates(&input), + [("+15555550101".to_string(), Some(1_500_000_000_000))] + ); +} + +/// Two backups in one directory: a conversation both hold is as new as the +/// newer, and one only the older holds keeps the older's date. +#[test] +fn a_conversation_in_two_backups_is_as_new_as_the_newer() { + let dir = tempfile::tempdir().unwrap(); + fs::copy(dated_fixture(), dir.path().join("newer.xml")).unwrap(); + fs::write( + dir.path().join("older.xml"), + r#""#, + ) + .unwrap(); + assert_eq!( + backup_dates(dir.path()), + [ + ("+15555550101".to_string(), Some(1_790_793_912_000)), + ("+15555550102".to_string(), Some(1_790_793_912_000)), + ("+15555550103".to_string(), Some(1_780_000_000_000)), + ] + ); +} diff --git a/crates/exporters/sms-backup-restore-exporter/src/write.rs b/crates/exporters/sms-backup-restore-exporter/src/write.rs index 9450c1bcf..67110ed5f 100644 --- a/crates/exporters/sms-backup-restore-exporter/src/write.rs +++ b/crates/exporters/sms-backup-restore-exporter/src/write.rs @@ -54,11 +54,17 @@ impl SbrBackupSession { } /// Write the SMS and MMS of one conversation as SBR `` or `` - /// elements, and count every other message as left out. A conversation + /// elements, note its backup date for the file's `backup_date` when it + /// writes any, and count every other message as left out. A conversation /// with no SMS or MMS writes nothing. pub fn append_document(&mut self, doc: &ConversationDocument) -> Result<()> { self.not_sms_or_mms += doc.messages.iter().filter(|m| !m.is_sms_or_mms()).count() as u64; - for msg in document_to_sbr_messages(doc, &self.output_dir)? { + let messages = document_to_sbr_messages(doc, &self.output_dir)?; + if !messages.is_empty() { + self.writer + .note_backup_date(doc.export.backup_taken_at_unix_ms); + } + for msg in messages { self.writer.write_message(&msg)?; } Ok(()) diff --git a/crates/exporters/sms-backup-restore-exporter/src/write/tests.rs b/crates/exporters/sms-backup-restore-exporter/src/write/tests.rs index 64e2d8166..53e4097b8 100644 --- a/crates/exporters/sms-backup-restore-exporter/src/write/tests.rs +++ b/crates/exporters/sms-backup-restore-exporter/src/write/tests.rs @@ -596,3 +596,22 @@ fn the_archive_counts_the_characters_xml_cannot_carry() { let text = fs::read_to_string(&path).unwrap(); assert!(text.contains(r#"body="bell and escape""#), "{text}"); } + +/// The file's `backup_date` is the newest date of the conversations it +/// writes messages for: a WhatsApp conversation, which writes none, does +/// not move it. +#[test] +fn the_backup_date_comes_from_the_conversations_written() { + let mut sms = message_ir::testutil::sample_document("hello ir"); + sms.export.backup_taken_at_unix_ms = Some(1_788_256_800_000); + let mut whatsapp = message_ir::testutil::sample_whatsapp_document("hello whatsapp"); + whatsapp.export.backup_taken_at_unix_ms = Some(1_790_793_912_000); + + let tmp = tempfile::tempdir().unwrap(); + let mut report = message_crate_core::ExportReport::default(); + let path = SbrArchive + .write(tmp.path(), &[sms, whatsapp], &mut report) + .unwrap(); + let text = fs::read_to_string(&path).unwrap(); + assert!(text.contains(r#"backup_date="1788256800000""#), "{text}"); +} diff --git a/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs b/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs index 748265f38..3392361e3 100644 --- a/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs +++ b/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs @@ -344,7 +344,7 @@ fn convert_export_json_and_jsonl_use_pristine_v4() { .expect("expected .json"); let raw = fs::read_to_string(&json_path).unwrap(); let doc: serde_json::Value = serde_json::from_str(&raw).unwrap(); - assert_eq!(doc["schema_version"], 10); + assert_eq!(doc["schema_version"], 11); assert!( doc["conversation"]["stats"]["message_count"] .as_u64() @@ -389,7 +389,7 @@ fn convert_export_json_and_jsonl_use_pristine_v4() { let body = fs::read_to_string(&jsonl_path).unwrap(); let mut lines = body.lines(); let header: serde_json::Value = serde_json::from_str(lines.next().unwrap()).unwrap(); - assert_eq!(header["schema_version"], 10); + assert_eq!(header["schema_version"], 11); assert!(header.get("messages").is_none()); assert!( header["conversation"]["stats"]["message_count"] diff --git a/crates/exporters/sms-backup-restore-exporter/tests/fixtures/dated_backup.xml b/crates/exporters/sms-backup-restore-exporter/tests/fixtures/dated_backup.xml new file mode 100644 index 000000000..405005b23 --- /dev/null +++ b/crates/exporters/sms-backup-restore-exporter/tests/fixtures/dated_backup.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/crates/exporters/whatsapp-exporter/src/emit.rs b/crates/exporters/whatsapp-exporter/src/emit.rs index 5003f2871..a6725b6d7 100644 --- a/crates/exporters/whatsapp-exporter/src/emit.rs +++ b/crates/exporters/whatsapp-exporter/src/emit.rs @@ -42,6 +42,9 @@ pub(crate) struct ConvertRequest<'a> { /// so on every message as the address it was held at. `None` records no /// owner, which leaves the conversations counted toward no identity. pub owner_identity: Option, + /// When the backup was made, in Unix milliseconds, stamped on the export + /// header ([`backup_taken_at_unix_ms`](crate::run::backup_taken_at_unix_ms)). + pub backup_taken_at_unix_ms: Option, pub output_format: OutputFormat, /// Checked between chats (cooperative cancellation). pub cancel: Option<&'a CancelFlag>, @@ -64,6 +67,7 @@ pub(crate) fn convert_json(request: ConvertRequest<'_>) -> Result transforms, media_search_roots, owner_identity, + backup_taken_at_unix_ms, output_format, cancel, resume, @@ -107,6 +111,7 @@ pub(crate) fn convert_json(request: ConvertRequest<'_>) -> Result EXPORT_TOOL_VERSION, owner_identity, None, + backup_taken_at_unix_ms, ), }; let mut documents = Vec::new(); diff --git a/crates/exporters/whatsapp-exporter/src/run.rs b/crates/exporters/whatsapp-exporter/src/run.rs index e89acf4d2..828c21585 100644 --- a/crates/exporters/whatsapp-exporter/src/run.rs +++ b/crates/exporters/whatsapp-exporter/src/run.rs @@ -5,7 +5,8 @@ use crate::emit::{ConvertRequest, convert_json}; use crate::ios_backup::{decrypt_if_encrypted, extract_bytes}; use crate::owner::{owner_from_backup, owner_from_form}; use crate::wtsexporter::{ - Platform, WtsexporterArgs, extracts_ios_backup, resolve_wtsexporter, run_wtsexporter, + Platform, WtsexporterArgs, android_crypt_backup, extracts_ios_backup, resolve_wtsexporter, + run_wtsexporter, }; use anyhow::{Context, Result, bail}; use message_crate_core::{ @@ -58,9 +59,13 @@ pub fn run(config: &ExporterConfig) -> Result { .map(owner_from_form) .transpose()?; - let (json_path, media_roots, owner_identity, _work_keep_alive) = if let Some(json) = - &source.json - { + let ConversionInput { + json_path, + media_roots, + owner_identity, + backup_taken_at_unix_ms, + work, + } = if let Some(json) = &source.json { // Allowed roots are only the backup input and the JSON parent — never // the process CWD, which would let crafted paths copy arbitrary files. let mut media_roots = Vec::new(); @@ -73,8 +78,15 @@ pub fn run(config: &ExporterConfig) -> Result { media_roots.sort(); media_roots.dedup(); // A ready-made result.json names no owner; the form's number is all - // there is, and a conversion may leave it empty. - (json.clone(), media_roots, form_owner, None) + // there is, and a conversion may leave it empty. It names no backup + // date either, so it is dated by when it was written. + ConversionInput { + json_path: json.clone(), + media_roots, + owner_identity: form_owner, + backup_taken_at_unix_ms: message_crate_core::file_modified_unix_ms(json), + work: None, + } } else { let platform = platform.ok_or_else(|| anyhow::anyhow!("platform is required unless json is set"))?; @@ -102,6 +114,9 @@ pub fn run(config: &ExporterConfig) -> Result { db: source.db.clone(), business: source.business, }; + // Read before an encrypted backup's files are decrypted, which points + // `args` at the decrypted copy, dated the moment it was made. + let backup_taken_at_unix_ms = backup_taken_at_unix_ms(&args); // wtsexporter cannot be given an iPhone backup password, so an // encrypted backup's WhatsApp files are decrypted into the work // directory first and wtsexporter reads those instead of the backup. @@ -154,7 +169,13 @@ pub fn run(config: &ExporterConfig) -> Result { } }; - (kept, media_roots, Some(owner_identity), Some(work)) + ConversionInput { + json_path: kept, + media_roots, + owner_identity: Some(owner_identity), + backup_taken_at_unix_ms, + work: Some(work), + } }; if !json_path.is_file() { @@ -170,13 +191,14 @@ pub fn run(config: &ExporterConfig) -> Result { transforms, media_search_roots: &media_roots, owner_identity, + backup_taken_at_unix_ms, output_format: config.output_format, cancel: config.cancel.as_ref(), resume: config.resume, issues: config.issues.as_ref(), })?; // The work directory goes once the conversion has copied the media. - drop(_work_keep_alive); + drop(work); let mut result = message_crate_core::finish_run(config, &report, needs_media_tools)?; messages.append(&mut result.messages); @@ -184,6 +206,50 @@ pub fn run(config: &ExporterConfig) -> Result { Ok(result) } +/// The `result.json` a run converts, and what the conversion needs to know +/// about the backup it came from. +struct ConversionInput { + /// The JSON to convert. + json_path: std::path::PathBuf, + /// Where the conversion may look for media. + media_roots: Vec, + /// The owner's WhatsApp number, when known. + owner_identity: Option, + /// When the backup was made, in Unix milliseconds. + backup_taken_at_unix_ms: Option, + /// The work directory wtsexporter wrote into, kept until the media is + /// copied; `None` for a ready-made `result.json`. + work: Option, +} + +/// When the backup wtsexporter reads was made, in Unix milliseconds: an +/// iPhone backup's `Manifest.plist` date, or for Android the modification +/// time of the WhatsApp database file (`msgstore.db.crypt15` or a decrypted +/// `msgstore.db`), which WhatsApp writes when it backs up. `None` when +/// neither can be read, such as an iPhone backup extracted by hand without +/// its manifest. +pub(crate) fn backup_taken_at_unix_ms(args: &WtsexporterArgs) -> Option { + match args.platform { + Platform::Ios => ::ios_backup::ios_backup_date_unix_ms(&args.input), + Platform::Android => { + let database = args + .backup + .clone() + .or_else(|| args.db.clone()) + .or_else(|| android_crypt_backup(&args.input)) + .unwrap_or_else(|| { + let decrypted = args.input.join("msgstore.db"); + if decrypted.is_file() { + decrypted + } else { + args.input.clone() + } + }); + message_crate_core::file_modified_unix_ms(&database) + } + } +} + /// Mark the output directory as an export directory, then make the work /// directory wtsexporter runs in (its working directory, the extract, and /// `result.json`) under the Scratch Directory's [`WHATSAPP_DIRECTORY`]. The @@ -216,6 +282,7 @@ fn mark_output_and_make_work_directory(config: &ExporterConfig) -> Result WtsexporterArgs { + WtsexporterArgs { + platform, + input: input.to_path_buf(), + work_dir: input.to_path_buf(), + key: None, + backup: None, + wa: None, + media: None, + db: None, + business: false, + } + } + + /// An iPhone backup is dated by its `Manifest.plist`. + #[test] + fn an_iphone_backup_is_dated_by_its_manifest() { + let backup = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ios-backup"); + let args = args_for(crate::wtsexporter::Platform::Ios, &backup); + // 2026-09-30T18:45:12Z + assert_eq!( + super::backup_taken_at_unix_ms(&args), + Some(1_790_793_912_000) + ); + } + + /// An Android backup records no date inside it, so it is dated by when + /// WhatsApp wrote its database file; an iPhone backup with no manifest + /// has no date. + #[test] + fn an_android_backup_is_dated_by_its_database_file() { + use message_crate_core::testutil::{TEST_BACKUP_TAKEN_AT_UNIX_MS, set_modified_unix_ms}; + let dir = tempfile::tempdir().unwrap(); + let crypt = dir.path().join("msgstore.db.crypt15"); + fs::write(&crypt, b"encrypted").unwrap(); + set_modified_unix_ms(&crypt, TEST_BACKUP_TAKEN_AT_UNIX_MS); + let args = args_for(crate::wtsexporter::Platform::Android, dir.path()); + assert_eq!( + super::backup_taken_at_unix_ms(&args), + Some(TEST_BACKUP_TAKEN_AT_UNIX_MS) + ); + + let named = dir.path().join("elsewhere.crypt15"); + fs::write(&named, b"encrypted").unwrap(); + set_modified_unix_ms(&named, TEST_BACKUP_TAKEN_AT_UNIX_MS - 60_000); + let mut args = args_for(crate::wtsexporter::Platform::Android, dir.path()); + args.backup = Some(named); + assert_eq!( + super::backup_taken_at_unix_ms(&args), + Some(TEST_BACKUP_TAKEN_AT_UNIX_MS - 60_000), + "the file the form names wins" + ); + + let unpacked = tempfile::tempdir().unwrap(); + let args = args_for(crate::wtsexporter::Platform::Ios, unpacked.path()); + assert_eq!(super::backup_taken_at_unix_ms(&args), None); + } } diff --git a/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs b/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs index e9b315a9e..22d181bbf 100644 --- a/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs +++ b/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs @@ -16,6 +16,7 @@ fn convert_fixture_json_individual_and_group() { transforms: ExportTransforms::none(), media_search_roots: &[], owner_identity: None, + backup_taken_at_unix_ms: None, output_format: OutputFormat::Csv, cancel: None, resume: false, @@ -86,6 +87,7 @@ fn copies_ios_style_media_true_data_paths() { transforms: ExportTransforms::none(), media_search_roots: &[media_root.path().to_path_buf()], owner_identity: None, + backup_taken_at_unix_ms: None, output_format: OutputFormat::Csv, cancel: None, resume: false, @@ -123,6 +125,7 @@ fn jsonl_drains_the_write_queue_and_a_second_run_resumes_it() { transforms: ExportTransforms::none(), media_search_roots: &[], owner_identity: None, + backup_taken_at_unix_ms: None, output_format: OutputFormat::Jsonl, cancel: None, resume, @@ -150,6 +153,7 @@ fn convert_to_documents( transforms: ExportTransforms::none(), media_search_roots: &[dir.path().to_path_buf()], owner_identity: None, + backup_taken_at_unix_ms: None, output_format: OutputFormat::Json, cancel: None, resume: false, @@ -343,6 +347,7 @@ fn a_media_path_in_the_media_field_is_copied() { transforms: ExportTransforms::none(), media_search_roots: &[dir.path().to_path_buf()], owner_identity: None, + backup_taken_at_unix_ms: None, output_format: OutputFormat::Csv, cancel: None, resume: false, @@ -393,6 +398,7 @@ fn a_media_file_not_found_is_kept_as_file_missing_and_the_guid_does_not_change() transforms: ExportTransforms::none(), media_search_roots: &[dir.path().to_path_buf()], owner_identity: None, + backup_taken_at_unix_ms: None, output_format: OutputFormat::Json, cancel: None, resume: false, diff --git a/crates/exporters/whatsapp-exporter/tests/fixtures/ios-backup/Manifest.plist b/crates/exporters/whatsapp-exporter/tests/fixtures/ios-backup/Manifest.plist new file mode 100644 index 000000000..64224a8e8 --- /dev/null +++ b/crates/exporters/whatsapp-exporter/tests/fixtures/ios-backup/Manifest.plist @@ -0,0 +1,12 @@ + + + + + Date + 2026-09-30T18:45:12Z + IsEncrypted + + Version + 10.0 + + diff --git a/crates/libs/api-types/src/lib.rs b/crates/libs/api-types/src/lib.rs index 07ed2e9fe..ad4e4479f 100644 --- a/crates/libs/api-types/src/lib.rs +++ b/crates/libs/api-types/src/lib.rs @@ -421,6 +421,11 @@ api_shape! { /// each part; `text` is the final version. Empty for a message never /// edited, or from a source that records no edits. pub earlier_versions: Vec, + /// When the backup that gave the message its mark and text was made, + /// RFC 3339 in UTC with a `Z` suffix; `null` when the conversation + /// file did not say. Between two copies of one message from one + /// source, the copy from the later backup decides. + pub backup_taken_at: Option, /// True in a Messages search answer (`GET /v1/messages` with `q`) /// when the message is a hit only because of its earlier versions: /// its final text alone does not match the query, and the versions @@ -656,6 +661,7 @@ mod tests { edited_at: Some("2023-12-31T23:59:00Z".into()), matched: false, }], + backup_taken_at: Some("2024-01-02T08:00:00.000Z".into()), matched_earlier_version: false, }; diff --git a/crates/libs/export/src/project.rs b/crates/libs/export/src/project.rs index e92cd285e..0b5f6a134 100644 --- a/crates/libs/export/src/project.rs +++ b/crates/libs/export/src/project.rs @@ -22,6 +22,10 @@ pub fn conversation_key(msg: &Message) -> String { } /// Build one conversation document from a seed message and the mapped rows. +/// +/// The document's backup date is the seed's `backup_taken_at`, which the +/// Export Run keeps only while every message of the conversation has the +/// same one ([`common_backup_taken_at`]), and none otherwise. pub fn build_document( source: &str, seed: &Message, @@ -59,6 +63,10 @@ pub fn build_document( tool_version: env!("CARGO_PKG_VERSION").into(), owner_identity: shared_owner(&messages), owner_display_name: Some("Me".into()), + backup_taken_at_unix_ms: seed + .backup_taken_at + .as_deref() + .and_then(|at| parse_timestamp_unix_ms(at).ok()), }, conversation: ConversationMeta { chat_identifier: seed.conversation.chat_identifier.clone(), @@ -77,6 +85,20 @@ pub fn build_document( } } +/// Clear `seed`'s `backup_taken_at` when `msg`'s differs from it, so the +/// conversation file names a backup date only when every message of the +/// conversation came from that one backup. The file carries one date for +/// all its messages, and any one date would be wrong for some of them when +/// they differ: the newest would make a message decided by an older backup +/// win over a backup made between the two, and would date a message that +/// had none. With no date, an import of the file keeps the rules for files +/// without one. +pub fn common_backup_taken_at(seed: &mut Message, msg: &Message) { + if msg.backup_taken_at != seed.backup_taken_at { + seed.backup_taken_at = None; + } +} + /// Map one exported message into the shared conversation message type. /// /// # Errors @@ -617,6 +639,50 @@ mod tests { assert_eq!(doc.conversation.stats.message_count, 2); } + /// The file says the backup its messages came from when they all came + /// from one, and nothing when any two differ or none says. + #[test] + fn a_document_says_its_backup_only_when_every_message_came_from_it() { + let participant = || Participant { + identity: Some("+1".into()), + name: "Sam".into(), + service: None, + contact_id: None, + }; + let backup_of = |seed: &Message| { + build_document("imessage", seed, vec![]) + .export + .backup_taken_at_unix_ms + }; + let dated = |at: Option<&str>| { + let mut msg = seed_message_with_participant(participant()); + msg.backup_taken_at = at.map(str::to_string); + msg + }; + let later = Some("2026-09-30T18:45:12.000Z"); + let earlier = Some("2026-09-01T10:00:00.000Z"); + + assert_eq!(backup_of(&dated(None)), None); + + let mut seed = dated(later); + common_backup_taken_at(&mut seed, &dated(later)); + assert_eq!(backup_of(&seed), Some(1_790_793_912_000)); + + let mut seed = dated(earlier); + common_backup_taken_at(&mut seed, &dated(later)); + common_backup_taken_at(&mut seed, &dated(None)); + assert_eq!(backup_of(&seed), None, "two backups: no one date is right"); + + let mut seed = dated(later); + common_backup_taken_at(&mut seed, &dated(None)); + common_backup_taken_at(&mut seed, &dated(later)); + assert_eq!( + backup_of(&seed), + None, + "a message without a date stays without one" + ); + } + /// Whether the conversation is a group comes from the server's /// `is_group`, never from reading `conversation_type` again. #[test] @@ -727,6 +793,7 @@ mod tests { tapbacks: vec![], deletion: None, earlier_versions: Vec::new(), + backup_taken_at: None, matched_earlier_version: false, }; let ir = to_ir_message(&msg, false).unwrap(); @@ -878,6 +945,7 @@ mod tests { tapbacks: vec![], deletion: None, earlier_versions: Vec::new(), + backup_taken_at: None, matched_earlier_version: false, } } diff --git a/crates/libs/export/src/run.rs b/crates/libs/export/src/run.rs index a9131e104..7fd81a9b4 100644 --- a/crates/libs/export/src/run.rs +++ b/crates/libs/export/src/run.rs @@ -16,7 +16,10 @@ use serde::Serialize; use crate::http::{CloseAction, ExportMessagesArgs, HttpSession}; use crate::journal::{self, ExportJournalEvent, ExportJournalState, ServerTarget}; use crate::part_file::write_asset; -use crate::project::{ExportPath, build_document, conversation_key, export_path, to_ir_message}; +use crate::project::{ + ExportPath, build_document, common_backup_taken_at, conversation_key, export_path, + to_ir_message, +}; use message_crate_api_types::{ExportQueryList, ExportRun, ExportScope, Message}; /// Page size for `GET /v1/exports/{id}/messages`; the server's maximum. @@ -496,13 +499,13 @@ impl<'a> Export<'a> { } } let ir = to_ir_message(&msg, cfg.skip_attachments)?; - fetched + let (seed, messages) = fetched .by_conv .entry(conversation_key(&msg)) // Keep first message as seed for conversation metadata. - .or_insert_with(|| (msg.clone(), Vec::new())) - .1 - .push(ir); + .or_insert_with(|| (msg.clone(), Vec::new())); + common_backup_taken_at(seed, &msg); + messages.push(ir); } match next_offset(offset, limit, page.total) { Some(next) => offset = next, diff --git a/crates/libs/ios-backup/src/backup.rs b/crates/libs/ios-backup/src/backup.rs index 00a900a6c..8713bb97e 100644 --- a/crates/libs/ios-backup/src/backup.rs +++ b/crates/libs/ios-backup/src/backup.rs @@ -26,6 +26,23 @@ pub fn ios_backup_encrypted_flag(backup_root: &Path) -> Option { } } +/// When the iPhone backup at `backup_root` was made, in Unix milliseconds: +/// the `Date` its `Manifest.plist` records. The plist is not encrypted, even +/// in an encrypted backup, so no password is needed. +/// +/// Returns `None` when the file is missing, cannot be parsed, or has no +/// date: the conversation file then says nothing about when the backup was +/// made, and the import falls back to its rules for a file without one. +pub fn ios_backup_date_unix_ms(backup_root: &Path) -> Option { + let file = File::open(backup_root.join("Manifest.plist")).ok()?; + let value = plist::Value::from_reader(file).ok()?; + let date = value.as_dictionary()?.get("Date")?.as_date()?; + let since = std::time::SystemTime::from(date) + .duration_since(std::time::UNIX_EPOCH) + .ok()?; + i64::try_from(since.as_millis()).ok() +} + /// Every regular file `Manifest.db` lists under `domain` in the iPhone /// backup at `backup_root`, which is not encrypted, as its path inside the /// domain and its size where the backup keeps it @@ -77,8 +94,9 @@ pub fn ios_backup_domain_files(backup_root: &Path, domain: &str) -> Result + + + + Date + 2026-09-30T18:45:12Z + IsEncrypted + + Version + 10.0 + + diff --git a/crates/libs/ir-format/src/export_transforms/tests.rs b/crates/libs/ir-format/src/export_transforms/tests.rs index 7e5ae1ce2..44a850d02 100644 --- a/crates/libs/ir-format/src/export_transforms/tests.rs +++ b/crates/libs/ir-format/src/export_transforms/tests.rs @@ -18,6 +18,7 @@ fn doc_with_image_attachment() -> ConversationDocument { tool_version: "0".into(), owner_identity: None, owner_display_name: None, + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "+15555550101".into(), @@ -218,6 +219,7 @@ fn doc_with_a_marker_in_every_field() -> ConversationDocument { tool_version: "10.20".into(), owner_identity: Some("LEAK-01".into()), owner_display_name: Some("LEAK-02".into()), + backup_taken_at_unix_ms: Some(1_790_793_912_000), }, conversation: ConversationMeta { chat_identifier: "LEAK-03".into(), diff --git a/crates/libs/ir-format/src/lib_tests.rs b/crates/libs/ir-format/src/lib_tests.rs index 182ae2cd4..e8f07c436 100644 --- a/crates/libs/ir-format/src/lib_tests.rs +++ b/crates/libs/ir-format/src/lib_tests.rs @@ -820,3 +820,87 @@ fn eml_keeps_each_messages_owner_and_each_attachments_size_and_missing_reason() fn mbox_keeps_each_messages_owner_each_attachments_metadata_and_a_text_attachments_bytes() { assert_hard_fields_survive(OutputFormat::Mbox); } + +/// Write a document whose backup has `backup_taken_at_unix_ms` in `format` +/// and read it back. +fn backup_date_after_round_trip( + format: OutputFormat, + backup_taken_at_unix_ms: Option, +) -> Option { + let mut doc = message_ir::testutil::sample_document("from the backup"); + doc.export.backup_taken_at_unix_ms = backup_taken_at_unix_ms; + let tmp = tempfile::tempdir().unwrap(); + let path = write_format(tmp.path(), format, doc).unwrap(); + let back = match format { + OutputFormat::Json => read_conversation_json(&path), + OutputFormat::Jsonl => read_conversation_jsonl(&path), + OutputFormat::Csv => read_conversation_csv(&path), + OutputFormat::Eml => read_conversation_eml_dir(&path), + OutputFormat::Mbox => read_conversation_mbox(&path), + other => panic!("no round trip for {}", other.as_str()), + } + .unwrap(); + back.export.backup_taken_at_unix_ms +} + +/// Every format keeps when the backup was made, and a file that does not +/// say reads back saying nothing, never a date of the reader's own. +#[test] +fn every_format_keeps_when_the_backup_was_made() { + for format in [ + OutputFormat::Json, + OutputFormat::Jsonl, + OutputFormat::Csv, + OutputFormat::Eml, + OutputFormat::Mbox, + ] { + assert_eq!( + backup_date_after_round_trip(format, Some(1_790_793_912_345)), + Some(1_790_793_912_345), + "{}", + format.as_str() + ); + assert_eq!( + backup_date_after_round_trip(format, None), + None, + "{}", + format.as_str() + ); + } +} + +#[test] +fn csv_refuses_a_backup_date_that_is_not_a_number() { + let doc = message_ir::testutil::sample_document("hello"); + let tmp = tempfile::tempdir().unwrap(); + let path = write_format(tmp.path(), OutputFormat::Csv, doc).unwrap(); + let text = fs::read_to_string(&path).unwrap(); + let mut lines: Vec = text.lines().map(str::to_string).collect(); + let column = CSV_HEADERS + .iter() + .position(|h| *h == "backup_taken_at_unix_ms") + .unwrap(); + let mut row = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(lines[1].as_bytes()) + .records() + .next() + .unwrap() + .unwrap() + .iter() + .map(str::to_string) + .collect::>(); + row[column] = "yesterday".into(); + let mut out = csv::Writer::from_writer(Vec::new()); + out.write_record(&row).unwrap(); + lines[1] = String::from_utf8(out.into_inner().unwrap()) + .unwrap() + .trim_end() + .to_string(); + fs::write(&path, lines.join("\n")).unwrap(); + let err = read_conversation_csv(&path).unwrap_err(); + assert!( + format!("{err:#}").contains("bad backup_taken_at_unix_ms \"yesterday\""), + "{err:#}" + ); +} diff --git a/crates/libs/ir-format/src/read_csv.rs b/crates/libs/ir-format/src/read_csv.rs index b5b4483f7..bc24e55ff 100644 --- a/crates/libs/ir-format/src/read_csv.rs +++ b/crates/libs/ir-format/src/read_csv.rs @@ -48,7 +48,8 @@ pub fn read_conversation_csv(path: &Path) -> Result { bail!("CSV has no data rows: {}", path.display()); } - let header = header_from_row(&cols, &rows[0]); + let header = header_from_row(&cols, &rows[0]) + .with_context(|| format!("parse CSV row 1 in {}", path.display()))?; let mut messages = Vec::with_capacity(rows.len()); for (i, record) in rows.iter().enumerate() { messages.push( @@ -63,7 +64,15 @@ pub fn read_conversation_csv(path: &Path) -> Result { } /// Rebuild the conversation header from the first CSV row's conversation columns. -fn header_from_row(cols: &HashMap<&str, usize>, row: &csv::StringRecord) -> ConversationHeader { +/// +/// # Errors +/// +/// Returns an error when `backup_taken_at_unix_ms` holds anything but a +/// whole number or a blank. +fn header_from_row( + cols: &HashMap<&str, usize>, + row: &csv::StringRecord, +) -> Result { let get = |name: &str| cell(cols, row, name).unwrap_or(""); let participants = parse_participants(get("participants_json")); let group_title = { @@ -74,7 +83,14 @@ fn header_from_row(cols: &HashMap<&str, usize>, row: &csv::StringRecord) -> Conv Some(t.to_string()) } }; - ConversationHeader { + let backup_taken_at_unix_ms = match get("backup_taken_at_unix_ms").trim() { + "" => None, + ms => Some( + ms.parse::() + .with_context(|| format!("bad backup_taken_at_unix_ms {ms:?}"))?, + ), + }; + Ok(ConversationHeader { schema_version: SCHEMA_VERSION, export: ExportMeta { source: get("export_source").to_string(), @@ -82,6 +98,7 @@ fn header_from_row(cols: &HashMap<&str, usize>, row: &csv::StringRecord) -> Conv tool_version: get("export_tool_version").to_string(), owner_identity: nonempty(get("owner_identity")), owner_display_name: nonempty(get("owner_display_name")), + backup_taken_at_unix_ms, }, conversation: ConversationMeta { chat_identifier: get("chat_identifier").to_string(), @@ -90,7 +107,7 @@ fn header_from_row(cols: &HashMap<&str, usize>, row: &csv::StringRecord) -> Conv participants, stats: ConversationStats::default(), }, - } + }) } /// Rebuild one message from a CSV row. diff --git a/crates/libs/ir-format/src/read_mail.rs b/crates/libs/ir-format/src/read_mail.rs index 92d40522b..7be13e041 100644 --- a/crates/libs/ir-format/src/read_mail.rs +++ b/crates/libs/ir-format/src/read_mail.rs @@ -86,6 +86,7 @@ fn document_from_mail_messages(messages: &[MailMessage]) -> Result = first diff --git a/crates/libs/ir-format/src/write.rs b/crates/libs/ir-format/src/write.rs index 1d1a88c61..f7d6fbaae 100644 --- a/crates/libs/ir-format/src/write.rs +++ b/crates/libs/ir-format/src/write.rs @@ -46,6 +46,7 @@ pub const CSV_HEADERS: &[&str] = &[ "export_tool_version", "owner_identity", "owner_display_name", + "backup_taken_at_unix_ms", "message_owner_identity", "android_type", "source_fields_json", @@ -214,6 +215,11 @@ pub(crate) fn write_conversation_csv( }) .collect::>(), ); + let backup_taken_at = doc + .export + .backup_taken_at_unix_ms + .map(|ms| ms.to_string()) + .unwrap_or_default(); message_ir::write_atomic(&path, |out| { let mut wtr = csv::Writer::from_writer(out); @@ -221,8 +227,14 @@ pub(crate) fn write_conversation_csv( .with_context(|| format!("write header {}", path.display()))?; for msg in &doc.messages { let cells = MessageCells::new(msg)?; - wtr.write_record(csv_record(doc, &participants_json, msg, &cells)) - .with_context(|| format!("write row {}", path.display()))?; + wtr.write_record(csv_record( + doc, + &participants_json, + &backup_taken_at, + msg, + &cells, + )) + .with_context(|| format!("write row {}", path.display()))?; } wtr.flush()?; Ok(()) @@ -366,9 +378,10 @@ fn bool_cell(value: bool) -> &'static str { fn csv_record<'a>( doc: &'a ConversationDocument, participants_json: &'a str, + backup_taken_at: &'a str, msg: &'a IrMessage, cells: &'a MessageCells, -) -> [&'a str; 46] { +) -> [&'a str; 47] { let im = &cells.imessage; [ doc.conversation.chat_identifier.as_str(), @@ -403,6 +416,7 @@ fn csv_record<'a>( doc.export.tool_version.as_str(), doc.export.owner_identity.as_deref().unwrap_or(""), doc.export.owner_display_name.as_deref().unwrap_or(""), + backup_taken_at, msg.owner_identity.as_deref().unwrap_or(""), cells.android_type.as_str(), cells.source_fields_json.as_str(), @@ -479,6 +493,7 @@ pub fn document_to_mail_messages( export_source: doc.export.source.clone(), export_tool: doc.export.tool.clone(), export_tool_version: doc.export.tool_version.clone(), + backup_taken_at_unix_ms: doc.export.backup_taken_at_unix_ms, filename_suffix: doc.packaging_stem_suffix.clone(), message: msg.clone(), attachments, diff --git a/crates/libs/ir/src/lib.rs b/crates/libs/ir/src/lib.rs index 44aef8a1c..ba8709bf6 100644 --- a/crates/libs/ir/src/lib.rs +++ b/crates/libs/ir/src/lib.rs @@ -138,7 +138,7 @@ impl std::fmt::Display for UnknownDeletion { impl std::error::Error for UnknownDeletion {} /// Schema version written into every [`ConversationDocument`]. -pub const SCHEMA_VERSION: u32 = 10; +pub const SCHEMA_VERSION: u32 = 11; /// One exported chat: export metadata, conversation roster and stats, and messages. /// @@ -172,6 +172,13 @@ pub struct ExportMeta { pub owner_identity: Option, /// Outgoing display name. Set when known (iMessage caller-id or `"Me"`). pub owner_display_name: Option, + /// When the backup this file was read from was made, in Unix + /// milliseconds: an iPhone backup's `Manifest.plist` date, an SMS Backup + /// & Restore file's `backup_date`, or the backup file's modification + /// time where the source records nothing better. `None` when nothing + /// says. Between two copies of one message from one source, the import + /// lets the copy from the later backup decide its deletion mark and text. + pub backup_taken_at_unix_ms: Option, } /// The shape of a conversation: one-to-one, group, or orphaned. diff --git a/crates/libs/ir/src/projection.rs b/crates/libs/ir/src/projection.rs index 75e1e3fb9..aee12212e 100644 --- a/crates/libs/ir/src/projection.rs +++ b/crates/libs/ir/src/projection.rs @@ -516,6 +516,7 @@ mod tests { tool_version: "0".into(), owner_identity: Some("+15555550100".into()), owner_display_name: None, + backup_taken_at_unix_ms: None, } } diff --git a/crates/libs/ir/src/schema_version.rs b/crates/libs/ir/src/schema_version.rs index 851c94781..3b99d3f3a 100644 --- a/crates/libs/ir/src/schema_version.rs +++ b/crates/libs/ir/src/schema_version.rs @@ -3,12 +3,12 @@ //! Every reader of a [`ConversationDocument`](crate::ConversationDocument) or //! its JSON Lines header — the format reader, the push client, the server's //! import — refuses a version other than [`SCHEMA_VERSION`] with the same -//! words, and refuses it before parsing the rest of the file: a version-9 -//! file is not expected to match version 10 (version 9 kept a reply's link in -//! `imessage.is_reply` and `imessage.in_reply_to_guid`, which version 10 would -//! pass over, so every reply would arrive as a plain message), and the person -//! should read "schema version 9", not a file that imports without its -//! replies. +//! words, and refuses it before parsing the rest of the file: a version-10 +//! file is not expected to match version 11 (version 10 did not say when its +//! backup was made, so an import could not tell which of two backups of one +//! phone is the later one, and a file read as version 11 would claim it has +//! no date when it only never asked), and the person should read "schema +//! version 10", not a file that imports by other rules than its own. use crate::SCHEMA_VERSION; use serde::Deserialize; @@ -95,6 +95,21 @@ mod tests { ); } + /// Version 10 had no `export.backup_taken_at_unix_ms`; version 11 says + /// when the backup was made, and the import lets the later backup decide + /// a message's mark and text. A version-10 file is refused by its + /// version, never read as a file whose backup has no date. + #[test] + fn refuses_a_version_10_file_by_name() { + assert_eq!( + check_schema_version_in_json(r#"{"schema_version":10,"export":{}}"#) + .unwrap_err() + .to_string(), + format!("This file is schema version 10; Message Crate reads version {SCHEMA_VERSION}") + ); + assert_eq!(SCHEMA_VERSION, 11); + } + #[test] fn peeks_at_the_version_before_anything_else() { assert_eq!( diff --git a/crates/libs/ir/src/testutil.rs b/crates/libs/ir/src/testutil.rs index 706de1f62..f394d34a9 100644 --- a/crates/libs/ir/src/testutil.rs +++ b/crates/libs/ir/src/testutil.rs @@ -19,6 +19,7 @@ pub fn sample_document(text: &str) -> ConversationDocument { tool_version: "10.26.003".into(), owner_identity: Some("+15555550100".into()), owner_display_name: Some("Me".into()), + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "+15555550101".into(), @@ -95,6 +96,7 @@ pub fn sample_imessage_document() -> ConversationDocument { tool_version: "0.1.0".into(), owner_identity: Some("+15555550100".into()), owner_display_name: Some("Me".into()), + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "+15555550101".into(), diff --git a/crates/libs/mail/src/headers.rs b/crates/libs/mail/src/headers.rs index 4e46e1d13..50702809b 100644 --- a/crates/libs/mail/src/headers.rs +++ b/crates/libs/mail/src/headers.rs @@ -23,6 +23,8 @@ pub(crate) const EXPORT_SOURCE: &str = "X-ME-Export-Source"; pub(crate) const EXPORT_TOOL: &str = "X-ME-Export-Tool"; /// Export tool version. pub(crate) const EXPORT_TOOL_VERSION: &str = "X-ME-Export-Tool-Version"; +/// When the backup the export was read from was made, in Unix milliseconds. +pub(crate) const BACKUP_TAKEN_AT_UNIX_MS: &str = "X-ME-Backup-Taken-At-Unix-Ms"; /// Group chat title. pub(crate) const GROUP_TITLE: &str = "X-ME-Group-Title"; /// Conversation roster as JSON. diff --git a/crates/libs/mail/src/lib.rs b/crates/libs/mail/src/lib.rs index 722e6b3bb..7590e3a7b 100644 --- a/crates/libs/mail/src/lib.rs +++ b/crates/libs/mail/src/lib.rs @@ -112,6 +112,9 @@ pub struct MailMessage { pub export_tool: String, /// → `X-ME-Export-Tool-Version`. pub export_tool_version: String, + /// When the backup was made, in Unix milliseconds → + /// `X-ME-Backup-Taken-At-Unix-Ms`; `None` when the export does not say. + pub backup_taken_at_unix_ms: Option, /// Optional stem suffix (e.g. `"__whatsapp"`) for conversation directory / mbox names. pub filename_suffix: Option, /// The message itself (headers read guid, timestamp, direction, service, @@ -911,6 +914,10 @@ fn conversation_headers<'m>( msg.message.sender_display_name.clone(), ), (headers::OWNER_IDENTITY, Some(msg.owner_identity.clone())), + ( + headers::BACKUP_TAKEN_AT_UNIX_MS, + msg.backup_taken_at_unix_ms.map(|ms| ms.to_string()), + ), (headers::OWNER_DISPLAY_NAME, msg.owner_display_name.clone()), ( headers::MESSAGE_OWNER_IDENTITY, diff --git a/crates/libs/mail/src/parse.rs b/crates/libs/mail/src/parse.rs index 5b0c2dc12..17854ba1c 100644 --- a/crates/libs/mail/src/parse.rs +++ b/crates/libs/mail/src/parse.rs @@ -108,6 +108,7 @@ pub fn mail_message_from_eml_bytes(bytes: &[u8]) -> Result { let export_source = optional_header(headers, hn::EXPORT_SOURCE).unwrap_or_default(); let export_tool = optional_header(headers, hn::EXPORT_TOOL).unwrap_or_default(); let export_tool_version = optional_header(headers, hn::EXPORT_TOOL_VERSION).unwrap_or_default(); + let backup_taken_at_unix_ms = parse_backup_taken_at(headers)?; let text = extract_text_body(&mail).unwrap_or_default(); let attachments = merge_attachments(&mail, headers)?; @@ -160,6 +161,7 @@ pub fn mail_message_from_eml_bytes(bytes: &[u8]) -> Result { export_source, export_tool, export_tool_version, + backup_taken_at_unix_ms, filename_suffix: None, message: IrMessage { guid, @@ -323,6 +325,18 @@ fn header_u32(headers: &[MailHeader<'_>], name: &str) -> Option { typed_header(headers, name)?.parse().ok() } +/// When the backup was made, from `X-ME-Backup-Taken-At-Unix-Ms`, or none +/// when the header is absent. A value that is not a whole number is refused +/// rather than read as no date. +fn parse_backup_taken_at(headers: &[MailHeader<'_>]) -> Result> { + let Some(raw) = typed_header(headers, hn::BACKUP_TAKEN_AT_UNIX_MS) else { + return Ok(None); + }; + raw.parse() + .map(Some) + .with_context(|| format!("This mail's {} header {raw:?}", hn::BACKUP_TAKEN_AT_UNIX_MS)) +} + /// The message's mark from `X-ME-Deletion`, or none when the header is /// absent. A value that names neither mark is refused rather than dropped. fn parse_deletion(headers: &[MailHeader<'_>]) -> Result> { @@ -469,6 +483,7 @@ mod tests { export_source: "sms-backup-restore".into(), export_tool: "SMS Backup & Restore".into(), export_tool_version: "10.26.003".into(), + backup_taken_at_unix_ms: None, filename_suffix: None, message: IrMessage { guid: "aabbccddeeff00112233445566778899".into(), @@ -586,6 +601,7 @@ mod tests { export_source: "imessage".into(), export_tool: "imessage-exporter".into(), export_tool_version: "3.1.0".into(), + backup_taken_at_unix_ms: None, filename_suffix: None, message: IrMessage { guid: "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE".into(), @@ -713,6 +729,7 @@ mod tests { export_source: "imessage".into(), export_tool: "imessage-exporter".into(), export_tool_version: "3.1.0".into(), + backup_taken_at_unix_ms: None, filename_suffix: None, message: IrMessage { guid: "11111111-2222-3333-4444-555555555555".into(), diff --git a/crates/libs/mail/src/tests.rs b/crates/libs/mail/src/tests.rs index 507ba644d..f21893cc9 100644 --- a/crates/libs/mail/src/tests.rs +++ b/crates/libs/mail/src/tests.rs @@ -15,6 +15,7 @@ fn base_sms() -> MailMessage { export_source: "sms-backup-restore".into(), export_tool: "SMS Backup & Restore".into(), export_tool_version: "10.26.003".into(), + backup_taken_at_unix_ms: None, filename_suffix: None, message: IrMessage { guid: "aabbccddeeff00112233445566778899".into(), @@ -912,6 +913,7 @@ fn x_me_values(msg: &MailMessage) -> serde_json::Value { "export_source": msg.export_source, "export_tool": msg.export_tool, "export_tool_version": msg.export_tool_version, + "backup_taken_at_unix_ms": msg.backup_taken_at_unix_ms, "message": msg.message, "attachment_names": msg.attachments.iter().map(|a| &a.meta.original_name).collect::>(), "attachment_types": msg.attachments.iter().map(|a| &a.meta.mime_type).collect::>(), diff --git a/crates/libs/push/src/project.rs b/crates/libs/push/src/project.rs index 48131145e..d7e493e2a 100644 --- a/crates/libs/push/src/project.rs +++ b/crates/libs/push/src/project.rs @@ -138,6 +138,7 @@ mod tests { tool_version: "10.26.003".into(), owner_identity: Some("+15555550100".into()), owner_display_name: Some("Me".into()), + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "+15555550101".into(), @@ -154,7 +155,7 @@ mod tests { packaging_stem_suffix: None, }; let header = String::from_utf8(document_header_line(&doc).unwrap()).unwrap(); - assert!(header.contains(r#""schema_version":10"#)); + assert!(header.contains(r#""schema_version":11"#)); assert!(header.contains(r#""sms-backup-restore""#)); assert!(!header.contains(r#""record":"conversation""#)); diff --git a/crates/libs/push/tests/push_mock.rs b/crates/libs/push/tests/push_mock.rs index bee436f8c..93d26f719 100644 --- a/crates/libs/push/tests/push_mock.rs +++ b/crates/libs/push/tests/push_mock.rs @@ -34,6 +34,7 @@ fn sample_doc() -> ConversationDocument { tool_version: "10.26.003".into(), owner_identity: Some("+15555550100".into()), owner_display_name: Some("Me".into()), + backup_taken_at_unix_ms: None, }, conversation: ConversationMeta { chat_identifier: "+15555550101".into(), diff --git a/crates/libs/sbr/src/lib.rs b/crates/libs/sbr/src/lib.rs index bb0227c19..550d96845 100644 --- a/crates/libs/sbr/src/lib.rs +++ b/crates/libs/sbr/src/lib.rs @@ -67,6 +67,7 @@ pub struct SbrBackupWriter { body: BufWriter, count: u64, characters_left_out: u64, + backup_date_unix_ms: Option, } impl SbrBackupWriter { @@ -94,6 +95,7 @@ impl SbrBackupWriter { body, count: 0, characters_left_out: 0, + backup_date_unix_ms: None, }) } @@ -107,6 +109,15 @@ impl SbrBackupWriter { self.characters_left_out } + /// Note when the backup of messages written to this file was made, in + /// Unix milliseconds. The file's root `backup_date` says the newest date + /// noted, as the reader dates a conversation read from two files by the + /// newer, and is left out when none was noted. Without it, the reader + /// would date the file by when it was written. + pub fn note_backup_date(&mut self, unix_ms: Option) { + self.backup_date_unix_ms = self.backup_date_unix_ms.max(unix_ms); + } + /// Serialize one SMS/MMS element into the sidecar body file and increment /// the count. /// @@ -130,7 +141,8 @@ impl SbrBackupWriter { Ok(()) } - /// Finalize `count`, close ``, and replace `path`. + /// Finalize `count` and `backup_date`, close ``, and replace + /// `path`. /// /// # Errors /// @@ -154,7 +166,11 @@ impl SbrBackupWriter { out, r"" )?; - writeln!(out, r#""#, self.count)?; + let backup_date = self + .backup_date_unix_ms + .map(|date| format!(r#" backup_date="{date}""#)) + .unwrap_or_default(); + writeln!(out, r#""#, self.count)?; // Every element written to the body ends with a line break, so // the closing tag starts a line of its own. io::copy(&mut body, &mut out) diff --git a/crates/libs/sbr/src/read.rs b/crates/libs/sbr/src/read.rs index de4f5472b..a03470fb3 100644 --- a/crates/libs/sbr/src/read.rs +++ b/crates/libs/sbr/src/read.rs @@ -147,9 +147,14 @@ pub struct Record { pub dropped_character_references: u64, } -/// Counters for seen and skipped messages. +/// Counters for seen and skipped messages, and when the file's backup was +/// made. #[derive(Debug, Default, Clone, Copy)] pub struct ParseStats { + /// The root `` element's `backup_date`: when SMS Backup & Restore + /// made the backup, in Unix milliseconds. `None` when the file has no + /// such attribute or it is not a number. + pub backup_date_unix_ms: Option, /// Number of `` elements encountered. pub sms_seen: u64, /// Number of `` elements encountered. @@ -827,6 +832,7 @@ where loop { match xml.read_event_into(&mut buf) { Ok(Event::Start(e)) => match e.name().as_ref().to_ascii_lowercase().as_str() { + "smses" => stats.backup_date_unix_ms = backup_date(&attrs(&e, &mut 0)), "sms" => { dropped = 0; sms = attrs(&e, &mut dropped); @@ -842,6 +848,7 @@ where _ => {} }, Ok(Event::Empty(e)) => match e.name().as_ref().to_ascii_lowercase().as_str() { + "smses" => stats.backup_date_unix_ms = backup_date(&attrs(&e, &mut 0)), "sms" => { let mut own = 0; let attrs = attrs(&e, &mut own); @@ -888,6 +895,12 @@ where Ok(()) } +/// The root element's `backup_date` in Unix milliseconds, as SMS Backup & +/// Restore writes it, or `None` when it is missing or not a number. +fn backup_date(attrs: &HashMap) -> Option { + get(attrs, "backup_date").trim().parse().ok() +} + #[cfg(test)] fn parse_reader( reader: R, @@ -1321,6 +1334,45 @@ mod tests { assert_eq!(records[0].text, "line1\nline2\ttab"); } + /// The newest backup date noted is written as the root `backup_date`, + /// which the reader reads back, and a file with none noted has none. + #[test] + fn the_backup_date_noted_survives_a_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let written = |name: &str, dates: &[Option]| { + let mut writer = crate::SbrBackupWriter::create(&dir.path().join(name)).unwrap(); + for &date in dates { + writer.note_backup_date(date); + } + let attrs: BTreeMap = [ + ("protocol", "0"), + ("address", "+15555550101"), + ("date", "1"), + ("type", "1"), + ("body", "hi"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + writer + .write_message(&crate::SbrMessage::sms(attrs)) + .unwrap(); + let path = writer.finish().unwrap(); + parse_reader(std::fs::read(&path).unwrap().as_slice(), None) + .unwrap() + .1 + .backup_date_unix_ms + }; + assert_eq!( + written( + "dated.xml", + &[Some(1_788_256_800_000), None, Some(1_790_793_912_000)] + ), + Some(1_790_793_912_000) + ); + assert_eq!(written("undated.xml", &[None]), None); + } + #[test] fn a_literal_line_break_in_an_attribute_is_kept() { let xml = b""; diff --git a/crates/server/demo-seed/src/conversations.rs b/crates/server/demo-seed/src/conversations.rs index 4d8433b8b..c5d810dbb 100644 --- a/crates/server/demo-seed/src/conversations.rs +++ b/crates/server/demo-seed/src/conversations.rs @@ -80,14 +80,17 @@ const PHOTO_CAPTIONS: &[&str] = &[ const EMOJI_ONLY: &[&str] = &["👍", "😂", "❤️", "🎉", "😊"]; /// Export metadata stamped on a conversation header. `owner_identity` is the -/// Demo Account's identity the conversation's messages are held at. -fn export_meta(source: &str, owner_identity: &str) -> ExportMeta { +/// Demo Account's identity the conversation's messages are held at, and +/// `backup_taken_at_unix_ms` when the backup was made: the settings' +/// reference time, which every generated message is before. +fn export_meta(source: &str, owner_identity: &str, backup_taken_at_unix_ms: i64) -> ExportMeta { ExportMeta { source: source.into(), tool: "demo-seed".into(), tool_version: "0.2.0".into(), owner_identity: Some(owner_identity.into()), owner_display_name: Some("Me".into()), + backup_taken_at_unix_ms: Some(backup_taken_at_unix_ms), } } @@ -194,6 +197,7 @@ impl Seeder<'_, R> { IrConversationType::Individual, &[], IMESSAGE_SOURCE, + self.cfg.reference_time.timestamp_millis(), )?; self.stats.conversation_files += 1; } @@ -204,6 +208,7 @@ impl Seeder<'_, R> { IrConversationType::Group, &EMPTY_GROUP_MEMBERS, IMESSAGE_SOURCE, + self.cfg.reference_time.timestamp_millis(), )?; self.stats.conversation_files += 1; } @@ -387,7 +392,11 @@ impl Seeder<'_, R> { None, participants, msg_count, - export_meta(source_id(flavor), OWNER_PHONE), + export_meta( + source_id(flavor), + OWNER_PHONE, + self.cfg.reference_time.timestamp_millis(), + ), )?; let timestamps = self.timestamps(msg_count, spec.span_years, sample_direct_day_burst); @@ -479,7 +488,11 @@ impl Seeder<'_, R> { None, individual_participants(chat_id, overlap.display_name.clone()), overlap.msg_count, - export_meta(IMESSAGE_SOURCE, OWNER_PHONE), + export_meta( + IMESSAGE_SOURCE, + OWNER_PHONE, + self.cfg.reference_time.timestamp_millis(), + ), )?; let mut origin_guid: Option = None; for (i, shared) in overlap.shared.iter().enumerate() { @@ -534,7 +547,11 @@ impl Seeder<'_, R> { None, individual_participants(chat_id, overlap.display_name.clone()), android_total, - export_meta(SBR_SOURCE, OWNER_PHONE), + export_meta( + SBR_SOURCE, + OWNER_PHONE, + self.cfg.reference_time.timestamp_millis(), + ), )?; for (i, shared) in overlap.shared.iter().enumerate() { let msg = shared.message( @@ -621,7 +638,11 @@ impl Seeder<'_, R> { None, participants, msg_count, - export_meta(IMESSAGE_SOURCE, owner_identity), + export_meta( + IMESSAGE_SOURCE, + owner_identity, + self.cfg.reference_time.timestamp_millis(), + ), )?; let timestamps = self.timestamps(msg_count, 1.5, sample_direct_day_burst); @@ -683,7 +704,11 @@ impl Seeder<'_, R> { group.title.clone(), participants, header_message_count, - export_meta(IMESSAGE_SOURCE, OWNER_PHONE), + export_meta( + IMESSAGE_SOURCE, + OWNER_PHONE, + self.cfg.reference_time.timestamp_millis(), + ), )?; if let Some(title) = rename_title { @@ -835,7 +860,11 @@ impl Seeder<'_, R> { None, sender.into_iter().collect(), messages.len(), - export_meta(IMESSAGE_SOURCE, OWNER_PHONE), + export_meta( + IMESSAGE_SOURCE, + OWNER_PHONE, + self.cfg.reference_time.timestamp_millis(), + ), )?; for msg in messages { self.emit(&mut file, msg)?; @@ -857,6 +886,7 @@ fn write_header_only( conv_type: IrConversationType, member_phones: &[&str], source: &str, + backup_taken_at_unix_ms: i64, ) -> Result<()> { let path = staging.join(format!("empty-{}.jsonl", sanitize_filename(chat_id))); let mut file = open_jsonl(&path)?; @@ -875,7 +905,7 @@ fn write_header_only( None, participants, 0, - export_meta(source, OWNER_PHONE), + export_meta(source, OWNER_PHONE, backup_taken_at_unix_ms), )?; Ok(()) } diff --git a/crates/server/server/src/accounts_api.rs b/crates/server/server/src/accounts_api.rs index e9a598df9..93c719fd7 100644 --- a/crates/server/server/src/accounts_api.rs +++ b/crates/server/server/src/accounts_api.rs @@ -1301,9 +1301,9 @@ pub(crate) enum AccountImportRuns { #[serde(untagged)] pub(crate) enum AccountImportRun { /// The account's own run. - Own(ImportRun), + Own(Box), /// Another account's run, as the owner reads it. - Owner(OwnerImportRun), + Owner(Box), } /// An account's Export Runs as its reader may see them: in full for the @@ -1382,10 +1382,10 @@ pub(crate) async fn get_account_import( .await .map_err(ApiError::from)?; let run = crate::imports_api::owner_import_run(&mut conn, row).await?; - return Ok(Json(AccountImportRun::Owner(run))); + return Ok(Json(AccountImportRun::Owner(Box::new(run)))); } let run = crate::imports_api::full_import_run(&mut conn, target, import_id).await?; - Ok(Json(AccountImportRun::Own(run))) + Ok(Json(AccountImportRun::Own(Box::new(run)))) } /// An account's Export Runs as a page, newest first unless `sort` says diff --git a/crates/server/server/src/cli/tests.rs b/crates/server/server/src/cli/tests.rs index 42f8d6447..e6533584d 100644 --- a/crates/server/server/src/cli/tests.rs +++ b/crates/server/server/src/cli/tests.rs @@ -431,6 +431,7 @@ fn imports_discard_prints_the_import_run_or_that_there_was_none() { form_json: None, source_fingerprint: None, source_identities: None, + backup_taken_at: None, }; assert_eq!( format_discarded_import("alice", Some(&row)), diff --git a/crates/server/server/src/db/conversation_messages.rs b/crates/server/server/src/db/conversation_messages.rs index 29ab268df..3ce6af2eb 100644 --- a/crates/server/server/src/db/conversation_messages.rs +++ b/crates/server/server/src/db/conversation_messages.rs @@ -54,6 +54,7 @@ struct RawRow { reply_to: Option, reply_count: i64, deletion: Option, + backup_taken_at: Option, chat_identifier: String, conversation_type: String, group_title: Option, @@ -474,7 +475,7 @@ fn message_page_sql( m.is_announcement, m.is_reply, m.reply_to_guid, m.reply_to_part, ({reply_count}) AS reply_count, hc.raw AS chat_identifier, c.conversation_type, c.group_title, - ho.raw AS owner, {label} AS label, m.deletion + ho.raw AS owner, {label} AS label, m.deletion, m.backup_taken_at {from_sql} WHERE {where_sql} ORDER BY {order_by} LIMIT ? OFFSET ?", @@ -527,6 +528,7 @@ async fn fetch_message_page( owner: row.try_get(19)?, label: row.try_get(20)?, deletion: row.try_get(21)?, + backup_taken_at: row.try_get(22)?, }) }) .collect::, ApiError>>()?; @@ -574,6 +576,7 @@ async fn fetch_message_page( // The column's CHECK admits only the two marks or NULL. deletion: r.deletion.as_deref().and_then(Deletion::parse), earlier_versions: earlier_versions.remove(&r.id).unwrap_or_default(), + backup_taken_at: r.backup_taken_at, matched_earlier_version: false, } }) diff --git a/crates/server/server/src/db/imports.rs b/crates/server/server/src/db/imports.rs index 7714e9d5a..619e71cab 100644 --- a/crates/server/server/src/db/imports.rs +++ b/crates/server/server/src/db/imports.rs @@ -200,6 +200,9 @@ pub struct ImportRow { pub source_fingerprint: Option, /// Addresses the backup's device sent from (JSON array). pub source_identities: Option, + /// When the backup the run read was made: the newest its conversation + /// files name, in the form a message's timestamp takes. + pub backup_taken_at: Option, } /// Outcome fields written when a run completes. @@ -448,7 +451,7 @@ fn is_unique_violation(err: &sqlx::Error) -> bool { const IMPORT_COLUMNS: &str = "id, account_id, source, tool, mode, status, started_at, \ finished_at, message_count, attachment_count, bytes_uploaded, duration_ms, parse_ms, \ attachments_ms, prepare_ms, upload_ms, summary_json, stage, run_dir, device_id, \ - form_json, source_fingerprint, source_identities, dedupe"; + form_json, source_fingerprint, source_identities, dedupe, backup_taken_at"; /// Map one `imports` row by column position. fn import_from_row(row: &SqliteRow) -> Result { @@ -491,9 +494,34 @@ fn import_from_row(row: &SqliteRow) -> Result { source_fingerprint: row.try_get(21)?, source_identities: row.try_get(22)?, dedupe: row.try_get::(23)? != 0, + backup_taken_at: row.try_get(24)?, }) } +/// Record on the run `import_id` that one of its conversation files was read +/// from a backup made at `backup_taken_at`, in the form a message's timestamp +/// takes. The run keeps the newest such date, so a run that read two backups +/// shows the later. +/// +/// # Errors +/// +/// Returns an error when the update fails. +pub async fn note_backup_taken_at( + conn: &mut SqliteConnection, + import_id: i64, + backup_taken_at: &str, +) -> Result<()> { + sqlx::query( + "UPDATE imports SET backup_taken_at = $2 + WHERE id = $1 AND (backup_taken_at IS NULL OR backup_taken_at < $2)", + ) + .bind(import_id) + .bind(backup_taken_at) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// Load an import row owned by `account_id`, or error. pub async fn get_owned_import( conn: &mut SqliteConnection, @@ -1238,7 +1266,8 @@ pub async fn detach_from_account( .await?; sqlx::query( "UPDATE imports SET username = $2, deletion_entry_id = $3, form_json = NULL, run_dir = NULL, - source_fingerprint = NULL, source_identities = NULL, summary_json = NULL + source_fingerprint = NULL, source_identities = NULL, summary_json = NULL, + backup_taken_at = NULL WHERE account_id = $1", ) .bind(account_id) diff --git a/crates/server/server/src/db/schema.rs b/crates/server/server/src/db/schema.rs index 605b2f97f..e399b52e4 100644 --- a/crates/server/server/src/db/schema.rs +++ b/crates/server/server/src/db/schema.rs @@ -375,7 +375,8 @@ pub(crate) async fn create_messages_secondary_indexes(conn: &mut SqliteConnectio /// The stored messages, those at or below `$1` (`min_new_message_id`), whose /// index row a promotion removes and writes again: each one that gained an /// attachment above `$2` (`min_new_attachment_id`), took a later edit, or -/// is marked by a staged row. One set for the delete and the insert in +/// whose mark `staging::promote_deletion_marks` changed, set or cleared +/// (`_promote_mark_map`). One set for the delete and the insert in /// [`index_messages_fts_from_promote_map`], so a row removed is always a row /// considered for writing again. const STORED_MESSAGES_TO_REINDEX: &str = " @@ -384,9 +385,7 @@ const STORED_MESSAGES_TO_REINDEX: &str = " UNION SELECT prod_id FROM _promote_edit_map UNION - SELECT pm.prod_id FROM _promote_msg_map pm - JOIN staging_messages sm ON sm.id = pm.staging_id - WHERE sm.deletion IS NOT NULL AND pm.prod_id <= $1"; + SELECT prod_id FROM _promote_mark_map WHERE prod_id <= $1"; /// Bulk-index promoted messages (joined via temp `_promote_msg_map`). /// Call after attachment rows exist so `attachment_text` is complete. @@ -407,14 +406,14 @@ const STORED_MESSAGES_TO_REINDEX: &str = " /// attachment above it was inserted by this promotion. The same goes for an /// existing message that took a later edit, which `_promote_edit_map` /// names: its index row holds the text the edit replaced, and for an -/// existing message a staged row marks, whose mark -/// `staging::promote_deletion_marks` may have changed. +/// existing message whose mark `staging::promote_deletion_marks` changed, +/// which `_promote_mark_map` names. /// /// An Unsent message is given no index row, as the sync triggers give it /// none (`fts_triggers_create.sql`): it shows none of its text, so a word of /// that text must not find it (#1758). A stored message this promotion marks /// Unsent loses its row here, and one whose Unsent mark becomes Deleted in -/// the source app is written again. +/// the source app, or is cleared by a later backup, is written again. pub(crate) async fn index_messages_fts_from_promote_map( conn: &mut SqliteConnection, min_new_message_id: i64, diff --git a/crates/server/server/src/db/schema/tests.rs b/crates/server/server/src/db/schema/tests.rs index 4194b6443..59f6565ee 100644 --- a/crates/server/server/src/db/schema/tests.rs +++ b/crates/server/server/src/db/schema/tests.rs @@ -98,6 +98,10 @@ async fn promote_fts_indexing_covers_only_rows_inserted_by_this_promotion() { staging_id INTEGER PRIMARY KEY, prod_id INTEGER NOT NULL ); + CREATE TEMP TABLE _promote_mark_map ( + staging_id INTEGER PRIMARY KEY, + prod_id INTEGER NOT NULL + ); INSERT INTO _promote_msg_map (staging_id, prod_id) VALUES (1, 10), (2, 11), (3, 11); ", ) @@ -173,6 +177,10 @@ async fn promote_fts_indexing_reindexes_an_existing_message_that_gained_an_attac staging_id INTEGER PRIMARY KEY, prod_id INTEGER NOT NULL ); + CREATE TEMP TABLE _promote_mark_map ( + staging_id INTEGER PRIMARY KEY, + prod_id INTEGER NOT NULL + ); INSERT INTO _promote_msg_map (staging_id, prod_id) VALUES (1, 10), (2, 12); INSERT INTO attachments (message_id, original_name) VALUES (10, 'lateinvoice.pdf'); ", diff --git a/crates/server/server/src/db/staging.rs b/crates/server/server/src/db/staging.rs index 7f6f0f55f..424a6f9e9 100644 --- a/crates/server/server/src/db/staging.rs +++ b/crates/server/server/src/db/staging.rs @@ -3,8 +3,8 @@ //! `staging_conversations`, `staging_participants`, `staging_messages`, //! `staging_attachments`, `staging_tapbacks` and `staging_message_versions`, //! and the temp id maps -//! (`_promote_conv_map`, `_promote_msg_map`, `_promote_edit_map`) the -//! promotion joins through. +//! (`_promote_conv_map`, `_promote_msg_map`, `_promote_mark_map`, +//! `_promote_edit_map`) the promotion joins through. //! //! `imports_api::staging` fills the tables and `imports_api::promote` runs //! the promotion. Those stages sequence the statements, log them and keep @@ -229,6 +229,9 @@ pub struct StagingMessage<'a> { pub sort_order: i64, /// Import run that staged the row. pub import_id: Option, + /// When the backup the row was read from was made, in the form + /// `timestamp` takes; `None` when its file did not say. + pub backup_taken_at: Option<&'a str>, } /// One attachment row as the import stages it: the stored blob's digest, @@ -341,7 +344,7 @@ const TAPBACK_COLUMNS: &[&str] = &[ ]; /// Bind counts, in lockstep with the `INSERT` column lists below. -const MESSAGE_BIND_COLUMNS: usize = 18; +const MESSAGE_BIND_COLUMNS: usize = 19; const ATTACHMENT_BIND_COLUMNS: usize = ATTACHMENT_COLUMNS.len(); const TAPBACK_BIND_COLUMNS: usize = TAPBACK_COLUMNS.len(); const EARLIER_VERSION_BIND_COLUMNS: usize = 4; @@ -368,7 +371,7 @@ pub async fn insert_messages( INSERT INTO staging_messages ( conversation_id, account_id, source, guid, timestamp, is_from_me, sender_handle_id, owner_handle_id, service, subject, body, is_announcement, is_reply, - reply_to_guid, reply_to_part, deletion, sort_order, import_id + reply_to_guid, reply_to_part, deletion, sort_order, import_id, backup_taken_at ) VALUES {} ON CONFLICT DO NOTHING RETURNING id, sort_order @@ -395,7 +398,8 @@ pub async fn insert_messages( .bind(row.reply_to_part) .bind(row.deletion.map(message_ir::Deletion::as_str)) .bind(row.sort_order) - .bind(row.import_id); + .bind(row.import_id) + .bind(row.backup_taken_at); } let returned = q.fetch_all(&mut *conn).await?; let mut by_sort = HashMap::with_capacity(returned.len()); @@ -554,10 +558,102 @@ pub async fn staged_message_id( .await?) } +/// What a second copy of a staged message from the same import decides +/// when its backup and the staged row's both have a date: everything the +/// later backup says about the message's mark and text. +pub struct StagedCopy<'a> { + /// The copy's text. + pub body: Option<&'a str>, + /// The copy's mark, or `None` for none. + pub deletion: Option, + /// The copy's earlier versions, in the order its file listed them. + pub versions: &'a [crate::models::EarlierVersionRecord], + /// When the copy's backup was made. + pub backup_taken_at: &'a str, +} + +/// When the backup the staged message `staged` was read from was made, or +/// `None` when its file did not say. +/// +/// # Errors +/// +/// Returns an error when the query fails. +pub async fn staged_backup_taken_at( + conn: &mut SqliteConnection, + staged: i64, +) -> Result> { + Ok( + sqlx::query_scalar("SELECT backup_taken_at FROM staging_messages WHERE id = $1") + .bind(staged) + .fetch_one(&mut *conn) + .await?, + ) +} + +/// Give the staged message `staged` everything `copy`, another copy of it +/// from a later backup in the same import, says about its mark and text: +/// its text, its earlier versions, its mark or no mark, and its backup's +/// date. The caller has compared the two backups' dates +/// ([`later_backup_sql`]'s rule), so the copy staged first no longer +/// decides whatever its age (#1741, #1804). +/// +/// # Errors +/// +/// Returns an error when a statement fails. +pub async fn take_staged_copy_from_later_backup( + conn: &mut SqliteConnection, + staged: i64, + copy: &StagedCopy<'_>, +) -> Result<()> { + sqlx::query( + "UPDATE staging_messages SET body = $1, deletion = $2, backup_taken_at = $3 WHERE id = $4", + ) + .bind(copy.body) + .bind(copy.deletion.map(message_ir::Deletion::as_str)) + .bind(copy.backup_taken_at) + .bind(staged) + .execute(&mut *conn) + .await?; + sqlx::query("DELETE FROM staging_message_versions WHERE message_id = $1") + .bind(staged) + .execute(&mut *conn) + .await?; + let rows: Vec> = copy + .versions + .iter() + .map(|version| StagingEarlierVersion::from_record(staged, version)) + .collect(); + insert_earlier_versions(conn, &rows).await?; + Ok(()) +} + +/// Give the staged message `staged` the mark `deletion` of another copy of +/// it from the same import, when the two backups' dates cannot decide +/// ([`later_backup`]): a copy that carries a mark adds it, and one with +/// none leaves the staged mark, as [`promote_deletion_marks`] does for a +/// stored message. +/// +/// # Errors +/// +/// Returns an error when the update fails. +pub async fn add_staged_copy_mark( + conn: &mut SqliteConnection, + staged: i64, + deletion: message_ir::Deletion, +) -> Result<()> { + sqlx::query("UPDATE staging_messages SET deletion = $1 WHERE id = $2") + .bind(deletion.as_str()) + .bind(staged) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// Give the staged message `staged` the text `body` and the earlier /// versions `versions` of another copy of it from the same import, when /// that copy records a later edit ([`later_edit_sql`]). Returns whether it -/// did. +/// did. The caller uses it only when one of the two backups has no date; +/// otherwise [`take_staged_copy_from_later_backup`] decides. /// /// Staging keeps one row per guid and skips a second copy, so without this /// the copy staged first counted whatever its age: one import of an @@ -976,12 +1072,13 @@ const INSERT_MESSAGES_FROM_STAGING: &str = r" INSERT INTO messages ( conversation_id, account_id, source, guid, timestamp, is_from_me, sender_handle_id, owner_handle_id, service, subject, body, is_announcement, is_reply, - reply_to_guid, reply_to_part, deletion, sort_order, import_id + reply_to_guid, reply_to_part, deletion, sort_order, import_id, backup_taken_at ) SELECT cm.prod_id, sm.account_id, sm.source, sm.guid, sm.timestamp, sm.is_from_me, sm.sender_handle_id, sm.owner_handle_id, sm.service, sm.subject, sm.body, sm.is_announcement, sm.is_reply, - sm.reply_to_guid, sm.reply_to_part, sm.deletion, sm.sort_order, sm.import_id + sm.reply_to_guid, sm.reply_to_part, sm.deletion, sm.sort_order, sm.import_id, + sm.backup_taken_at FROM staging_messages sm JOIN _promote_conv_map cm ON cm.staging_id = sm.conversation_id WHERE sm.account_id = $1 @@ -1133,29 +1230,100 @@ pub async fn write_message_map( Ok(()) } +/// Whether a staged copy of a message comes from a later backup than the +/// copy held, as an SQL expression over the two backups' dates `staged` and +/// `held`: true when both have a date and the staged one is later, false +/// when both have one and it is earlier, and NULL when either has none or +/// the two are equal. NULL means the dates cannot decide, and the caller +/// falls back on the rule for files without a date. Equal dates are the +/// same backup read again, where that rule changes nothing because the two +/// copies agree, or two reads of one Mac's `chat.db` that Messages did +/// not write between, where it adds a mark and takes a later edit as it +/// would with no dates. +/// The one rule for which of two copies of a message from one source is +/// the later backup, for a stored message ([`promote_deletion_marks`], +/// [`write_edit_map`]) and, in Rust ([`later_backup`]), for two copies +/// staged in one import (`imports_api::staging`). +/// +/// Both dates have the one text form of a stored time, to the millisecond +/// (`models::utc_timestamp_text`), so the text orders as the time. +fn later_backup_sql(staged: &str, held: &str) -> String { + format!( + "CASE WHEN {staged} IS NOT NULL AND {held} IS NOT NULL AND {staged} <> {held} \ + THEN {staged} > {held} END" + ) +} + +/// Which of two copies of a message comes from the later backup, by +/// [`later_backup`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackupOrder<'a> { + /// The copy comes from a later backup, made at this date. + Later(&'a str), + /// The copy comes from an earlier backup. + Earlier, + /// Either copy has no date, or the two dates are equal: the dates + /// cannot decide, and the rules for files without one hold. + Undecided, +} + +/// [`later_backup_sql`]'s rule in Rust, for two copies of a message staged +/// in one import: whether the copy from the backup made at `staged` is +/// later than the copy held from the backup made at `held`. +#[must_use] +pub fn later_backup<'a>(staged: Option<&'a str>, held: Option<&str>) -> BackupOrder<'a> { + match (staged, held) { + (Some(staged), Some(held)) if staged > held => BackupOrder::Later(staged), + (Some(staged), Some(held)) if staged < held => BackupOrder::Earlier, + _ => BackupOrder::Undecided, + } +} + /// Give each stored message the mark its staged row carries, through /// `_promote_msg_map`. An append-mode import skips a message production /// already holds, so a message imported before it was deleted or unsent -/// takes the mark only here. A staged row with no mark leaves the stored -/// mark as it is: a backup that does not say a message was deleted does not -/// say it was restored. The search index follows the mark when the -/// promotion indexes (`schema::index_messages_fts_from_promote_map`): a -/// message marked Unsent loses its index row, because it shows none of its -/// text (#1758). Returns how many messages changed. +/// takes the mark only here. +/// +/// When both the staged row's backup and the stored message's have a date +/// ([`later_backup_sql`]), the later backup decides: a staged row from a +/// later backup gives its mark or clears the one held, and one from an +/// earlier backup changes nothing. When either has no date, or the two are +/// equal, a staged row with no mark leaves the stored mark as it is: a +/// backup that does not say a message was deleted does not say it was +/// restored, and nothing says which backup is newer. +/// +/// Each message whose mark changes is named in `_promote_mark_map`, so the +/// search index follows the mark when the promotion indexes +/// (`schema::index_messages_fts_from_promote_map`): a message marked Unsent +/// loses its index row, because it shows none of its text (#1758), and one +/// whose Unsent mark a later backup clears or turns into Deleted in the +/// source app has it written again. Returns how many messages changed. /// /// # Errors /// /// Returns an error when the update fails. pub async fn promote_deletion_marks(conn: &mut SqliteConnection) -> Result { + reset_id_map(conn, "_promote_mark_map").await?; + let sql = format!( + r" + INSERT INTO _promote_mark_map (staging_id, prod_id) + SELECT mm.staging_id, mm.prod_id + FROM _promote_msg_map mm + JOIN staging_messages sm ON sm.id = mm.staging_id + JOIN messages m ON m.id = mm.prod_id + WHERE m.deletion IS NOT sm.deletion + AND COALESCE({later}, sm.deletion IS NOT NULL) + ", + later = later_backup_sql("sm.backup_taken_at", "m.backup_taken_at"), + ); + sqlx::query(&sql).execute(&mut *conn).await?; Ok(sqlx::query( r" UPDATE messages SET deletion = sm.deletion - FROM _promote_msg_map mm - JOIN staging_messages sm ON sm.id = mm.staging_id - WHERE messages.id = mm.prod_id - AND sm.deletion IS NOT NULL - AND messages.deletion IS NOT sm.deletion + FROM _promote_mark_map pm + JOIN staging_messages sm ON sm.id = pm.staging_id + WHERE messages.id = pm.prod_id ", ) .execute(&mut *conn) @@ -1163,6 +1331,30 @@ pub async fn promote_deletion_marks(conn: &mut SqliteConnection) -> Result .rows_affected()) } +/// Give each stored message the backup date of its staged row when that +/// row comes from a later backup ([`later_backup_sql`]), after its mark +/// and text were taken from that row, so a later import compares with the +/// backup the message now reflects. A stored message from a file with no +/// date keeps none. Returns how many messages changed. +/// +/// # Errors +/// +/// Returns an error when the update fails. +pub async fn promote_backup_dates(conn: &mut SqliteConnection) -> Result { + let sql = format!( + r" + UPDATE messages + SET backup_taken_at = sm.backup_taken_at + FROM _promote_msg_map mm + JOIN staging_messages sm ON sm.id = mm.staging_id + WHERE messages.id = mm.prod_id + AND COALESCE({later}, 0) + ", + later = later_backup_sql("sm.backup_taken_at", "messages.backup_taken_at"), + ); + Ok(sqlx::query(&sql).execute(&mut *conn).await?.rows_affected()) +} + /// Whether one copy of a message records a later edit than another, as an /// SQL expression over four SQL values: the copy's earlier-version count /// `n` and newest `edited_at` `newest`, and the other copy's `held_n` and @@ -1181,10 +1373,11 @@ pub async fn promote_deletion_marks(conn: &mut SqliteConnection) -> Result /// The newest earlier version is the edit before the last one: the time of /// a part's last edit is recorded nowhere. So a later backup that differs /// only by an unsent part, or by one edit after an unsend, can read as not -/// later (#1804). +/// later (#1804). The rule is used only where one of the two backups has no +/// date; where both have one, [`later_backup_sql`] decides instead. /// -/// `edited_at` is one fixed whole-second UTC form on both sides -/// (`models::earlier_version_from_ir`), so the text orders as the time. +/// `edited_at` has the one text form of a stored time on both sides +/// (`models::utc_timestamp_text`), so the text orders as the time. fn later_edit_sql(n: &str, newest: &str, held_n: &str, held_newest: &str) -> String { format!( "CASE \ @@ -1196,20 +1389,34 @@ fn later_edit_sql(n: &str, newest: &str, held_n: &str, held_newest: &str) -> Str } /// Write `_promote_edit_map`: each message production held before this -/// promotion, those at or below `messages_before`, whose staged row records -/// a later edit than the message holds ([`later_edit_sql`]). Returns how -/// many messages it names. +/// promotion, those at or below `messages_before`, whose staged row gives +/// it a later text. Returns how many messages it names. +/// +/// When both backups have a date ([`later_backup_sql`]), a staged row from +/// a later backup gives its text and earlier versions whatever their times +/// say, when either differs from what the message holds (#1804); one from +/// an earlier backup gives nothing. When either has no date, or the two +/// are equal, the staged row gives them when it records a later edit +/// ([`later_edit_sql`]). /// /// An append skips a message production already holds, so a later backup in /// which it was edited again reaches it only here. A message has one staged /// row: staging keeps one row per guid, the later copy when one import -/// carries two ([`take_later_staged_copy`]). +/// carries two ([`take_staged_copy_from_later_backup`], [`take_later_staged_copy`]). /// /// # Errors /// /// Returns an error when a statement fails. pub async fn write_edit_map(conn: &mut SqliteConnection, messages_before: i64) -> Result { reset_id_map(conn, "_promote_edit_map").await?; + // A version list as one value, in the order its rows were written, so + // two lists compare whole. + let versions = |table: &str, id: &str| { + format!( + "(SELECT json_group_array(json_array(v.part_index, v.text, v.edited_at) ORDER BY v.id) \ + FROM {table} v WHERE v.message_id = {id})" + ) + }; let sql = format!( r" INSERT INTO _promote_edit_map (staging_id, prod_id) @@ -1218,6 +1425,10 @@ pub async fn write_edit_map(conn: &mut SqliteConnection, messages_before: i64) - SELECT mm.staging_id, mm.prod_id, + {later_backup} AS later_backup, + sm.body IS NOT m.body + OR {staged_versions} IS NOT {held_versions} AS differs, + sv.message_id IS NOT NULL AS has_versions, sv.n, sv.newest, (SELECT COUNT(*) FROM message_versions v WHERE v.message_id = mm.prod_id) @@ -1225,16 +1436,24 @@ pub async fn write_edit_map(conn: &mut SqliteConnection, messages_before: i64) - (SELECT MAX(v.edited_at) FROM message_versions v WHERE v.message_id = mm.prod_id) AS held_newest FROM _promote_msg_map mm - JOIN ( + JOIN staging_messages sm ON sm.id = mm.staging_id + JOIN messages m ON m.id = mm.prod_id + LEFT JOIN ( SELECT message_id, COUNT(*) AS n, MAX(edited_at) AS newest FROM staging_message_versions GROUP BY message_id ) sv ON sv.message_id = mm.staging_id WHERE mm.prod_id <= $1 ) - WHERE {later} + WHERE CASE + WHEN later_backup IS NOT NULL THEN later_backup AND differs + ELSE has_versions AND {later_edit} + END ", - later = later_edit_sql("n", "newest", "held_n", "held_newest"), + later_backup = later_backup_sql("sm.backup_taken_at", "m.backup_taken_at"), + staged_versions = versions("staging_message_versions", "mm.staging_id"), + held_versions = versions("message_versions", "mm.prod_id"), + later_edit = later_edit_sql("n", "newest", "held_n", "held_newest"), ); Ok(sqlx::query(&sql) .bind(messages_before) diff --git a/crates/server/server/src/db/staging/tests.rs b/crates/server/server/src/db/staging/tests.rs index 98ed3d733..90522030c 100644 --- a/crates/server/server/src/db/staging/tests.rs +++ b/crates/server/server/src/db/staging/tests.rs @@ -44,6 +44,7 @@ async fn reset_for_account_leaves_other_accounts() { deletion: None, sort_order: 0, import_id: None, + backup_taken_at: None, }], ) .await diff --git a/crates/server/server/src/imports_api/mod.rs b/crates/server/server/src/imports_api/mod.rs index 444a046c0..5ad274a3d 100644 --- a/crates/server/server/src/imports_api/mod.rs +++ b/crates/server/server/src/imports_api/mod.rs @@ -919,6 +919,10 @@ pub(crate) struct ImportRunSummary { pub(crate) source_fingerprint: serde_json::Value, /// Addresses the backup's device sent from (JSON array), or null. pub(crate) source_identities: serde_json::Value, + /// When the backup the run read was made, UTC: the newest date its + /// conversation files name (`export.backup_taken_at_unix_ms`). Null until + /// a file that names one is imported, and for a run whose files name none. + pub(crate) backup_taken_at: Option, /// What the person approved at the last Review they passed, which /// `PATCH /v1/imports/{id}` writes with its `summary`. A cancelled run /// keeps it. A run that completed or failed holds instead the final @@ -962,6 +966,7 @@ impl From for ImportRunSummary { form: crate::db::imports::json_column(row.form_json), source_fingerprint: crate::db::imports::json_column(row.source_fingerprint), source_identities: crate::db::imports::json_column(row.source_identities), + backup_taken_at: row.backup_taken_at, summary: crate::db::imports::json_column(row.summary_json), issue_count: listed.issue_count, note_count: listed.note_count, diff --git a/crates/server/server/src/imports_api/promote.rs b/crates/server/server/src/imports_api/promote.rs index fd0314e3a..7a7575f8f 100644 --- a/crates/server/server/src/imports_api/promote.rs +++ b/crates/server/server/src/imports_api/promote.rs @@ -276,6 +276,17 @@ impl Promote<'_> { words(as_count(unindexed), "1 search entry", "{n} search entries"), ), ); + + let phase = Self::begin("Recording which backup each changed message came from…"); + let dated = staging::promote_backup_dates(self.tx).await?; + self.done( + phase, + words( + dated, + "1 message now holds a later backup", + "{n} messages now hold a later backup", + ), + ); Ok(messages_before) } diff --git a/crates/server/server/src/imports_api/staging.rs b/crates/server/server/src/imports_api/staging.rs index f44f36079..e89054f9d 100644 --- a/crates/server/server/src/imports_api/staging.rs +++ b/crates/server/server/src/imports_api/staging.rs @@ -12,8 +12,8 @@ use crate::db::handles::{ HandleIdCache, handle_type_on, upsert_handle_row, upsert_handle_row_cached, }; use crate::db::staging::{ - self as db_staging, StagingAttachment, StagingConversation, StagingEarlierVersion, - StagingMessage, StagingMessageKey, StagingTapback, + self as db_staging, BackupOrder, StagedCopy, StagingAttachment, StagingConversation, + StagingEarlierVersion, StagingMessage, StagingMessageKey, StagingTapback, }; use crate::import_media; use crate::jsonl::{self, ReadRecordsError}; @@ -367,6 +367,9 @@ struct StagedConversation { group_title: Option, participants: Vec, source: String, + /// When the backup the file was read from was made, in the form a + /// message's timestamp takes; `None` when the file does not say. + backup_taken_at: Option, } impl StagedConversation { @@ -384,6 +387,7 @@ impl StagedConversation { .map(|p| (p.handle, p.name_alias, p.handle_type)) .collect(), source, + backup_taken_at: record.backup_taken_at, } } } @@ -545,6 +549,12 @@ impl FileStaging<'_> { ) .await?; counts.conversations = 1; + if let (Some(import_id), Some(backup_taken_at)) = ( + self.stmts.import_id, + conversation.backup_taken_at.as_deref(), + ) { + crate::db::imports::note_backup_taken_at(self.tx, import_id, backup_taken_at).await?; + } for participant in conversation.participants { insert_participant( @@ -588,8 +598,11 @@ impl FileStaging<'_> { self.tx, self.stmts, &mut counts, - conversation_id, - &conversation.source, + StagedSource { + conversation_id, + source: &conversation.source, + backup_taken_at: conversation.backup_taken_at.as_deref(), + }, self.opts.assets_dir, chunk, ) @@ -826,21 +839,29 @@ struct PendingStagingMessage { sort_order: i64, } +/// Where one conversation's message rows are staged from: its staging +/// conversation, its source, and when its backup was made. +#[derive(Clone, Copy)] +struct StagedSource<'a> { + conversation_id: i64, + source: &'a str, + backup_taken_at: Option<&'a str>, +} + /// Bulk-insert one chunk of message rows, then their attachments, tapbacks /// and earlier versions keyed by the ids returned. async fn flush_staging_message_chunk( tx: &mut SqliteConnection, stmts: &mut StagingInserts, counts: &mut ImportCounts, - conversation_id: i64, - source: &str, + staged_source: StagedSource<'_>, assets_dir: &Path, chunk: &[PendingStagingMessage], ) -> Result<()> { if chunk.is_empty() { return Ok(()); } - let mut by_sort = insert_message_rows(tx, stmts, conversation_id, source, chunk).await?; + let mut by_sort = insert_message_rows(tx, stmts, staged_source, chunk).await?; let mut att_rows = Vec::new(); let mut tap_rows = Vec::new(); @@ -877,47 +898,78 @@ async fn flush_staging_message_chunk( counts.tapbacks += db_staging::insert_tapbacks(tx, &tap_rows).await?; db_staging::insert_earlier_versions(tx, &version_rows).await?; for row in copies { - add_staged_copy(tx, stmts, counts, source, assets_dir, row).await?; + add_staged_copy(tx, stmts, counts, staged_source, assets_dir, row).await?; } Ok(()) } /// Give the message staged under `row`'s guid what `row`, another copy of -/// it from the same import, adds: its text and earlier versions when it -/// records a later edit, and the attachments and reactions the staged -/// message does not hold yet. One import of two backups then stores what -/// two separate imports of them store (#1806, #1837). The copy's deletion -/// mark is not taken (#1741). +/// it from the same import, adds, by the rules a later import of the copy +/// would follow (`db::staging::promote_deletion_marks`, +/// `db::staging::write_edit_map`): the attachments and reactions the staged +/// message does not hold yet, and its mark and text as follows. When both +/// backups have a date, a copy from a later backup gives its text, earlier +/// versions and mark, mark or no mark, and one from an earlier backup gives +/// neither (#1741, #1804). When either has no date, or the two dates are +/// equal ([`db_staging::later_backup`]), the copy gives its text and +/// earlier versions when it records a later edit, and its mark when it +/// carries one. One import of two backups then stores +/// what two separate imports of them store, in either file order (#1806, +/// #1837). async fn add_staged_copy( tx: &mut SqliteConnection, stmts: &mut StagingInserts, counts: &mut ImportCounts, - source: &str, + staged_source: StagedSource<'_>, assets_dir: &Path, row: &PendingStagingMessage, ) -> Result<()> { if row.msg.earlier_versions.is_empty() && row.attachments.is_empty() && row.msg.tapbacks.is_empty() + && row.msg.deletion.is_none() + && staged_source.backup_taken_at.is_none() { return Ok(()); } let key = StagingMessageKey { account_id: stmts.account_id, - source, + source: staged_source.source, guid: &row.msg.guid, }; let staged = db_staging::staged_message_id(tx, key) .await? .with_context(|| format!("no staged message holds the copy of {}", row.msg.guid))?; - if !row.msg.earlier_versions.is_empty() { - db_staging::take_later_staged_copy( - tx, - staged, - row.body.as_deref(), - &row.msg.earlier_versions, - ) - .await?; + let held_backup = db_staging::staged_backup_taken_at(tx, staged).await?; + match db_staging::later_backup(staged_source.backup_taken_at, held_backup.as_deref()) { + BackupOrder::Later(copy_backup) => { + db_staging::take_staged_copy_from_later_backup( + tx, + staged, + &StagedCopy { + body: row.body.as_deref(), + deletion: row.msg.deletion, + versions: &row.msg.earlier_versions, + backup_taken_at: copy_backup, + }, + ) + .await?; + } + BackupOrder::Earlier => {} + BackupOrder::Undecided => { + if !row.msg.earlier_versions.is_empty() { + db_staging::take_later_staged_copy( + tx, + staged, + row.body.as_deref(), + &row.msg.earlier_versions, + ) + .await?; + } + if let Some(deletion) = row.msg.deletion { + db_staging::add_staged_copy_mark(tx, staged, deletion).await?; + } + } } let att_rows: Vec = row .attachments @@ -938,16 +990,15 @@ async fn add_staged_copy( async fn insert_message_rows( tx: &mut SqliteConnection, stmts: &StagingInserts, - conversation_id: i64, - source: &str, + staged_source: StagedSource<'_>, chunk: &[PendingStagingMessage], ) -> Result> { let rows: Vec> = chunk .iter() .map(|row| StagingMessage { - conversation_id, + conversation_id: staged_source.conversation_id, account_id: stmts.account_id, - source, + source: staged_source.source, guid: &row.msg.guid, timestamp: &row.msg.timestamp, is_from_me: row.msg.is_from_me as i64, @@ -968,6 +1019,7 @@ async fn insert_message_rows( deletion: row.msg.deletion, sort_order: row.sort_order, import_id: stmts.import_id, + backup_taken_at: staged_source.backup_taken_at, }) .collect(); db_staging::insert_messages(tx, &rows).await diff --git a/crates/server/server/src/imports_api/tests.rs b/crates/server/server/src/imports_api/tests.rs index 231fb9a6d..fb124b95f 100644 --- a/crates/server/server/src/imports_api/tests.rs +++ b/crates/server/server/src/imports_api/tests.rs @@ -5808,3 +5808,5 @@ async fn a_page_of_import_runs_is_read_without_a_statement_per_row() { let owner: Page = runs_page(rows); assert_eq!(owner.items[2].issue_count, 3); } + +mod backup_dates; diff --git a/crates/server/server/src/imports_api/tests/backup_dates.rs b/crates/server/server/src/imports_api/tests/backup_dates.rs new file mode 100644 index 000000000..4fc078adc --- /dev/null +++ b/crates/server/server/src/imports_api/tests/backup_dates.rs @@ -0,0 +1,550 @@ +//! Two copies of one message from one source, from two backups of one +//! phone: the copy from the later backup decides the message's mark and +//! text, in one import in either file order and across imports in either +//! order (#1924, #1741, #1804). A file without a backup date keeps the +//! rules for files without one: marks add, and edits compare their times. + +use super::*; + +/// The earlier backup of the phone: 2026-09-01T10:00:00Z. +const EARLIER_BACKUP: i64 = 1_788_256_800_000; +/// The later backup of the phone: 2026-09-30T18:45:12Z. +const LATER_BACKUP: i64 = 1_790_793_912_000; + +/// One backup's copy of the message `g-backup`. +struct Copy<'a> { + /// When the backup was made, or `None` for a file that does not say. + backup: Option, + text: &'a str, + versions: &'a [EarlierVersion], + deletion: Option, +} + +/// A conversation file holding `copy` as the one message `g-backup`, +/// written to `name` under `dir`. +fn backup_file(dir: &Path, name: &str, copy: &Copy<'_>) -> PathBuf { + let header = conversation_header("imessage", "+15555550123").participant("+15555550123", None); + let header = match copy.backup { + Some(ms) => header.backup_taken_at(ms), + None => header, + }; + let line = copy.versions.iter().cloned().fold( + message_line("g-backup", copy.text).sender("+15555550123"), + MessageLine::edit, + ); + let line = match copy.deletion { + Some(deletion) => line.deletion(deletion), + None => line, + }; + write_jsonl(dir, name, &format!("{header}\n{line}\n")) +} + +/// What the server holds of `g-backup`: its text, its mark, its earlier +/// versions as `(part_index, text)`, and its backup's date. +#[derive(Debug, PartialEq)] +struct Held { + text: String, + deletion: Option, + versions: Vec<(i64, String)>, + backup_taken_at: Option, +} + +/// The [`Held`] of `g-backup` in the database at `db`. +async fn held(db: &Path) -> Held { + let (_pool, mut conn) = open_verify(db).await; + let (text, deletion, backup_taken_at): (String, Option, Option) = + sqlx::query_as( + "SELECT body, deletion, backup_taken_at FROM messages WHERE guid = 'g-backup'", + ) + .fetch_one(&mut *conn) + .await + .unwrap(); + let versions = sqlx::query_as( + "SELECT v.part_index, v.text FROM message_versions v + JOIN messages m ON m.id = v.message_id + WHERE m.guid = 'g-backup' ORDER BY v.id", + ) + .fetch_all(&mut *conn) + .await + .unwrap(); + Held { + text, + deletion, + versions, + backup_taken_at, + } +} + +/// Import `files` into `db` in one import, appending to what it holds. +async fn import(db: &Path, assets: &Path, root: &Path, files: &[PathBuf]) { + import_jsonl_files(db, files, &edit_options(assets, root, false)) + .await + .unwrap(); +} + +/// What `files` give `g-backup`: imported in one import in the order +/// given, in one import in the other order, one import after another in +/// the order given, and one after another in the other order. The four +/// must agree, so each is returned to compare. +async fn every_order(tmp: &Path, label: &str, files: [&PathBuf; 2]) -> [Held; 4] { + let assets = tmp.join("assets"); + let [a, b] = files; + let mut out = Vec::new(); + for (name, batches) in [ + ("together", vec![vec![a.clone(), b.clone()]]), + ("together-reversed", vec![vec![b.clone(), a.clone()]]), + ("apart", vec![vec![a.clone()], vec![b.clone()]]), + ("apart-reversed", vec![vec![b.clone()], vec![a.clone()]]), + ] { + let db = tmp.join(format!("{label}-{name}.db")); + for batch in batches { + import(&db, &assets, tmp, &batch).await; + } + out.push(held(&db).await); + } + out.try_into().unwrap() +} + +/// The stored date of [`LATER_BACKUP`], in the form a timestamp takes. +const LATER_BACKUP_AT: &str = "2026-09-30T18:45:12.000Z"; +/// The stored date of [`EARLIER_BACKUP`]. +const EARLIER_BACKUP_AT: &str = "2026-09-01T10:00:00.000Z"; + +/// Backup A marks the message Deleted in the source app; backup B, made +/// later, after the person recovered it, does not. Whichever is imported +/// first, and whether the two arrive in one import or two, the message +/// is unmarked, and swapping the dates keeps the mark. +#[tokio::test] +async fn the_later_backup_decides_the_deletion_mark_in_every_order() { + let tmp = TempDir::new().unwrap(); + let file = |name: &str, backup: i64, deletion: Option| { + backup_file( + tmp.path(), + name, + &Copy { + backup: Some(backup), + text: "recovered later", + versions: &[], + deletion, + }, + ) + }; + let marked = file( + "marked-earlier.jsonl", + EARLIER_BACKUP, + Some(Deletion::DeletedInSourceApp), + ); + let recovered = file("recovered-later.jsonl", LATER_BACKUP, None); + for held in every_order(tmp.path(), "recovered", [&marked, &recovered]).await { + assert_eq!(held.deletion, None, "{held:?}"); + assert_eq!(held.backup_taken_at.as_deref(), Some(LATER_BACKUP_AT)); + } + + let marked = file( + "marked-later.jsonl", + LATER_BACKUP, + Some(Deletion::DeletedInSourceApp), + ); + let unmarked = file("unmarked-earlier.jsonl", EARLIER_BACKUP, None); + for held in every_order(tmp.path(), "deleted", [&unmarked, &marked]).await { + assert_eq!( + held.deletion.as_deref(), + Some("deleted_in_source_app"), + "{held:?}" + ); + assert_eq!(held.backup_taken_at.as_deref(), Some(LATER_BACKUP_AT)); + } +} + +/// Backup A marks the message Unsent, so it has no search index row +/// (#1758); backup B, made later, carries it unmarked with the same text. +/// In every order the message is unmarked and a word of its text finds it: +/// a later backup that clears the mark gives the index row back. +#[tokio::test] +async fn a_later_backup_that_clears_an_unsent_mark_makes_the_text_searchable() { + let tmp = TempDir::new().unwrap(); + let file = |name: &str, backup: i64, deletion: Option| { + backup_file( + tmp.path(), + name, + &Copy { + backup: Some(backup), + text: "zqlighthouse", + versions: &[], + deletion, + }, + ) + }; + let unsent = file( + "unsent-earlier.jsonl", + EARLIER_BACKUP, + Some(Deletion::Unsent), + ); + let shown = file("shown-later.jsonl", LATER_BACKUP, None); + for held in every_order(tmp.path(), "unsent", [&unsent, &shown]).await { + assert_eq!(held.deletion, None, "{held:?}"); + } + for name in ["together", "together-reversed", "apart", "apart-reversed"] { + let (_pool, mut conn) = open_verify(&tmp.path().join(format!("unsent-{name}.db"))).await; + let hits: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM messages_fts WHERE messages_fts MATCH 'zqlighthouse'", + ) + .fetch_one(&mut *conn) + .await + .unwrap(); + assert_eq!(hits, 1, "{name}: the text is searchable again"); + } +} + +/// Two files with the same backup date, as two reads of one Mac's +/// `chat.db` that Messages did not write between give: the date cannot say which is newer, +/// so the rules for files without one hold, and the mark one of them +/// carries is added in every order rather than lost to the other. +#[tokio::test] +async fn equal_backup_dates_fall_back_to_the_rules_for_files_without_one() { + let tmp = TempDir::new().unwrap(); + let file = |name: &str, deletion: Option| { + backup_file( + tmp.path(), + name, + &Copy { + backup: Some(LATER_BACKUP), + text: "read twice", + versions: &[], + deletion, + }, + ) + }; + let unmarked = file("unmarked-same.jsonl", None); + let marked = file("marked-same.jsonl", Some(Deletion::Unsent)); + for held in every_order(tmp.path(), "same-date", [&unmarked, &marked]).await { + assert_eq!(held.deletion.as_deref(), Some("unsent"), "{held:?}"); + assert_eq!(held.backup_taken_at.as_deref(), Some(LATER_BACKUP_AT)); + } +} + +/// The scenario of #1804: backup A lists part 1's earlier versions +/// [x@t0, y@t100]; backup B, made later, after part 1 was unsent (which +/// drops its versions) and part 0 was edited, lists part 0's [a@t0] only. +/// B's newest version is older than A's, so the version times say A is +/// later; the backups' dates say B, and B's text and versions are kept in +/// every order. With the dates swapped, A's are. +#[tokio::test] +async fn the_later_backup_decides_the_text_whatever_the_version_times_say() { + let tmp = TempDir::new().unwrap(); + let t0 = 1_426_183_462_000; + let a_versions = [edit_version(1, "x", t0), edit_version(1, "y", t0 + 100_000)]; + let b_versions = [edit_version(0, "a", t0)]; + let a = |backup| Copy { + backup: Some(backup), + text: "a z", + versions: &a_versions, + deletion: None, + }; + let b = |backup| Copy { + backup: Some(backup), + text: "b", + versions: &b_versions, + deletion: None, + }; + + let a_earlier = backup_file(tmp.path(), "a-earlier.jsonl", &a(EARLIER_BACKUP)); + let b_later = backup_file(tmp.path(), "b-later.jsonl", &b(LATER_BACKUP)); + for held in every_order(tmp.path(), "b-later", [&a_earlier, &b_later]).await { + assert_eq!( + held, + Held { + text: "b".into(), + deletion: None, + versions: vec![(0, "a".into())], + backup_taken_at: Some(LATER_BACKUP_AT.into()), + } + ); + } + + let a_later = backup_file(tmp.path(), "a-later.jsonl", &a(LATER_BACKUP)); + let b_earlier = backup_file(tmp.path(), "b-earlier.jsonl", &b(EARLIER_BACKUP)); + for held in every_order(tmp.path(), "a-later", [&a_later, &b_earlier]).await { + assert_eq!( + held, + Held { + text: "a z".into(), + deletion: None, + versions: vec![(1, "x".into()), (1, "y".into())], + backup_taken_at: Some(LATER_BACKUP_AT.into()), + } + ); + } +} + +/// An append of the older backup after the newer one changes nothing: +/// not the mark, not the text, not the earlier versions, not the date, +/// even though the older copy carries a mark and lists more versions. +#[tokio::test] +async fn an_append_of_the_older_backup_after_the_newer_changes_nothing() { + let tmp = TempDir::new().unwrap(); + let assets = tmp.path().join("assets"); + let db = tmp.path().join("messagecrate.db"); + let t0 = 1_426_183_462_000; + let newer = backup_file( + tmp.path(), + "newer.jsonl", + &Copy { + backup: Some(LATER_BACKUP), + text: "see you at seven", + versions: &[edit_version(0, "see you at six", t0)], + deletion: None, + }, + ); + let older = backup_file( + tmp.path(), + "older.jsonl", + &Copy { + backup: Some(EARLIER_BACKUP), + text: "see you at eight", + versions: &[ + edit_version(0, "see you at six", t0), + edit_version(0, "see you at seven", t0 + 60_000), + ], + deletion: Some(Deletion::Unsent), + }, + ); + import(&db, &assets, tmp.path(), std::slice::from_ref(&newer)).await; + let before = held(&db).await; + assert_eq!(before.text, "see you at seven"); + import(&db, &assets, tmp.path(), &[older]).await; + assert_eq!(held(&db).await, before); +} + +/// Where either file has no backup date, the rules for files without one +/// hold. In one import, a second file's mark is added and its edit is +/// taken when its versions are newer; across imports, a file without the +/// mark leaves it, and a stored message from an undated file keeps no date +/// when a dated file later gives it nothing. +#[tokio::test] +async fn without_a_backup_date_marks_add_and_edits_compare_their_times() { + let tmp = TempDir::new().unwrap(); + let t0 = 1_426_183_462_000; + let unmarked_undated = backup_file( + tmp.path(), + "unmarked-undated.jsonl", + &Copy { + backup: None, + text: "see you at seven", + versions: &[edit_version(0, "see you at six", t0)], + deletion: None, + }, + ); + let marked_dated = backup_file( + tmp.path(), + "marked-dated.jsonl", + &Copy { + backup: Some(EARLIER_BACKUP), + text: "see you at six", + versions: &[], + deletion: Some(Deletion::Unsent), + }, + ); + // Undated against dated: the mark adds, and the copy with the later edit + // keeps its text, whichever is the dated one. + for held in every_order(tmp.path(), "mixed", [&unmarked_undated, &marked_dated]).await { + assert_eq!(held.deletion.as_deref(), Some("unsent"), "{held:?}"); + assert_eq!(held.text, "see you at seven", "{held:?}"); + assert_eq!(held.versions, vec![(0, "see you at six".into())]); + } + + // Both undated, the mark in the second file of one import: the mark + // adds rather than being lost to the copy staged first. + let marked_undated = backup_file( + tmp.path(), + "marked-undated.jsonl", + &Copy { + backup: None, + text: "see you at seven", + versions: &[edit_version(0, "see you at six", t0)], + deletion: Some(Deletion::DeletedInSourceApp), + }, + ); + for held in every_order(tmp.path(), "undated", [&unmarked_undated, &marked_undated]).await { + assert_eq!( + held.deletion.as_deref(), + Some("deleted_in_source_app"), + "{held:?}" + ); + assert_eq!(held.backup_taken_at, None); + } +} + +/// Two backups in one import give the message the later backup's text, and +/// so the later backup's duplicate flag: with another source holding the +/// later text, the dedupe hides one of the two, in either file order, and +/// with the earlier text winning it would hide neither. The earlier backup's +/// newest earlier version is the newer one, so the version times alone pick +/// the earlier text. +#[tokio::test] +async fn the_later_backup_decides_the_duplicate_flag() { + let tmp = TempDir::new().unwrap(); + let assets = tmp.path().join("assets"); + let t0 = 1_426_183_462_000; + let earlier = backup_file( + tmp.path(), + "earlier.jsonl", + &Copy { + backup: Some(EARLIER_BACKUP), + text: "see you at six", + versions: &[edit_version(1, "bring cake", t0 + 100_000)], + deletion: None, + }, + ); + let later = backup_file( + tmp.path(), + "later.jsonl", + &Copy { + backup: Some(LATER_BACKUP), + text: "see you at seven", + versions: &[edit_version(0, "see you at six", t0)], + deletion: None, + }, + ); + let header = + conversation_header("sms-backup-restore", "+15555550123").participant("+15555550123", None); + let sms = write_jsonl( + tmp.path(), + "sms.jsonl", + &format!( + "{header}\n{}\n", + message_line("g-sms", "see you at seven") + .sender("+15555550123") + .sms() + ), + ); + for (name, files) in [ + ("earlier-first.db", [earlier.clone(), later.clone()]), + ("later-first.db", [later.clone(), earlier.clone()]), + ] { + let db = tmp.path().join(name); + let options = edit_options(&assets, tmp.path(), true); + import_jsonl_files(&db, &files, &options).await.unwrap(); + let sms_options = ImportOptions::fixed(FixedImportArgs { + assets_dir: &assets, + asset_root: tmp.path(), + mode: ImportMode::Append, + source: "sms-backup-restore", + account_id: TEST_ACCOUNT, + fill_content_keys: true, + import_id: None, + }); + import_jsonl_files(&db, std::slice::from_ref(&sms), &sms_options) + .await + .unwrap(); + let (_pool, mut conn) = open_verify(&db).await; + crate::dedupe::dedupe_cross_source(&mut conn, TEST_ACCOUNT, None, 2) + .await + .unwrap(); + let hidden: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM messages WHERE duplicate_of IS NOT NULL") + .fetch_one(&mut *conn) + .await + .unwrap(); + assert_eq!( + hidden, 1, + "{name}: the two copies of the later text are one" + ); + } +} + +/// The Import Run says when the backup it read was made: the newest date +/// its files name, as `backup_taken_at`, and null for a run whose files +/// name none. +#[tokio::test] +async fn the_import_run_says_when_its_backup_was_made() { + let (state, _fixture, token) = importer().await; + let batch = |backup: Option, guid: &str| { + let header = + conversation_header("imessage", "+15555550123").participant("+15555550123", None); + let header = match backup { + Some(ms) => header.backup_taken_at(ms), + None => header, + }; + format!( + "{header}\n{}\n", + message_line(guid, "hello").sender("+15555550123") + ) + }; + let run = |body: Vec| { + let state = state.clone(); + let token = token.clone(); + async move { + let (_, created): (String, serde_json::Value) = post_created_json( + &state, + "/v1/imports", + &token, + serde_json::json!({ "source": "imessage", "mode": "append" }), + ) + .await; + let id = created["id"].as_i64().unwrap(); + assert_eq!(created["backup_taken_at"], serde_json::Value::Null); + for body in body { + let (status, text) = crate::test_support::post_raw( + &state, + &format!("/v1/imports/{id}/batches"), + &token, + "application/jsonl", + body, + ) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{text}"); + } + let read: serde_json::Value = + get_json(&state, &format!("/v1/imports/{id}"), &token).await; + let _: serde_json::Value = post_json( + &state, + &format!("/v1/imports/{id}/complete"), + &token, + serde_json::json!({ "status": "completed" }), + ) + .await; + read["backup_taken_at"].clone() + } + }; + assert_eq!( + run(vec![ + batch(Some(LATER_BACKUP), "g-1"), + batch(Some(EARLIER_BACKUP), "g-2"), + batch(None, "g-3"), + ]) + .await, + serde_json::json!(LATER_BACKUP_AT) + ); + assert_eq!( + run(vec![batch(Some(EARLIER_BACKUP), "g-4")]).await, + serde_json::json!(EARLIER_BACKUP_AT) + ); + assert_eq!(run(vec![batch(None, "g-5")]).await, serde_json::Value::Null); +} + +/// A message read back through the API carries the date of the backup that +/// decided it, which an Export Run writes back into the conversation file. +#[tokio::test] +async fn a_message_read_back_carries_its_backup_date() { + let (state, _fixture, token) = importer().await; + let header = conversation_header("imessage", "+15555550123") + .participant("+15555550123", None) + .backup_taken_at(LATER_BACKUP); + import_one_batch( + &state, + &token, + "imessage", + "append", + format!( + "{header}\n{}\n", + message_line("g-read", "hello").sender("+15555550123") + ), + ) + .await; + let page: serde_json::Value = get_json(&state, "/v1/messages", &token).await; + assert_eq!( + page["items"][0]["backup_taken_at"], + serde_json::json!(LATER_BACKUP_AT) + ); +} diff --git a/crates/server/server/src/models.rs b/crates/server/server/src/models.rs index 3a558d855..9c76eb3b3 100644 --- a/crates/server/server/src/models.rs +++ b/crates/server/server/src/models.rs @@ -42,6 +42,9 @@ pub struct ConversationRecord { pub participants: Vec, /// IR `export.source` — used as `messages.source` for directory import. pub export_source: Option, + /// When the backup the file was read from was made, in the form a + /// message's timestamp takes; `None` when the file does not say. + pub backup_taken_at: Option, } impl ConversationRecord { @@ -236,9 +239,12 @@ pub fn parse_ir_lines( line: line_no, detail: format!("the conversation header is not valid: {e}"), })?; - out.push(ExportRecord::Conversation(conversation_from_ir( - &header, line_no, - ))); + let conversation = + conversation_from_ir(&header, line_no).map_err(|e| ImportFailure::Invalid { + line: line_no, + detail: format!("{e:#}"), + })?; + out.push(ExportRecord::Conversation(conversation)); header_owner = header.export.owner_identity.as_deref().and_then(nonempty); saw_header = true; } else { @@ -299,7 +305,12 @@ fn is_ir_header(value: &Value) -> bool { } /// Map a JSON Lines header onto the server's conversation record. -fn conversation_from_ir(header: &ConversationHeader, line: usize) -> ConversationRecord { +/// +/// # Errors +/// +/// Returns an error when the backup date is outside the times a timestamp +/// can hold. +fn conversation_from_ir(header: &ConversationHeader, line: usize) -> Result { let export_source = { let s = header.export.source.trim(); if s.is_empty() { @@ -308,7 +319,15 @@ fn conversation_from_ir(header: &ConversationHeader, line: usize) -> Conversatio Some(s.to_string()) } }; - ConversationRecord { + let backup_taken_at = header + .export + .backup_taken_at_unix_ms + .map(|ms| { + format_utc_timestamp(ms) + .with_context(|| format!("unrepresentable backup_taken_at_unix_ms {ms}")) + }) + .transpose()?; + Ok(ConversationRecord { line, chat_identifier: header.conversation.chat_identifier.clone(), // Platform identity for handles (phone | whatsapp), not SMS/iMessage/RCS. @@ -331,7 +350,8 @@ fn conversation_from_ir(header: &ConversationHeader, line: usize) -> Conversatio .filter_map(participant_from_ir) .collect(), export_source, - } + backup_taken_at, + }) } /// Map one IR message onto the server's message record. `header_owner` is the diff --git a/crates/server/server/src/test_support/lines.rs b/crates/server/server/src/test_support/lines.rs index 6b4ff3bfb..4a95458b5 100644 --- a/crates/server/server/src/test_support/lines.rs +++ b/crates/server/server/src/test_support/lines.rs @@ -30,6 +30,7 @@ pub fn conversation_header(source: &str, chat_identifier: &str) -> ConversationH tool_version: "0".to_string(), owner_identity: None, owner_display_name: None, + backup_taken_at_unix_ms: None, }, conversation: message_ir::ConversationMeta { chat_identifier: chat_identifier.to_string(), @@ -49,6 +50,13 @@ impl ConversationHeaderLine { self } + /// When the backup the batch was read from was made, in Unix + /// milliseconds. + pub fn backup_taken_at(mut self, unix_ms: i64) -> Self { + self.0.export.backup_taken_at_unix_ms = Some(unix_ms); + self + } + /// A group conversation. pub fn group(mut self) -> Self { self.0.conversation.conversation_type = message_ir::IrConversationType::Group; diff --git a/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl b/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl index 05736d0ca..1e9fb24e9 100644 --- a/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl +++ b/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl @@ -1,4 +1,4 @@ -{"schema_version":10,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578308040000,"last_timestamp_unix_ms":1578308160000}}} +{"schema_version":11,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578308040000,"last_timestamp_unix_ms":1578308160000}}} {"guid":"guid-16","timestamp_unix_ms":1578308040000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Delete me","attachments":[],"deletion":"deleted_in_source_app","imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Delete me"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} {"guid":"guid-17","timestamp_unix_ms":1578308100000,"direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"","attachments":[],"deletion":"unsent","imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} {"guid":"guid-18","timestamp_unix_ms":1578308160000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Still here","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Still here"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} diff --git a/crates/server/server/tests/fixtures/apple-messages-edits.jsonl b/crates/server/server/tests/fixtures/apple-messages-edits.jsonl index c9979cd92..2342100ba 100644 --- a/crates/server/server/tests/fixtures/apple-messages-edits.jsonl +++ b/crates/server/server/tests/fixtures/apple-messages-edits.jsonl @@ -1,4 +1,4 @@ -{"schema_version":10,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578309000000,"last_timestamp_unix_ms":1578309120000}}} +{"schema_version":11,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578309000000,"last_timestamp_unix_ms":1578309120000}}} {"guid":"guid-edited-twice","timestamp_unix_ms":1578309000000,"direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Meet at the bakery","attachments":[],"edits":[{"part_index":0,"text":"Meet at the library","edited_at_unix_ms":1578309000000},{"part_index":0,"text":"Meet at the museum","edited_at_unix_ms":1578309030000}],"imessage":null,"source":null} {"guid":"guid-edited-final-match","timestamp_unix_ms":1578309060000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"The library opens at nine","attachments":[],"edits":[{"part_index":0,"text":"The library opens at eight","edited_at_unix_ms":1578309060000}],"imessage":null,"source":null} {"guid":"guid-never-edited","timestamp_unix_ms":1578309120000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Nothing changed here","attachments":[],"imessage":null,"source":null} diff --git a/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl b/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl index b5b4d7200..3f2cc0f3d 100644 --- a/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl +++ b/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl @@ -1,4 +1,4 @@ -{"schema_version":10,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"chat100","conversation_type":"group","group_title":"Weekend plans","participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"},{"identity":"friend@example.com","display_name":null,"identity_type":"email"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578307860000,"last_timestamp_unix_ms":1578307980000}}} +{"schema_version":11,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"chat100","conversation_type":"group","group_title":"Weekend plans","participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"},{"identity":"friend@example.com","display_name":null,"identity_type":"email"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578307860000,"last_timestamp_unix_ms":1578307980000}}} {"guid":"00000000-0000-4000-8000-000000000013","timestamp_unix_ms":1578307860000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"friend@example.com","sender_display_name":"friend@example.com","owner_identity":"+15555550106","subject":null,"text":"Pizza?","attachments":[],"reactions":[{"part_index":0,"kind":"loved","is_from_me":false,"reactor_identity":"+15555550107","reactor_display_name":"+15555550107"},{"part_index":0,"kind":"emoji","emoji":"🔥","is_from_me":true,"reactor_display_name":"+15555550106"}],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Pizza?"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} {"guid":"guid-14","timestamp_unix_ms":1578307920000,"direction":"incoming","service":"imessage","message_kind":"tapback","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Loved a message","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":"00000000-0000-4000-8000-000000000013","associated_part":0,"tapback_kind":"loved","tapback_emoji":null,"tapback_action":"add"},"source":null} {"guid":"guid-15","timestamp_unix_ms":1578307980000,"direction":"outgoing","service":"imessage","message_kind":"tapback","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"🔥 reacted","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":"00000000-0000-4000-8000-000000000013","associated_part":0,"tapback_kind":"emoji","tapback_emoji":"🔥","tapback_action":"add"},"source":null} diff --git a/docs/architecture/contacts-identities-and-messages.md b/docs/architecture/contacts-identities-and-messages.md index e283b01f0..bf002ae3a 100644 --- a/docs/architecture/contacts-identities-and-messages.md +++ b/docs/architecture/contacts-identities-and-messages.md @@ -405,11 +405,44 @@ otherwise keep an older title against a newer one. A title of only spaces counts as no title. The rule is the same in one batch as across several, in any order. Why: a group is renamed over time, so the copy whose messages run later carries the name the group has now. An old backup uploaded after a newer one can't bring the old -name back, because its messages stop earlier. The time a backup was made is not -recorded by every source, and the time an exporter ran says nothing about the -backup, so neither decides it +name back, because its messages stop earlier. The backup date a conversation +file carries (`export.backup_taken_at_unix_ms`) is missing where a source records +none, and for several sources is a file's modification time, which copying the +file can change; the time an exporter ran says nothing about the backup; so +neither decides the title ([#1408](https://github.com/messagecrate/message-crate/issues/1408)). +**Between two copies of one message from one source, the copy from the later +backup decides its mark and text.** One message from one source is one row +(`UNIQUE (account_id, source, guid)` on `messages`, and the same on +`staging_messages`), so a second backup of the same phone meets the copy +already there, in the same import or a later one. The conversation file says +when its backup was made (`export.backup_taken_at_unix_ms`), staging keeps +that date on each staged row, and `messages.backup_taken_at` keeps the date of +the backup that decided the stored copy. When both copies have a date, the +copy from the later backup gives the message its deletion mark, mark or no +mark, and its text and earlier versions, whatever the versions' times say; a +copy from an earlier backup changes neither. The duplicate flag follows the +text, because the dedupe compares the text. The date is kept to the +millisecond, the form every stored time takes. When either copy has no date, +or the two dates are equal, nothing says which backup is newer, so the rules +for files without one hold: a copy with a mark adds it and one without leaves +the mark held, and a copy takes the text when its newest earlier version is +newer +(`later_edit_sql` in `db/staging.rs`). Attachments and reactions add from +either copy, because a backup that lacks one does not say it is gone. The +rule is the same in one import as across several, in any file order +(`later_backup_sql` in `db/staging.rs`, `add_staged_copy` in +`imports_api/staging.rs`). Why: a person recovers a deleted message and +unsends or edits a sent one between two backups, and only the backup's own +date says which state is the newer; the message's own times record when it +was written, not when a part was unsent, so they cannot tell +([#1741](https://github.com/messagecrate/message-crate/issues/1741), +[#1804](https://github.com/messagecrate/message-crate/issues/1804), +[#1924](https://github.com/messagecrate/message-crate/issues/1924)). An append +with dedupe off still leaves a changed message's duplicate flag until the next +dedupe ([#1805](https://github.com/messagecrate/message-crate/issues/1805)). + **A participant's display name has one rule.** The contact's name, else what that backup called them in that conversation, else the identity. One loader applies it for the conversation list, the message pane, and Export. diff --git a/docs/architecture/http-api.md b/docs/architecture/http-api.md index 9f7005dfe..aa10cf8b2 100644 --- a/docs/architecture/http-api.md +++ b/docs/architecture/http-api.md @@ -608,6 +608,11 @@ What each reaches: `OwnerImportRun` or `OwnerExportRun`: the source, mode, tool, times, outcome and counts, with the counts an import's summary reported and how many issues it recorded, and for an export only which form its scope took. + The times are the run's own, when it started and ended; the owner reads + nothing of the backup an import read, neither the file the desktop app + recorded (`source_fingerprint`) nor when the backup was made + (`backup_taken_at`), because both say when and from what the account's + phone was backed up. Why: a staging summary lists the addresses of everyone in the backup, an issue names its conversation's file, a note names a file or an address, and an export's query is a search over diff --git a/docs/src/assets/openapi.json b/docs/src/assets/openapi.json index b7c74dcfe..d31bd5f69 100644 --- a/docs/src/assets/openapi.json +++ b/docs/src/assets/openapi.json @@ -15326,6 +15326,7 @@ "contacts_new", "contacts_changed", "attachments_ms", + "backup_taken_at", "device_id", "duration_ms", "finished_at", @@ -15350,6 +15351,13 @@ "format": "int64", "description": "Time spent on attachments, when finished." }, + "backup_taken_at": { + "type": [ + "string", + "null" + ], + "description": "When the backup the run read was made, UTC: the newest date its\nconversation files name (`export.backup_taken_at_unix_ms`). Null until\na file that names one is imported, and for a run whose files name none." + }, "bytes_uploaded": { "type": "integer", "format": "int64", @@ -15770,6 +15778,7 @@ "tapbacks", "earlier_versions", "matched_earlier_version", + "backup_taken_at", "deletion", "owner", "reply_to", @@ -15786,6 +15795,13 @@ }, "description": "Attachments on this message." }, + "backup_taken_at": { + "type": [ + "string", + "null" + ], + "description": "When the backup that gave the message its mark and text was made,\nRFC 3339 in UTC with a `Z` suffix; `null` when the conversation\nfile did not say. Between two copies of one message from one\nsource, the copy from the later backup decides." + }, "conversation": { "$ref": "#/components/schemas/MessageConversation", "description": "The conversation this message belongs to." @@ -17538,6 +17554,7 @@ "contacts_new", "contacts_changed", "attachments_ms", + "backup_taken_at", "device_id", "duration_ms", "finished_at", @@ -17562,6 +17579,13 @@ "format": "int64", "description": "Time spent on attachments, when finished." }, + "backup_taken_at": { + "type": [ + "string", + "null" + ], + "description": "When the backup the run read was made, UTC: the newest date its\nconversation files name (`export.backup_taken_at_unix_ms`). Null until\na file that names one is imported, and for a run whose files name none." + }, "bytes_uploaded": { "type": "integer", "format": "int64", @@ -17818,6 +17842,7 @@ "tapbacks", "earlier_versions", "matched_earlier_version", + "backup_taken_at", "deletion", "owner", "reply_to", @@ -17834,6 +17859,13 @@ }, "description": "Attachments on this message." }, + "backup_taken_at": { + "type": [ + "string", + "null" + ], + "description": "When the backup that gave the message its mark and text was made,\nRFC 3339 in UTC with a `Z` suffix; `null` when the conversation\nfile did not say. Between two copies of one message from one\nsource, the copy from the later backup decides." + }, "conversation": { "$ref": "#/components/schemas/MessageConversation", "description": "The conversation this message belongs to." diff --git a/docs/src/content/docs/docs/developer/architecture/common-message.md b/docs/src/content/docs/docs/developer/architecture/common-message.md index b58a8ad77..802dbaca5 100644 --- a/docs/src/content/docs/docs/developer/architecture/common-message.md +++ b/docs/src/content/docs/docs/developer/architecture/common-message.md @@ -30,19 +30,20 @@ Pipeline: `backup → common message → FormatSink → user-picked format`. - **Common-message path** (`ConversationDocument` → `message_ir_format::FormatSink`, one of json/jsonl/csv/eml/mbox/xml): all exporters, including iMessage (`imessage-ir-exporter`). Per-chat formats also accept `write_format`; XML uses a single `smses.xml` via the sink. - **Media + obfuscate** run inside `FormatSink::finish` for every format (`message_crate_core::ExportTransforms`: none / copy / convert / compress, plus optional obfuscate). When obfuscate is on, exporters skip staging real attachment bytes and convert/compress is not run — only placeholder files are written. Exporters pass transforms from `ExporterConfig.media` / `.obfuscate`; there is no CSV-only post-step. EML / MBOX / XML embed media and drop the staged `attachments/` directory afterward. -- **Schema version 10 only** (breaking). Version 10 keeps the message a reply quotes in the message's own `reply_to`, for every source (see [Replies](#replies)), where version 9 kept the Apple Messages reply link in `imessage.is_reply` and `imessage.in_reply_to_guid`, and a reply count in `imessage.num_replies`. Version 9 had given orphaned messages conversations of type `orphaned` (see [Orphaned messages](#orphaned-messages)), where version 8 put them all in one `individual` conversation named `orphaned`. Version 8 had kept an edited message's earlier versions in its own `edits`, for every source, where version 7 kept the Apple Messages edit history as a JSON value in `imessage.edits`. Version 7 had moved a message's mark, Deleted in the source app or Unsent, in its own `deletion`, for every source, where version 6 kept the Apple Messages deleted mark in `imessage.is_deleted`. Version 6 had moved a message's reactions into its own `reactions` list, one shape for every source, where version 5 kept Apple Messages reactions as a JSON value in `imessage.tapbacks`. Version 5 had named every address an identity (`identity`, `identity_type`, `owner_identity`, `sender_identity`, `reactor_identity`) where version 4 said `handle`. Version 9 and older are refused, never upgraded. Typed enums/bags, filled outgoing identity, conversation stats, stable null/`[]` keys. Older common-message JSON is not read — regenerate exports after schema changes. +- **Schema version 11 only** (breaking). Version 11 says when the backup was made, in `export.backup_taken_at_unix_ms` (see [When the backup was made](#when-the-backup-was-made)), which version 10 did not, so an import could not tell which of two backups of one phone is the later one. Version 10 had kept the message a reply quotes in the message's own `reply_to`, for every source (see [Replies](#replies)), where version 9 kept the Apple Messages reply link in `imessage.is_reply` and `imessage.in_reply_to_guid`, and a reply count in `imessage.num_replies`. Version 9 had given orphaned messages conversations of type `orphaned` (see [Orphaned messages](#orphaned-messages)), where version 8 put them all in one `individual` conversation named `orphaned`. Version 8 had kept an edited message's earlier versions in its own `edits`, for every source, where version 7 kept the Apple Messages edit history as a JSON value in `imessage.edits`. Version 7 had moved a message's mark, Deleted in the source app or Unsent, in its own `deletion`, for every source, where version 6 kept the Apple Messages deleted mark in `imessage.is_deleted`. Version 6 had moved a message's reactions into its own `reactions` list, one shape for every source, where version 5 kept Apple Messages reactions as a JSON value in `imessage.tapbacks`. Version 5 had named every address an identity (`identity`, `identity_type`, `owner_identity`, `sender_identity`, `reactor_identity`) where version 4 said `handle`. Version 10 and older are refused, never upgraded. Typed enums/bags, filled outgoing identity, conversation stats, stable null/`[]` keys. Older common-message JSON is not read — regenerate exports after schema changes. -## Document schema (`schema_version: 10`) +## Document schema (`schema_version: 11`) ```json { - "schema_version": 10, + "schema_version": 11, "export": { "source": "sms-backup-restore", "tool": "SMS Backup & Restore", "tool_version": "10.26.003", "owner_identity": "+15555550100", - "owner_display_name": "Me" + "owner_display_name": "Me", + "backup_taken_at_unix_ms": 1400800000000 }, "conversation": { "chat_identifier": "+15555550101", @@ -95,6 +96,26 @@ Pipeline: `backup → common message → FormatSink → user-picked format`. - `guid` is Apple's own id for Apple Messages. Every other source's `guid` is a `MessageGuid`: SHA-256 of the chat id, the direction, the sender of an incoming message, the UTC instant in milliseconds, the text with whitespace collapsed, the sorted attachment digests, and the source's own key where it has one (WhatsApp's `key_id`). It reads no time zone and no display format, so one backup gives the same ids on any computer. The server refuses a message whose `guid` is empty. - Two records a backup cannot tell apart are one message, and the exporter keeps one (`message_ir::one_copy_per_message`). The server's content key, which matches one message across sources, is the same identity at whole seconds. +### When the backup was made + +`export.backup_taken_at_unix_ms` is when the backup the file was read from was made, in Unix milliseconds, or `null` when nothing says. Each exporter reads it from its source: + +| Source | Date | +|--------|------| +| Apple Messages from an iPhone backup | The `Date` in the backup's `Manifest.plist`, which is readable whether or not the backup is encrypted | +| Apple Messages from a Mac `chat.db` | When Messages last wrote the database: the newest modification time of `chat.db` and its `chat.db-wal` and `chat.db-shm` files | +| WhatsApp from an iPhone backup | The `Date` in the backup's `Manifest.plist` | +| WhatsApp from Android | The modification time of the database file, `msgstore.db.crypt15` or a decrypted `msgstore.db` | +| WhatsApp from a ready-made `result.json` | The JSON file's modification time | +| SMS Backup & Restore | The root element's `backup_date` attribute; a file without one is dated by its modification time. A conversation read from two files is as new as the newer. Message Crate's own XML export writes `backup_date` back, the newest date of the conversations it holds | +| iMazing | The export date: the newest modification time of the CSV files iMazing wrote | +| OpenExtract, GO SMS Pro, SMS Backup+ | The newest modification time of the files read, because none of them records a date of its own | +| An Export Run of the server | The backup date of the conversation's messages when they all have the same one, else `null`, because one date for messages from two backups would be wrong for some of them | + +The import reads it to decide between two copies of one message from one source. The copy from the later backup gives the message its deletion mark, mark or no mark, and its text and earlier versions, in one import in either file order and across imports in either order. A copy from an earlier backup changes neither. The server keeps the date to the millisecond. A file that says nothing, or two copies with the same date, keep the rules for files without a date: a mark adds and is never cleared, and an edit counts as later when its newest earlier version is newer. Attachments and reactions add from either copy. + +CSV carries it in the `backup_taken_at_unix_ms` column of every row, and EML and MBOX in the `X-ME-Backup-Taken-At-Unix-Ms` header of every mail, blank or absent when the file says nothing. A value that is not a whole number is refused rather than read as no date. + ### Reactions `reactions` is the list of reactions that stand on a message, the same shape for every source. Each one is a `Reaction`: @@ -183,7 +204,7 @@ Attachment **bytes** are never stored in JSON/JSONL (`#[serde(skip)]`). Paths + ## JSONL layout ```text -{"schema_version":10,"export":{…},"conversation":{…}} +{"schema_version":11,"export":{…},"conversation":{…}} {"guid":"…","timestamp_unix_ms":…, …} … ``` diff --git a/docs/src/content/docs/docs/developer/formats/mail-archive.md b/docs/src/content/docs/docs/developer/formats/mail-archive.md index b75275248..50229f3aa 100644 --- a/docs/src/content/docs/docs/developer/formats/mail-archive.md +++ b/docs/src/content/docs/docs/developer/formats/mail-archive.md @@ -149,6 +149,7 @@ A mail an earlier Message Crate wrote names its addresses with `X-ME-Sender-Hand | `X-ME-Export-Source` | string | e.g. `sms-backup-restore` | | `X-ME-Export-Tool` | string | | | `X-ME-Export-Tool-Version` | string | | +| `X-ME-Backup-Taken-At-Unix-Ms` | integer string | When the backup was made, in Unix milliseconds; omitted when the export does not say. A value that is not a whole number is refused | | `X-ME-Android-Type` | integer string | Optional; SMS `type` / MMS `msg_box` | | `X-ME-Source-Fields` | JSON | Optional full-fidelity bag (CSV `source_fields_json` / PDU extras) | | `X-ME-Attachment-Meta` | JSON array | Parallel to MIME attachment parts (see Attachments) | @@ -350,6 +351,7 @@ Normal sticker sends: image MIME part + `X-ME-Attachment-Meta` (`is_sticker`, `s | `source_fields_json` / PDU extras | `X-ME-Source-Fields` | | `export_*` | `X-ME-Export-*` | | `owner_identity` / `owner_display_name` | `X-ME-Owner-*` | +| `backup_taken_at_unix_ms` | `X-ME-Backup-Taken-At-Unix-Ms` | | `message_owner_identity` | `X-ME-Message-Owner-Identity` | | `participants_json` (iMessage) | `X-ME-Participants` | | `reactions_json` | `X-ME-Reactions` | diff --git a/docs/src/content/docs/docs/developer/message-transfer.md b/docs/src/content/docs/docs/developer/message-transfer.md index 78965d276..ea18e9124 100644 --- a/docs/src/content/docs/docs/developer/message-transfer.md +++ b/docs/src/content/docs/docs/developer/message-transfer.md @@ -41,7 +41,7 @@ Each conversation is one text file whose name ends in `.jsonl`. JSON Lines means Pictures and other media sit next to those files in `attachments/`. ```jsonl title="One conversation file" -{"schema_version":10,"export":{"source":"sms-backup-restore","tool":"SMS Backup & Restore","owner_identity":"+15555550100","owner_display_name":"Me"},"conversation":{"chat_identifier":"+15555550101","conversation_type":"individual","participants":[{"identity":"+15555550101","display_name":"Sam"}]}} +{"schema_version":11,"export":{"source":"sms-backup-restore","tool":"SMS Backup & Restore","owner_identity":"+15555550100","owner_display_name":"Me","backup_taken_at_unix_ms":1400800000000},"conversation":{"chat_identifier":"+15555550101","conversation_type":"individual","participants":[{"identity":"+15555550101","display_name":"Sam"}]}} {"guid":"msg-1","timestamp_unix_ms":1400773261000,"direction":"outgoing","service":"sms","text":"Hello"} ``` diff --git a/docs/src/content/docs/docs/developer/reference/csv-columns.md b/docs/src/content/docs/docs/developer/reference/csv-columns.md index a98184fee..5d4d77615 100644 --- a/docs/src/content/docs/docs/developer/reference/csv-columns.md +++ b/docs/src/content/docs/docs/developer/reference/csv-columns.md @@ -48,6 +48,7 @@ CSV output contains one row per message. Conversation and export identity are re | `export_tool_version` | Source version recorded by the importer. | | `owner_identity` | Phone number or email for the person whose backup was exported. | | `owner_display_name` | Display name for the owner. | +| `backup_taken_at_unix_ms` | When the backup the export was read from was made, in Unix milliseconds; blank when the export does not say. | | `message_owner_identity` | The owner's own address on this message: the one it was sent from or received at. Apple Messages records it per message, so one conversation can hold rows from a phone number and an Apple ID. Empty when the source records no owner per message, and `owner_identity` then stands for the row. | | `android_type` | Original Android SMS type or MMS box number, or empty for other sources. | | `source_fields_json` | Compact JSON containing source-specific fields that do not have shared columns. | diff --git a/docs/src/content/docs/docs/user/features/messages/import.md b/docs/src/content/docs/docs/user/features/messages/import.md index 9924be4c9..025f5e030 100644 --- a/docs/src/content/docs/docs/user/features/messages/import.md +++ b/docs/src/content/docs/docs/user/features/messages/import.md @@ -381,7 +381,12 @@ The record of the run stays under **Import history** in [**Settings → Storage* A second import of the same backup creates no duplicates. The Message Crate recognises the messages it already holds and skips them, so a newer backup of the same phone adds only what is new. -A message edited again since the first import is the exception: it takes the newer backup's text and earlier versions ([Edited messages](/docs/user/features/messages/browse/#edited-messages)). +A message that changed since the first import is the exception: it takes the newer backup's text, earlier versions, and mark ([Edited messages](/docs/user/features/messages/browse/#edited-messages)). +A message recovered after it was deleted loses its **Deleted in the source app** mark, and a message unsent since is marked **Unsent**. An older backup imported after a newer one leaves the message as it is. -One import that carries an older and a newer backup of the same phone gives an edited message the newer backup's text and earlier versions, whichever order the files are in. +One import that carries an older and a newer backup of the same phone gives each message the newer backup's text, earlier versions, and mark, whichever order the files are in. + +Which backup is newer comes from the backup's own date: the date an iPhone backup records, the date an SMS Backup & Restore file records, or the date its files were last written where the backup records none. +**Settings → Storage → Import history** shows it beside the backup each run read. +A backup that records no date at all keeps a mark once given, and takes a newer text only when its edits are newer. It also keeps the attachments and reactions each backup holds of a message, each one once, so one import of two backups stores what two separate imports of them store. diff --git a/docs/src/content/docs/docs/user/features/settings/storage.md b/docs/src/content/docs/docs/user/features/settings/storage.md index 2d612d44f..dfc380bb1 100644 --- a/docs/src/content/docs/docs/user/features/settings/storage.md +++ b/docs/src/content/docs/docs/user/features/settings/storage.md @@ -39,7 +39,13 @@ Selecting the row again, or the **×** button, closes it. Three labels sit at the top: **Type**, the source, **Mode**, `Append` or `Replace`, and **Status**. -Five figures follow: **Started**, **Finished**, **Messages**, **Attachments**, and **Bytes uploaded**. +Six figures follow: **Started**, **Finished**, **Messages**, **Attachments**, **Bytes uploaded**, and **Issues**. + +**Backup** names the backup the run read, as the desktop app recorded it, and when that backup was made, so two imports of one phone can be told apart. +The date comes from the backup itself: an iPhone backup's own date, the date an SMS Backup & Restore file records, or the date its files were last written where the backup records none. +When the run read two backups, it shows the later. +"The backup does not say when it was made" appears for a backup that records no date. +The owner, reading another account's runs, does not see this line. A list of four steps comes next, each with the time it took: **Parse backup**, **Attachments**, **Preparing messages**, and **Upload to Message Crate**. diff --git a/schema/sql/accounts.sql b/schema/sql/accounts.sql index fd98436aa..08a5b6a1c 100644 --- a/schema/sql/accounts.sql +++ b/schema/sql/accounts.sql @@ -219,7 +219,11 @@ CREATE TABLE IF NOT EXISTS imports ( -- Addresses the backup's device sent from (JSON array), read by the -- client before parsing. Lets a resumed Staging Review show the identity -- list without re-reading the backup. - source_identities TEXT + source_identities TEXT, + -- When the backup the run read was made, in the form messages.timestamp + -- holds: the newest its conversation files name. NULL until a file that + -- says is imported, and for a run whose files say nothing. + backup_taken_at TEXT ); CREATE INDEX IF NOT EXISTS ix_imports_account_started diff --git a/schema/sql/messages.sql b/schema/sql/messages.sql index 323872b81..063323782 100644 --- a/schema/sql/messages.sql +++ b/schema/sql/messages.sql @@ -99,7 +99,13 @@ CREATE TABLE IF NOT EXISTS messages ( -- Points at the kept message when this row is a flagged duplicate. duplicate_of INTEGER REFERENCES messages(id) ON DELETE SET NULL, -- Import run that inserted this row (`imports.id`). - import_id INTEGER REFERENCES imports(id) ON DELETE SET NULL + import_id INTEGER REFERENCES imports(id) ON DELETE SET NULL, + -- When the backup that gave the message its deletion mark and text was + -- made, in the form timestamp holds; NULL when its file did not say. A + -- later import's copy from a later backup replaces both; one from an + -- earlier backup changes neither + -- (docs/architecture/contacts-identities-and-messages.md). + backup_taken_at TEXT ); CREATE INDEX IF NOT EXISTS ix_messages_conversation_timestamp diff --git a/schema/sql/staging.sql b/schema/sql/staging.sql index 5d015b579..c7246c47f 100644 --- a/schema/sql/staging.sql +++ b/schema/sql/staging.sql @@ -77,7 +77,12 @@ CREATE TABLE IF NOT EXISTS staging_messages ( -- Stable order within the conversation when timestamps collide. sort_order INTEGER NOT NULL, -- Import run that staged this row (`imports.id`). - import_id INTEGER REFERENCES imports(id) ON DELETE SET NULL + import_id INTEGER REFERENCES imports(id) ON DELETE SET NULL, + -- When the backup this row was read from was made, in the form timestamp + -- holds; NULL when its file did not say. Between two copies of one message + -- from one source, the copy from the later backup decides its deletion mark + -- and text (docs/architecture/contacts-identities-and-messages.md). + backup_taken_at TEXT ); CREATE INDEX IF NOT EXISTS ix_staging_messages_conversation_timestamp diff --git a/src-tauri/src/commands/upload.rs b/src-tauri/src/commands/upload.rs index 405523ae7..85ab7fc06 100644 --- a/src-tauri/src/commands/upload.rs +++ b/src-tauri/src/commands/upload.rs @@ -377,6 +377,7 @@ mod tests { tool_version: "10.26.003".into(), owner_identity: Some("+15555550100".into()), owner_display_name: Some("Me".into()), + backup_taken_at_unix_ms: None, }, "conversation": ConversationMeta { chat_identifier: "+15555550101".into(), diff --git a/web/src/lib/serverApi.types.ts b/web/src/lib/serverApi.types.ts index 1437b36f7..bab9af5dc 100644 --- a/web/src/lib/serverApi.types.ts +++ b/web/src/lib/serverApi.types.ts @@ -3072,6 +3072,12 @@ export interface components { * @description Time spent on attachments, when finished. */ attachments_ms: number | null; + /** + * @description When the backup the run read was made, UTC: the newest date its + * conversation files name (`export.backup_taken_at_unix_ms`). Null until + * a file that names one is imported, and for a run whose files name none. + */ + backup_taken_at: string | null; /** * Format: int64 * @description Bytes uploaded so far. @@ -3327,6 +3333,13 @@ export interface components { Message: { /** @description Attachments on this message. */ attachments: components["schemas"]["Attachment"][]; + /** + * @description When the backup that gave the message its mark and text was made, + * RFC 3339 in UTC with a `Z` suffix; `null` when the conversation + * file did not say. Between two copies of one message from one + * source, the copy from the later backup decides. + */ + backup_taken_at: string | null; /** @description The conversation this message belongs to. */ conversation: components["schemas"]["MessageConversation"]; /** @@ -4262,6 +4275,12 @@ export interface components { * @description Time spent on attachments, when finished. */ attachments_ms: number | null; + /** + * @description When the backup the run read was made, UTC: the newest date its + * conversation files name (`export.backup_taken_at_unix_ms`). Null until + * a file that names one is imported, and for a run whose files name none. + */ + backup_taken_at: string | null; /** * Format: int64 * @description Bytes uploaded so far. @@ -4399,6 +4418,13 @@ export interface components { items: { /** @description Attachments on this message. */ attachments: components["schemas"]["Attachment"][]; + /** + * @description When the backup that gave the message its mark and text was made, + * RFC 3339 in UTC with a `Z` suffix; `null` when the conversation + * file did not say. Between two copies of one message from one + * source, the copy from the later backup decides. + */ + backup_taken_at: string | null; /** @description The conversation this message belongs to. */ conversation: components["schemas"]["MessageConversation"]; /** diff --git a/web/src/screens/settings/storage/ImportDetailPanel.tsx b/web/src/screens/settings/storage/ImportDetailPanel.tsx index c18b41386..4845d9df3 100644 --- a/web/src/screens/settings/storage/ImportDetailPanel.tsx +++ b/web/src/screens/settings/storage/ImportDetailPanel.tsx @@ -7,6 +7,7 @@ import type { AccountImportRun } from "./storageUtils"; import { formatBytes, formatImportDate, + importBackup, importStatusLabel, sectionHint, sectionTitle, @@ -109,6 +110,7 @@ export default function ImportDetailPanel({
Issues
{selectedImport.issue_count.toLocaleString()}
+
@@ -131,3 +133,24 @@ export default function ImportDetailPanel({
); } + +/** + * The backup the run read, beside when that backup was made, so two imports of + * one phone can be told apart. The owner's view of another account's run + * carries neither, and shows nothing here. + */ +function ImportBackupDetail({ run }: { run: AccountImportRun }) { + const backup = importBackup(run); + if (!backup) return null; + return ( +
+
Backup
+
{backup.file ?? "Not recorded"}
+
+ {backup.takenAt + ? `Made ${formatImportDate(backup.takenAt)}` + : "The backup does not say when it was made"} +
+
+ ); +} diff --git a/web/src/screens/settings/storage/ImportHistoryTable.test.tsx b/web/src/screens/settings/storage/ImportHistoryTable.test.tsx index fbe872448..60b1c40c3 100644 --- a/web/src/screens/settings/storage/ImportHistoryTable.test.tsx +++ b/web/src/screens/settings/storage/ImportHistoryTable.test.tsx @@ -163,4 +163,52 @@ describe("Import history", () => { expect(await screen.findByRole("heading", { name: "Import Errors" })).toBeInTheDocument(); expect(getAccountImport).toHaveBeenCalledWith(1, expect.anything(), undefined); }); + + it("shows the backup a run read beside when that backup was made", async () => { + const listed = { ...anImport(1) }; + listAccountImports.mockResolvedValue({ items: [listed], total: 1, limit: 50, offset: 0 }); + getAccountImport.mockResolvedValue({ + ...listed, + summary: null, + contacts_new: 0, + contacts_changed: 0, + issues: [], + source_fingerprint: { path: "/backups/iPhone/00008110", size: 12, mtime_ms: 1 }, + backup_taken_at: "2026-09-30T18:45:12.000Z", + }); + const user = setupUser(); + renderWithProviders(); + const heading = await screen.findByRole("heading", { name: "Import history" }); + const section = within(heading.parentElement as HTMLElement); + await user.click(await section.findByRole("button", { expanded: false })); + + const backup = await screen.findByText("Backup"); + const [file, made] = Array.from(backup.parentElement?.querySelectorAll("dd") ?? []); + expect(file?.textContent).toBe("/backups/iPhone/00008110"); + expect(made?.textContent).toMatch(/^Made .*2026/); + }); + + it("says when a run's backup does not say when it was made", async () => { + const listed = { ...anImport(1) }; + listAccountImports.mockResolvedValue({ items: [listed], total: 1, limit: 50, offset: 0 }); + getAccountImport.mockResolvedValue({ + ...listed, + summary: null, + contacts_new: 0, + contacts_changed: 0, + issues: [], + source_fingerprint: null, + backup_taken_at: null, + }); + const user = setupUser(); + renderWithProviders(); + const heading = await screen.findByRole("heading", { name: "Import history" }); + const section = within(heading.parentElement as HTMLElement); + await user.click(await section.findByRole("button", { expanded: false })); + + const backup = await screen.findByText("Backup"); + const [file, made] = Array.from(backup.parentElement?.querySelectorAll("dd") ?? []); + expect(file?.textContent).toBe("Not recorded"); + expect(made?.textContent).toBe("The backup does not say when it was made"); + }); }); diff --git a/web/src/screens/settings/storage/storageUtils.test.ts b/web/src/screens/settings/storage/storageUtils.test.ts index 1bbe53b10..4a3cea555 100644 --- a/web/src/screens/settings/storage/storageUtils.test.ts +++ b/web/src/screens/settings/storage/storageUtils.test.ts @@ -5,6 +5,7 @@ import { describeExportScope, formatBytes, formatImportDate, + importBackup, importStatusLabel, toImportSummaryView, } from "./storageUtils"; @@ -35,6 +36,7 @@ function accountImportRun(partial: Partial = {}): AccountImpor form: null, source_fingerprint: null, source_identities: null, + backup_taken_at: null, summary: {}, issue_count: 0, note_count: 0, @@ -217,3 +219,47 @@ describe("toImportSummaryView for the owner", () => { expect(view.issues).toEqual([]); }); }); + +describe("importBackup", () => { + it("reads the backup's file and when it was made from the account's own run", () => { + expect( + importBackup( + accountImportRun({ + source_fingerprint: { path: "/backups/iPhone/00008110", size: 12, mtime_ms: 1 }, + backup_taken_at: "2026-09-30T18:45:12.000Z", + }), + ), + ).toEqual({ file: "/backups/iPhone/00008110", takenAt: "2026-09-30T18:45:12.000Z" }); + }); + + it("says nothing it was not told", () => { + expect(importBackup(accountImportRun())).toEqual({ file: null, takenAt: null }); + }); + + it("gives the owner nothing of another account's backup", () => { + expect( + importBackup({ + id: 1, + source: "imessage-ios", + mode: "append", + status: "completed", + tool: null, + started_at: "2026-08-11T12:00:00Z", + finished_at: null, + message_count: 10, + attachment_count: 0, + bytes_uploaded: 0, + duration_ms: null, + parse_ms: null, + attachments_ms: null, + prepare_ms: null, + upload_ms: null, + counts: {}, + issue_count: 0, + note_count: 0, + contacts_new: 0, + contacts_changed: 0, + }), + ).toBeNull(); + }); +}); diff --git a/web/src/screens/settings/storage/storageUtils.ts b/web/src/screens/settings/storage/storageUtils.ts index bbe162293..4cff430de 100644 --- a/web/src/screens/settings/storage/storageUtils.ts +++ b/web/src/screens/settings/storage/storageUtils.ts @@ -92,6 +92,24 @@ export function describeExportScope(scope: Schema["ExportScope"]): string { */ export type AccountImportRun = Schema["AccountImportRun"]; +/** + * The backup an account's own Import Run read: the file the desktop app + * recorded for it, and when the backup was made, each null when the run does + * not say. Null for the owner's view of another account's run, which carries + * neither. + */ +export function importBackup( + run: AccountImportRun, +): { file: string | null; takenAt: string | null } | null { + if (!("backup_taken_at" in run)) return null; + const fingerprint = run.source_fingerprint; + const path = + fingerprint && typeof fingerprint === "object" && "path" in fingerprint + ? fingerprint.path + : null; + return { file: typeof path === "string" && path ? path : null, takenAt: run.backup_taken_at }; +} + /** Human-readable file size (for example "1.2 MB"). */ export function formatBytes(bytes: number): string { if (!Number.isFinite(bytes) || bytes <= 0) return "0 B"; diff --git a/web/src/test/apiShapes.ts b/web/src/test/apiShapes.ts index d5f439376..4e5b321ad 100644 --- a/web/src/test/apiShapes.ts +++ b/web/src/test/apiShapes.ts @@ -59,6 +59,7 @@ export function message(fields: Partial = {}): Schema["Messag tapbacks: [], earlier_versions: [], matched_earlier_version: false, + backup_taken_at: null, conversation: { id: 1, chat_identifier: "x",