fix(import): the later backup decides a message's mark and text, unsent part included (#1741, #1804)

CHANGELOG.md

@@ -48,7 +48,8 @@ released versions carry their date on the heading.
   backup imported after a newer one changes nothing. Import details under
   Settings → Storage show the backup each import read and when it was made.
   Message files exported before they said when their backup was made are
-  refused, and the backup must be exported again with this build.
+  refused, and the backup must be exported again with this build (#1741,
+  #1804).
 - 2026-10-05: **A WhatsApp reply now names the message it quotes.** When
   the quoted message is in the same chat of the same backup, the reply is
   linked to it, as Apple Messages replies already were: a mail export threads

crates/server/server/src/db/staging.rs

@@ -1252,6 +1252,7 @@ pub async fn write_message_map(
 /// copies agree, or two reads of one Mac's `chat.db` that Messages did
 /// not write between, where it adds a mark and takes a later edit as it
 /// would with no dates.
+///
 /// The one rule for which of two copies of a message from one source is
 /// the later backup, for a stored message ([`promote_deletion_marks`],
 /// [`write_edit_map`]) and, in Rust ([`later_backup`]), for two copies
@@ -1422,9 +1423,11 @@ pub async fn add_staged_copy_milliseconds(
 /// Whether one copy of a message records a later edit than another, as an
 /// SQL expression over four SQL values: the copy's earlier-version count
 /// `n` and newest `edited_at` `newest`, and the other copy's `held_n` and
-/// `held_newest`. The one rule for which of two copies of a message is the
-/// later backup, for a stored message ([`write_edit_map`]) and for two
-/// copies staged in one import ([`take_later_staged_copy`]).
+/// `held_newest`. It is the fallback for which of two copies of a message
+/// gives its text when either copy's backup has no date, or the two dates
+/// are equal: for a stored message ([`write_edit_map`]) and for two copies
+/// staged in one import ([`take_later_staged_copy`]). Where the two backups
+/// have different dates, [`later_backup_sql`] decides instead.
 ///
 /// The copy is the later one when its newest earlier version is newer than
 /// the other's, or as new and it lists more of them. When either side
@@ -1437,8 +1440,7 @@ pub async fn add_staged_copy_milliseconds(
 /// The newest earlier version is the edit before the last one: the time of
 /// a part's last edit is recorded nowhere. So a later backup that differs
 /// only by an unsent part, or by one edit after an unsend, can read as not
-/// later (#1804). The rule is used only where one of the two backups has no
-/// date; where both have one, [`later_backup_sql`] decides instead.
+/// later (#1804), which is why the backups' dates decide wherever they can.
 ///
 /// `edited_at` has the one text form of a stored time on both sides
 /// (`models::utc_timestamp_text`), so the text orders as the time.

crates/server/server/src/imports_api/tests/backup_dates.rs

@@ -75,34 +75,71 @@ async fn held(db: &Path) -> Held {
     }
 }
 
+/// How many messages a search of the message text for `word` finds in the
+/// database at `db`.
+async fn text_hits(db: &Path, word: &str) -> i64 {
+    fts_hits(db, "messages_fts", word).await
+}
+
+/// How many earlier versions a search for `word` finds in the database at
+/// `db`. The index keeps them for an Unsent message too.
+async fn version_hits(db: &Path, word: &str) -> i64 {
+    fts_hits(db, "message_versions_fts", word).await
+}
+
+/// How many rows of the search index `table` match `word` in the database
+/// at `db`.
+async fn fts_hits(db: &Path, table: &str, word: &str) -> i64 {
+    let (_pool, mut conn) = open_verify(db).await;
+    sqlx::query_scalar(&format!(
+        "SELECT COUNT(*) FROM {table} WHERE {table} MATCH $1"
+    ))
+    .bind(word)
+    .fetch_one(&mut *conn)
+    .await
+    .unwrap()
+}
+
 /// Import `files` into `db` in one import, appending to what it holds.
 async fn import(db: &Path, assets: &Path, root: &Path, files: &[PathBuf]) {
     import_jsonl_files(db, files, &edit_options(assets, root, false))
         .await
         .unwrap();
 }
 
+/// One way of importing two files, and what it gave `g-backup`.
+struct Imported {
+    /// The way, such as `apart-reversed`.
+    order: &'static str,
+    /// The database the files were imported into.
+    db: PathBuf,
+    held: Held,
+}
+
 /// What `files` give `g-backup`: imported in one import in the order
 /// given, in one import in the other order, one import after another in
 /// the order given, and one after another in the other order. The four
-/// must agree, so each is returned to compare.
-async fn every_order(tmp: &Path, label: &str, files: [&PathBuf; 2]) -> [Held; 4] {
+/// must agree, so each is returned to compare, with its database to look
+/// into further.
+async fn every_order(tmp: &Path, label: &str, files: [&PathBuf; 2]) -> [Imported; 4] {
     let assets = tmp.join("assets");
     let [a, b] = files;
     let mut out = Vec::new();
-    for (name, batches) in [
+    for (order, batches) in [
         ("together", vec![vec![a.clone(), b.clone()]]),
         ("together-reversed", vec![vec![b.clone(), a.clone()]]),
         ("apart", vec![vec![a.clone()], vec![b.clone()]]),
         ("apart-reversed", vec![vec![b.clone()], vec![a.clone()]]),
     ] {
-        let db = tmp.join(format!("{label}-{name}.db"));
+        let db = tmp.join(format!("{label}-{order}.db"));
         for batch in batches {
             import(&db, &assets, tmp, &batch).await;
         }
-        out.push(held(&db).await);
+        let held = held(&db).await;
+        out.push(Imported { order, db, held });
     }
-    out.try_into().unwrap()
+    out.try_into()
+        .unwrap_or_else(|_| unreachable!("four orders"))
 }
 
 /// The stored date of [`LATER_BACKUP`], in the form a timestamp takes.
@@ -135,7 +172,7 @@ async fn the_later_backup_decides_the_deletion_mark_in_every_order() {
         Some(Deletion::DeletedInSourceApp),
     );
     let recovered = file("recovered-later.jsonl", LATER_BACKUP, None);
-    for held in every_order(tmp.path(), "recovered", [&marked, &recovered]).await {
+    for Imported { held, .. } in every_order(tmp.path(), "recovered", [&marked, &recovered]).await {
         assert_eq!(held.deletion, None, "{held:?}");
         assert_eq!(held.backup_taken_at.as_deref(), Some(LATER_BACKUP_AT));
     }
@@ -146,7 +183,7 @@ async fn the_later_backup_decides_the_deletion_mark_in_every_order() {
         Some(Deletion::DeletedInSourceApp),
     );
     let unmarked = file("unmarked-earlier.jsonl", EARLIER_BACKUP, None);
-    for held in every_order(tmp.path(), "deleted", [&unmarked, &marked]).await {
+    for Imported { held, .. } in every_order(tmp.path(), "deleted", [&unmarked, &marked]).await {
         assert_eq!(
             held.deletion.as_deref(),
             Some("deleted_in_source_app"),
@@ -181,18 +218,13 @@ async fn a_later_backup_that_clears_an_unsent_mark_makes_the_text_searchable() {
         Some(Deletion::Unsent),
     );
     let shown = file("shown-later.jsonl", LATER_BACKUP, None);
-    for held in every_order(tmp.path(), "unsent", [&unsent, &shown]).await {
-        assert_eq!(held.deletion, None, "{held:?}");
-    }
-    for name in ["together", "together-reversed", "apart", "apart-reversed"] {
-        let (_pool, mut conn) = open_verify(&tmp.path().join(format!("unsent-{name}.db"))).await;
-        let hits: i64 = sqlx::query_scalar(
-            "SELECT COUNT(*) FROM messages_fts WHERE messages_fts MATCH 'zqlighthouse'",
-        )
-        .fetch_one(&mut *conn)
-        .await
-        .unwrap();
-        assert_eq!(hits, 1, "{name}: the text is searchable again");
+    for Imported { order, db, held } in every_order(tmp.path(), "unsent", [&unsent, &shown]).await {
+        assert_eq!(held.deletion, None, "{order}: {held:?}");
+        assert_eq!(
+            text_hits(&db, "zqlighthouse").await,
+            1,
+            "{order}: the text is searchable again"
+        );
     }
 }
 
@@ -217,63 +249,92 @@ async fn equal_backup_dates_fall_back_to_the_rules_for_files_without_one() {
     };
     let unmarked = file("unmarked-same.jsonl", None);
     let marked = file("marked-same.jsonl", Some(Deletion::Unsent));
-    for held in every_order(tmp.path(), "same-date", [&unmarked, &marked]).await {
+    for Imported { held, .. } in every_order(tmp.path(), "same-date", [&unmarked, &marked]).await {
         assert_eq!(held.deletion.as_deref(), Some("unsent"), "{held:?}");
         assert_eq!(held.backup_taken_at.as_deref(), Some(LATER_BACKUP_AT));
     }
 }
 
-/// The scenario of #1804: backup A lists part 1's earlier versions
-/// [x@t0, y@t100]; backup B, made later, after part 1 was unsent (which
-/// drops its versions) and part 0 was edited, lists part 0's [a@t0] only.
-/// B's newest version is older than A's, so the version times say A is
-/// later; the backups' dates say B, and B's text and versions are kept in
-/// every order. With the dates swapped, A's are.
+/// The scenario of #1804. A two-part message is sent at t0; part 1 is
+/// edited at t100 and again at t500, which gives it its current text;
+/// backup A is made; part 1 is unsent at t600, which drops its versions
+/// and marks the message Unsent; part 0 is edited at t700; backup B is
+/// made. A lists part 1's [x@t0, y@t100] and no mark; B lists part 0's
+/// [a@t0] and the Unsent mark. B's newest version is older than A's, so
+/// the version times say A is later.
+///
+/// With B the later backup, every order, one import of both files in
+/// either order included, holds B's text, versions and mark: search finds
+/// neither copy's text, finds B's earlier version, and finds none of A's.
+/// With the dates swapped, every order holds A's text and versions with no
+/// mark, and search finds A's text. With no dates, the version times
+/// decide the text as #1801 left them, and B's mark adds.
 #[tokio::test]
 async fn the_later_backup_decides_the_text_whatever_the_version_times_say() {
     let tmp = TempDir::new().unwrap();
     let t0 = 1_426_183_462_000;
-    let a_versions = [edit_version(1, "x", t0), edit_version(1, "y", t0 + 100_000)];
-    let b_versions = [edit_version(0, "a", t0)];
+    let a_versions = [
+        edit_version(1, "zqxylo", t0),
+        edit_version(1, "zqyarrow", t0 + 100_000),
+    ];
+    let b_versions = [edit_version(0, "zqharbor", t0)];
     let a = |backup| Copy {
-        backup: Some(backup),
-        text: "a z",
+        backup,
+        text: "zqharbor zqyonder",
         versions: &a_versions,
         deletion: None,
     };
     let b = |backup| Copy {
-        backup: Some(backup),
-        text: "b",
+        backup,
+        text: "zqbeacon",
         versions: &b_versions,
-        deletion: None,
+        deletion: Some(Deletion::Unsent),
+    };
+    let a_held = |deletion: Option<&str>, backup: Option<&str>| Held {
+        text: "zqharbor zqyonder".into(),
+        deletion: deletion.map(Into::into),
+        versions: vec![(1, "zqxylo".into()), (1, "zqyarrow".into())],
+        backup_taken_at: backup.map(Into::into),
     };
 
-    let a_earlier = backup_file(tmp.path(), "a-earlier.jsonl", &a(EARLIER_BACKUP));
-    let b_later = backup_file(tmp.path(), "b-later.jsonl", &b(LATER_BACKUP));
-    for held in every_order(tmp.path(), "b-later", [&a_earlier, &b_later]).await {
+    let a_earlier = backup_file(tmp.path(), "a-earlier.jsonl", &a(Some(EARLIER_BACKUP)));
+    let b_later = backup_file(tmp.path(), "b-later.jsonl", &b(Some(LATER_BACKUP)));
+    for Imported { order, db, held } in
+        every_order(tmp.path(), "b-later", [&a_earlier, &b_later]).await
+    {
         assert_eq!(
             held,
             Held {
-                text: "b".into(),
-                deletion: None,
-                versions: vec![(0, "a".into())],
+                text: "zqbeacon".into(),
+                deletion: Some("unsent".into()),
+                versions: vec![(0, "zqharbor".into())],
                 backup_taken_at: Some(LATER_BACKUP_AT.into()),
-            }
+            },
+            "{order}"
         );
+        assert_eq!(text_hits(&db, "zqbeacon").await, 0, "{order}");
+        assert_eq!(text_hits(&db, "zqyonder").await, 0, "{order}");
+        assert_eq!(version_hits(&db, "zqharbor").await, 1, "{order}");
+        assert_eq!(version_hits(&db, "zqxylo").await, 0, "{order}");
+        assert_eq!(version_hits(&db, "zqyarrow").await, 0, "{order}");
     }
 
-    let a_later = backup_file(tmp.path(), "a-later.jsonl", &a(LATER_BACKUP));
-    let b_earlier = backup_file(tmp.path(), "b-earlier.jsonl", &b(EARLIER_BACKUP));
-    for held in every_order(tmp.path(), "a-later", [&a_later, &b_earlier]).await {
-        assert_eq!(
-            held,
-            Held {
-                text: "a z".into(),
-                deletion: None,
-                versions: vec![(1, "x".into()), (1, "y".into())],
-                backup_taken_at: Some(LATER_BACKUP_AT.into()),
-            }
-        );
+    let a_later = backup_file(tmp.path(), "a-later.jsonl", &a(Some(LATER_BACKUP)));
+    let b_earlier = backup_file(tmp.path(), "b-earlier.jsonl", &b(Some(EARLIER_BACKUP)));
+    for Imported { order, db, held } in
+        every_order(tmp.path(), "a-later", [&a_later, &b_earlier]).await
+    {
+        assert_eq!(held, a_held(None, Some(LATER_BACKUP_AT)), "{order}");
+        assert_eq!(text_hits(&db, "zqyonder").await, 1, "{order}");
+        assert_eq!(text_hits(&db, "zqbeacon").await, 0, "{order}");
+    }
+
+    let a_undated = backup_file(tmp.path(), "a-undated.jsonl", &a(None));
+    let b_undated = backup_file(tmp.path(), "b-undated.jsonl", &b(None));
+    for Imported { order, held, .. } in
+        every_order(tmp.path(), "undated", [&a_undated, &b_undated]).await
+    {
+        assert_eq!(held, a_held(Some("unsent"), None), "{order}");
     }
 }
 
@@ -347,7 +408,9 @@ async fn without_a_backup_date_marks_add_and_edits_compare_their_times() {
     );
     // Undated against dated: the mark adds, and the copy with the later edit
     // keeps its text, whichever is the dated one.
-    for held in every_order(tmp.path(), "mixed", [&unmarked_undated, &marked_dated]).await {
+    for Imported { held, .. } in
+        every_order(tmp.path(), "mixed", [&unmarked_undated, &marked_dated]).await
+    {
         assert_eq!(held.deletion.as_deref(), Some("unsent"), "{held:?}");
         assert_eq!(held.text, "see you at seven", "{held:?}");
         assert_eq!(held.versions, vec![(0, "see you at six".into())]);
@@ -365,7 +428,9 @@ async fn without_a_backup_date_marks_add_and_edits_compare_their_times() {
             deletion: Some(Deletion::DeletedInSourceApp),
         },
     );
-    for held in every_order(tmp.path(), "undated", [&unmarked_undated, &marked_undated]).await {
+    for Imported { held, .. } in
+        every_order(tmp.path(), "undated", [&unmarked_undated, &marked_undated]).await
+    {
         assert_eq!(
             held.deletion.as_deref(),
             Some("deleted_in_source_app"),