diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d140aad5..911dac6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,21 @@ released versions carry their date on the heading. ### Features +- 2026-10-06: **A message recorded once to the second and once to the + millisecond shows once.** Every message file now says whether each + message's time has milliseconds or only whole seconds, as its backup app + recorded it. iMazing, OpenExtract, GO SMS Pro's PDU files and SMS Backup+ + mails timed only to the second record whole seconds; the other sources + record milliseconds. When one backup app holds a message twice, once to + the second and once to the millisecond, such as two SMS Backup+ mails of + one message, hiding duplicates hides the whole-second copy and shows the + message at its time to the millisecond. When another backup app holds the + message too, its copy may be the one shown, and it can have whole seconds + only. A time to the millisecond that ends in `.000` counts as + milliseconds. CSV exports carry it in a `time_precision` column, and mail + exports in an `X-ME-Time-Precision` header. Message files exported before + they said whether each time has milliseconds are refused, and the backup + must be exported again with this build. - 2026-10-05: **The newer backup decides when a message changed between two backups of one phone.** Every message file now says when its backup was made: an iPhone backup's own date, the date an SMS Backup & Restore file @@ -181,6 +196,10 @@ released versions carry their date on the heading. refused when you import or convert them. Export the backup again with this build. A program that reads the HTTP API finds the backup's date in a message's `backup_taken_at` and an Import Run's `backup_taken_at`. +- Message files exported before each message said whether its time has + milliseconds are refused when you import or convert them. Export the + backup again with this build. A program that reads the HTTP API finds it + in a message's `time_precision`, `seconds` or `milliseconds`. ## [0.10.1] - 2026-10-05 diff --git a/crates/core/message-crate-core/src/attachment_jobs/tests.rs b/crates/core/message-crate-core/src/attachment_jobs/tests.rs index 67b69de56..46868cc0c 100644 --- a/crates/core/message-crate-core/src/attachment_jobs/tests.rs +++ b/crates/core/message-crate-core/src/attachment_jobs/tests.rs @@ -480,6 +480,7 @@ fn staging_a_conversation_writes_the_files_counts_them_and_frees_the_bytes() { messages: vec![IrMessage { guid: "guid-1".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Mms, @@ -723,6 +724,7 @@ fn staging_frees_the_bytes_the_documents_were_carrying() { messages: vec![IrMessage { guid: "guid-1".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Mms, diff --git a/crates/exporters/go-sms-pro-exporter/tests/convert_smoke.rs b/crates/exporters/go-sms-pro-exporter/tests/convert_smoke.rs index cacad45b2..910a2dc3f 100644 --- a/crates/exporters/go-sms-pro-exporter/tests/convert_smoke.rs +++ b/crates/exporters/go-sms-pro-exporter/tests/convert_smoke.rs @@ -60,6 +60,8 @@ fn convert_smoke_writes_csv_not_json() { ("direction", "incoming"), ("sender_identity", "+14075550107"), ("timestamp_unix_ms", "1609459200000"), + // The XML records milliseconds, though these end in `.000`. + ("time_precision", "milliseconds"), ("chat_identifier", "+14075550107"), ], ); @@ -73,6 +75,7 @@ fn convert_smoke_writes_csv_not_json() { ("text", "smoke reply"), ("direction", "outgoing"), ("timestamp_unix_ms", "1609459260000"), + ("time_precision", "milliseconds"), ], ); // The PDU beside the XML lands in the same conversation. @@ -82,6 +85,8 @@ fn convert_smoke_writes_csv_not_json() { ("text", "Hello one to one"), ("direction", "incoming"), ("sender_identity", "+14075550107"), + // A PDU file's name records whole seconds only. + ("time_precision", "seconds"), ], ); } diff --git a/crates/exporters/imazing-exporter/tests/convert_smoke.rs b/crates/exporters/imazing-exporter/tests/convert_smoke.rs index 9feb6c825..ba4a0fe43 100644 --- a/crates/exporters/imazing-exporter/tests/convert_smoke.rs +++ b/crates/exporters/imazing-exporter/tests/convert_smoke.rs @@ -49,9 +49,18 @@ fn convert_messages_keys_the_chat_by_its_number() { ("direction", "incoming"), ("service", "sms"), ("sender_display_name", "Bob Sample"), + // iMazing writes whole seconds only. + ("time_precision", "seconds"), + ], + ); + assert_csv_row( + &out, + &[ + ("text", "Hi Bob"), + ("direction", "outgoing"), + ("time_precision", "seconds"), ], ); - assert_csv_row(&out, &[("text", "Hi Bob"), ("direction", "outgoing")]); // The third row is an iMessage carrying an attachment, so it proves both // that the service column follows the source and that the attachment file // name reached the row rather than only the directory. @@ -91,7 +100,11 @@ fn convert_whatsapp_csv_direct() { // parse that lost the column or put the flag on the wrong message fails. assert_csv_row( &out, - &[("text", "Hello on WhatsApp"), ("direction", "incoming")], + &[ + ("text", "Hello on WhatsApp"), + ("direction", "incoming"), + ("time_precision", "seconds"), + ], ); assert_csv_row( &out, diff --git a/crates/exporters/imessage-ir-exporter/src/convert.rs b/crates/exporters/imessage-ir-exporter/src/convert.rs index 6fbf5a7a2..ef316c9df 100644 --- a/crates/exporters/imessage-ir-exporter/src/convert.rs +++ b/crates/exporters/imessage-ir-exporter/src/convert.rs @@ -28,7 +28,7 @@ use message_crate_core::{ use message_ir::{ ConversationDocument, ConversationMeta, ExportMeta, HandleType, IrAttachment, IrConversationType, IrDirection, IrImessage, IrMessage, IrMessageKind, IrParticipant, - IrService, SCHEMA_VERSION, nonempty, owner_sender, + IrService, SCHEMA_VERSION, TimePrecision, nonempty, owner_sender, }; use message_ir_format::FormatSink; use message_staging::{ @@ -410,6 +410,9 @@ fn message_to_ir( let message = IrMessage { guid: record.guid, timestamp_unix_ms: record.timestamp_unix_ms, + // `chat.db` records a message's time in nanoseconds (since macOS + // 10.13 and iOS 11). + time_precision: TimePrecision::Milliseconds, direction, service: IrService::parse(&record.service), message_kind: IrMessageKind::parse(&record.message_kind), @@ -1317,6 +1320,7 @@ mod tests { IrMessage { guid: format!("guid-{ts}"), timestamp_unix_ms: ts, + time_precision: TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::IMessage, message_kind: IrMessageKind::IMessage, diff --git a/crates/exporters/imessage-ir-exporter/tests/helper_process.rs b/crates/exporters/imessage-ir-exporter/tests/helper_process.rs index 46dc3942c..1c58b262b 100644 --- a/crates/exporters/imessage-ir-exporter/tests/helper_process.rs +++ b/crates/exporters/imessage-ir-exporter/tests/helper_process.rs @@ -22,7 +22,7 @@ use chat_db_fixture::{ }; use common::{config, helper_binary}; use message_crate_core::{ExporterConfig, OutputFormat}; -use message_ir::{ConversationDocument, Deletion, IrDirection, IrMessage}; +use message_ir::{ConversationDocument, Deletion, IrDirection, IrMessage, TimePrecision}; use message_ir_format::{ read_conversation_csv, read_conversation_eml_dir, read_conversation_jsonl, read_conversation_mbox, @@ -82,6 +82,20 @@ fn exports_a_mac_chat_db_through_the_helper_process() { assert_eq!(staged.len(), 1, "{staged:?}"); assert_eq!(fs::read(&staged[0]).unwrap(), PHOTO_BYTES); assert!(all.contains("\"attachments/"), "{all}"); + + // `chat.db` records times below the second, so every message says + // milliseconds. + for path in &files { + let doc = read_conversation_jsonl(path).unwrap(); + for msg in &doc.messages { + assert_eq!( + msg.time_precision, + TimePrecision::Milliseconds, + "{}", + msg.guid + ); + } + } } /// Every conversation file of a Mac `chat.db` export says the database's diff --git a/crates/exporters/openextract-exporter/tests/convert_smoke.rs b/crates/exporters/openextract-exporter/tests/convert_smoke.rs index c29a10f8b..7feea154f 100644 --- a/crates/exporters/openextract-exporter/tests/convert_smoke.rs +++ b/crates/exporters/openextract-exporter/tests/convert_smoke.rs @@ -68,6 +68,8 @@ fn convert_all_conversations_keys_the_chat_by_its_number() { ("sender_identity", "+15555550122"), // 2020-01-01T17:00:00+00:00 in the source. ("timestamp_unix_ms", "1577898000000"), + // OpenExtract writes whole seconds only. + ("time_precision", "seconds"), ], ); // "Is From Me" is True on this row, and the direction column is where that @@ -78,6 +80,7 @@ fn convert_all_conversations_keys_the_chat_by_its_number() { ("text", "Hi Sam"), ("direction", "outgoing"), ("timestamp_unix_ms", "1577898060000"), + ("time_precision", "seconds"), ], ); } diff --git a/crates/exporters/sms-backup-plus-exporter/src/emit.rs b/crates/exporters/sms-backup-plus-exporter/src/emit.rs index 41d56152d..a2965c7ea 100644 --- a/crates/exporters/sms-backup-plus-exporter/src/emit.rs +++ b/crates/exporters/sms-backup-plus-exporter/src/emit.rs @@ -851,6 +851,7 @@ mod tests { messages: vec![IrMessage { guid: "g".into(), timestamp_unix_ms: 0, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Mms, diff --git a/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs b/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs index bb7a7a7f7..fe0c6555d 100644 --- a/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs +++ b/crates/exporters/sms-backup-plus-exporter/tests/convert_smoke.rs @@ -86,6 +86,9 @@ fn convert_smoke_writes_csv_not_json() { ("text", "Hello from Alice"), ("direction", "incoming"), ("timestamp_unix_ms", "1609459200000"), + // `X-smssync-date` records milliseconds, though these end in + // `.000`. + ("time_precision", "milliseconds"), ("chat_identifier", "+14075550107"), ], ); @@ -316,6 +319,7 @@ fn two_messages_sharing_an_smssync_id_both_survive() { ("text", "Hello from Alex"), ("direction", "outgoing"), ("timestamp_unix_ms", "1609459200313"), + ("time_precision", "milliseconds"), ], ); assert_csv_row( @@ -323,6 +327,7 @@ fn two_messages_sharing_an_smssync_id_both_survive() { &[ ("text", "Hello from Sam"), ("timestamp_unix_ms", "1609459300000"), + ("time_precision", "milliseconds"), ], ); // The same-chat pair: both messages, in one conversation file. @@ -434,3 +439,37 @@ fn the_backup_date_is_the_newest_mail_files_modification_time() { "{dates:?}" ); } + +/// A message with no `X-smssync-date` takes its time from the mail's `Date`, +/// which records whole seconds, and says so. +#[test] +fn a_message_timed_by_its_date_header_has_whole_seconds() { + let tmp = tempfile::tempdir().expect("tempdir"); + let input = tmp.path().join("in"); + let out = tmp.path().join("out"); + fs::create_dir_all(&input).expect("input dir"); + fs::write( + input.join("dated.eml"), + "From: dana@unknown.email\n\ + To: me@example.com\n\ + Subject: SMS with Dana\n\ + Date: Fri, 01 Jan 2021 00:00:05 +0000\n\ + X-smssync-type: 1\n\ + X-smssync-address: +15555550133\n\ + Content-Type: text/plain; charset=utf-8\n\ + \n\ + Timed by Date\n", + ) + .expect("write fixture"); + + convert(&[input.as_path()], &out).expect("convert"); + + assert_csv_row( + &out.join("+15555550133.csv"), + &[ + ("text", "Timed by Date"), + ("timestamp_unix_ms", "1609459205000"), + ("time_precision", "seconds"), + ], + ); +} diff --git a/crates/exporters/sms-backup-restore-exporter/src/read.rs b/crates/exporters/sms-backup-restore-exporter/src/read.rs index 9c4f8c715..8b95ceaa9 100644 --- a/crates/exporters/sms-backup-restore-exporter/src/read.rs +++ b/crates/exporters/sms-backup-restore-exporter/src/read.rs @@ -493,8 +493,10 @@ fn dedupe(messages: &mut Vec) -> u64 { let before = messages.len(); let mut kept = kept.into_iter(); messages.retain_mut(|m| match kept.next().flatten() { - Some(ms) => { - if m.time().0 != ms { + Some((ms, precision)) => { + // A whole-second copy that keeps a millisecond copy's time keeps + // its precision too, even when that time ends in `.000`. + if m.time() != (ms, precision) { m.date_ms = ms.to_string(); } true @@ -624,7 +626,7 @@ fn ir_message( message: &PendingMessage, owner: &(Option, Option), ) -> IrMessage { - let (timestamp_unix_ms, _) = message.time(); + let (timestamp_unix_ms, time_precision) = message.time(); let digests = message.attachment_digests(); let (sender_identity, sender_display_name) = if message.is_from_me { owner.clone() @@ -643,6 +645,7 @@ fn ir_message( }) .into_string(), timestamp_unix_ms, + time_precision, direction: if message.is_from_me { IrDirection::Outgoing } else { diff --git a/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs b/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs index 3392361e3..3af8a95e9 100644 --- a/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs +++ b/crates/exporters/sms-backup-restore-exporter/tests/convert_smoke.rs @@ -70,6 +70,7 @@ fn convert_export_smoke_on_sample_fixture() { ("text", "hello"), ("direction", "incoming"), ("timestamp_unix_ms", "1400773261000"), + ("time_precision", "milliseconds"), ("chat_identifier", "+15555550101"), ], ); @@ -83,6 +84,7 @@ fn convert_export_smoke_on_sample_fixture() { ("text", "hey"), ("direction", "outgoing"), ("timestamp_unix_ms", "1400773321000"), + ("time_precision", "milliseconds"), ], ); // The `` is a different parse: its text lives in a `text/plain` part @@ -95,6 +97,7 @@ fn convert_export_smoke_on_sample_fixture() { ("direction", "incoming"), ("message_kind", "mms"), ("timestamp_unix_ms", "1400773400000"), + ("time_precision", "milliseconds"), ], ); @@ -344,7 +347,7 @@ fn convert_export_json_and_jsonl_use_pristine_v4() { .expect("expected .json"); let raw = fs::read_to_string(&json_path).unwrap(); let doc: serde_json::Value = serde_json::from_str(&raw).unwrap(); - assert_eq!(doc["schema_version"], 11); + assert_eq!(doc["schema_version"], 12); assert!( doc["conversation"]["stats"]["message_count"] .as_u64() @@ -389,7 +392,7 @@ fn convert_export_json_and_jsonl_use_pristine_v4() { let body = fs::read_to_string(&jsonl_path).unwrap(); let mut lines = body.lines(); let header: serde_json::Value = serde_json::from_str(lines.next().unwrap()).unwrap(); - assert_eq!(header["schema_version"], 11); + assert_eq!(header["schema_version"], 12); assert!(header.get("messages").is_none()); assert!( header["conversation"]["stats"]["message_count"] diff --git a/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs b/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs index 22d181bbf..4daa16736 100644 --- a/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs +++ b/crates/exporters/whatsapp-exporter/tests/convert_smoke.rs @@ -230,6 +230,18 @@ fn messages_keep_their_time_and_their_identity() { (9_999_999_999_000, "last second"), ] ); + // WhatsApp stores milliseconds, and the seconds wtsexporter writes carry + // them as a fraction, so every time has milliseconds, whole or not. + assert!( + doc.messages + .iter() + .all(|m| m.time_precision == message_ir::TimePrecision::Milliseconds), + "{:?}", + doc.messages + .iter() + .map(|m| m.time_precision) + .collect::>() + ); assert_ne!( doc.messages[1].guid, doc.messages[2].guid, "the same text in the same second is two messages" diff --git a/crates/libs/api-types/src/lib.rs b/crates/libs/api-types/src/lib.rs index ad4e4479f..b4da4b702 100644 --- a/crates/libs/api-types/src/lib.rs +++ b/crates/libs/api-types/src/lib.rs @@ -380,6 +380,10 @@ api_shape! { /// (`Account.time_zone`); the database stores nothing /// about where the phone was. pub timestamp: String, + /// Whether the source recorded `timestamp` to the millisecond or in + /// whole seconds. A `timestamp` ending in `.000` is a whole second + /// only when this says `seconds`. + pub time_precision: TimePrecision, /// Ordering key within the conversation. pub sort_order: i64, /// True for messages sent by the account owner. @@ -470,6 +474,36 @@ api_shape! { } } +/// How finely the source recorded a message's time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +#[serde(rename_all = "lowercase")] +pub enum TimePrecision { + /// Whole seconds: the source records no milliseconds, and the time's + /// milliseconds are `.000`. + Seconds, + /// Milliseconds, as the phone stored them; they can be `.000` too. + Milliseconds, +} + +impl TimePrecision { + /// Both precisions. + pub const ALL: [Self; 2] = [Self::Seconds, Self::Milliseconds]; + + /// The precision as the wire and the database spell it. + pub const fn as_str(self) -> &'static str { + match self { + Self::Seconds => "seconds", + Self::Milliseconds => "milliseconds", + } + } + + /// Read a sent or stored value; anything else names no precision. + pub fn parse(value: &str) -> Option { + Self::ALL.into_iter().find(|p| p.as_str() == value) + } +} + /// Why a message's content is gone in the app it came from. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] @@ -612,6 +646,7 @@ mod tests { service: None, guid: "g1".into(), timestamp: "2024-01-01T00:00:00.000Z".into(), + time_precision: TimePrecision::Seconds, sort_order: 0, is_from_me: false, sender: None, @@ -725,6 +760,8 @@ mod tests { assert_eq!(read.attachments.len(), 1); assert_eq!(read.tapbacks[0].kind, "loved"); assert_eq!(written["deletion"], "deleted_in_source_app"); + assert_eq!(written["time_precision"], "seconds"); + assert_eq!(read.time_precision, TimePrecision::Seconds); assert_eq!(read.deletion, Some(Deletion::DeletedInSourceApp)); assert_eq!(read.earlier_versions[0].text, "helo"); assert_eq!( diff --git a/crates/libs/export/src/project.rs b/crates/libs/export/src/project.rs index 0b5f6a134..97948ffc8 100644 --- a/crates/libs/export/src/project.rs +++ b/crates/libs/export/src/project.rs @@ -11,6 +11,7 @@ use message_ir::{ ConversationDocument, ConversationMeta, ConversationStats, Deletion, EarlierVersion, ExportMeta, IrAttachment, IrConversationType, IrDirection, IrImessage, IrMessage, IrMessageKind, IrParticipant, IrService, IrSource, Reaction, ReplyTo, SCHEMA_VERSION, + TimePrecision, }; use serde_json::json; @@ -147,6 +148,7 @@ pub fn to_ir_message(msg: &Message, skip_attachments: bool) -> Result Ok(IrMessage { guid: msg.guid.clone(), timestamp_unix_ms, + time_precision: time_precision_from_api(msg.time_precision), direction, service, message_kind, @@ -191,6 +193,14 @@ fn earlier_version_from_api( }) } +/// The precision the server stored, as the conversation file writes it. +fn time_precision_from_api(precision: message_crate_api_types::TimePrecision) -> TimePrecision { + match precision { + message_crate_api_types::TimePrecision::Seconds => TimePrecision::Seconds, + message_crate_api_types::TimePrecision::Milliseconds => TimePrecision::Milliseconds, + } +} + /// The mark a server message carries, as the conversation file writes it. fn deletion_from_api(deletion: message_crate_api_types::Deletion) -> Deletion { match deletion { @@ -403,6 +413,7 @@ mod tests { "service": "iMessage", "guid": "3A9E-0001", "timestamp": "2015-03-12T18:05:22Z", + "time_precision": "milliseconds", "sort_order": 0, "is_from_me": false, "sender": "+15555550100", @@ -758,6 +769,36 @@ mod tests { assert!(ir.imessage.is_none()); } + /// The conversation file says the precision the server stored, so a + /// whole-second message is written back as whole seconds and a + /// millisecond time that ends in `.000` as milliseconds. + #[test] + fn a_message_keeps_the_precision_the_server_stored() { + let participant = Participant { + identity: Some("+1".into()), + name: "Sam".into(), + service: None, + contact_id: None, + }; + let mut msg = seed_message_with_participant(participant); + msg.timestamp = "2015-03-12T18:05:22.000Z".into(); + for (stored, written) in [ + ( + message_crate_api_types::TimePrecision::Seconds, + TimePrecision::Seconds, + ), + ( + message_crate_api_types::TimePrecision::Milliseconds, + TimePrecision::Milliseconds, + ), + ] { + msg.time_precision = stored; + let ir = to_ir_message(&msg, false).unwrap(); + assert_eq!(ir.timestamp_unix_ms, 1_426_183_522_000); + assert_eq!(ir.time_precision, written); + } + } + #[test] fn maps_basic_message() { let msg = Message { @@ -766,6 +807,7 @@ mod tests { service: Some("iMessage".into()), guid: "g1".into(), timestamp: "2015-03-12T18:05:22Z".into(), + time_precision: message_crate_api_types::TimePrecision::Milliseconds, is_from_me: false, sender: Some("+1".into()), owner: None, @@ -923,6 +965,7 @@ mod tests { service: Some("iMessage".into()), guid: "g1".into(), timestamp: "2015-03-12T18:05:22Z".into(), + time_precision: message_crate_api_types::TimePrecision::Milliseconds, is_from_me: false, sender: Some("+1".into()), owner: None, diff --git a/crates/libs/export/src/run.rs b/crates/libs/export/src/run.rs index 7fd81a9b4..68d9be104 100644 --- a/crates/libs/export/src/run.rs +++ b/crates/libs/export/src/run.rs @@ -980,6 +980,7 @@ mod asset_ref_tests { "source": source, "guid": "g1", "timestamp": "2015-03-12T18:05:22Z", + "time_precision": "milliseconds", "sort_order": 0, "is_from_me": false, "is_announcement": false, diff --git a/crates/libs/export/tests/export_mock.rs b/crates/libs/export/tests/export_mock.rs index 6c35abc6c..b8bb350bd 100644 --- a/crates/libs/export/tests/export_mock.rs +++ b/crates/libs/export/tests/export_mock.rs @@ -77,6 +77,7 @@ fn message( "service": "sms", "guid": guid, "timestamp": timestamp, + "time_precision": "milliseconds", "sort_order": id, "is_from_me": false, "sender": "+15555550101", diff --git a/crates/libs/ir-format/src/export_transforms/tests.rs b/crates/libs/ir-format/src/export_transforms/tests.rs index 44a850d02..9f9d0bfbe 100644 --- a/crates/libs/ir-format/src/export_transforms/tests.rs +++ b/crates/libs/ir-format/src/export_transforms/tests.rs @@ -34,6 +34,7 @@ fn doc_with_image_attachment() -> ConversationDocument { messages: vec![IrMessage { guid: "guid-1".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Sms, @@ -240,6 +241,7 @@ fn doc_with_a_marker_in_every_field() -> ConversationDocument { messages: vec![IrMessage { guid: "LEAK-27".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::IMessage, message_kind: IrMessageKind::IMessage, @@ -313,6 +315,7 @@ const KEPT_AS_IS: &[(&str, &str)] = &[ ), ("conversation.conversation_type", "enum value"), ("conversation.participants[].identity_type", "enum value"), + ("messages[].time_precision", "enum value"), ("messages[].direction", "enum value"), ("messages[].service", "enum value"), ("messages[].message_kind", "enum value"), diff --git a/crates/libs/ir-format/src/lib_tests.rs b/crates/libs/ir-format/src/lib_tests.rs index e8f07c436..efb13e591 100644 --- a/crates/libs/ir-format/src/lib_tests.rs +++ b/crates/libs/ir-format/src/lib_tests.rs @@ -4,6 +4,7 @@ use super::*; use message_crate_core::OutputFormat; use message_ir::{ ConversationDocument, IrDirection, IrImessage, IrMessage, IrMessageKind, IrService, + TimePrecision, }; use serde_json::{Value, json}; use std::fs; @@ -234,6 +235,7 @@ fn csv_omits_trivial_parts_json_keeps_rich_parts() { doc.messages.push(IrMessage { guid: "MULTI-PART-GUID".into(), timestamp_unix_ms: 1_400_773_263_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::IMessage, message_kind: IrMessageKind::IMessage, @@ -904,3 +906,99 @@ fn csv_refuses_a_backup_date_that_is_not_a_number() { "{err:#}" ); } + +/// Write a document of two messages, one whose source recorded whole +/// seconds and one whose source recorded milliseconds that end in `.000`, +/// in `format`, and read back each message's precision in time order. +fn precisions_after_round_trip(format: OutputFormat) -> Vec<(i64, TimePrecision)> { + let mut doc = message_ir::testutil::sample_document("whole second"); + let mut whole = doc.messages[0].clone(); + whole.timestamp_unix_ms = 1_400_773_261_000; + whole.time_precision = TimePrecision::Seconds; + let mut exact = whole.clone(); + exact.guid = "bbccddeeff00112233445566778899aa".into(); + exact.text = "milliseconds that end in .000".into(); + exact.timestamp_unix_ms = 1_400_773_262_000; + exact.time_precision = TimePrecision::Milliseconds; + doc.messages = vec![whole, exact]; + let tmp = tempfile::tempdir().unwrap(); + let path = write_format(tmp.path(), format, doc).unwrap(); + let back = match format { + OutputFormat::Json => read_conversation_json(&path), + OutputFormat::Jsonl => read_conversation_jsonl(&path), + OutputFormat::Csv => read_conversation_csv(&path), + OutputFormat::Eml => read_conversation_eml_dir(&path), + OutputFormat::Mbox => read_conversation_mbox(&path), + other => panic!("no round trip for {}", other.as_str()), + } + .unwrap(); + let mut out: Vec<(i64, TimePrecision)> = back + .messages + .iter() + .map(|m| (m.timestamp_unix_ms, m.time_precision)) + .collect(); + out.sort_unstable(); + out +} + +/// Every format keeps whether a message's time has milliseconds, and a +/// millisecond time that ends in `.000` reads back as milliseconds, never +/// as whole seconds. +#[test] +fn every_format_keeps_whether_a_time_has_milliseconds() { + for format in [ + OutputFormat::Json, + OutputFormat::Jsonl, + OutputFormat::Csv, + OutputFormat::Eml, + OutputFormat::Mbox, + ] { + assert_eq!( + precisions_after_round_trip(format), + [ + (1_400_773_261_000, TimePrecision::Seconds), + (1_400_773_262_000, TimePrecision::Milliseconds), + ], + "{}", + format.as_str() + ); + } +} + +/// A CSV row whose `time_precision` is blank is refused, never read as +/// either precision. +#[test] +fn csv_refuses_a_row_without_a_time_precision() { + let doc = message_ir::testutil::sample_document("hello"); + let tmp = tempfile::tempdir().unwrap(); + let path = write_format(tmp.path(), OutputFormat::Csv, doc).unwrap(); + let text = fs::read_to_string(&path).unwrap(); + let mut lines: Vec = text.lines().map(str::to_string).collect(); + let column = CSV_HEADERS + .iter() + .position(|h| *h == "time_precision") + .unwrap(); + let mut row = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(lines[1].as_bytes()) + .records() + .next() + .unwrap() + .unwrap() + .iter() + .map(str::to_string) + .collect::>(); + row[column] = String::new(); + let mut out = csv::Writer::from_writer(Vec::new()); + out.write_record(&row).unwrap(); + lines[1] = String::from_utf8(out.into_inner().unwrap()) + .unwrap() + .trim_end() + .to_string(); + fs::write(&path, lines.join("\n")).unwrap(); + let err = read_conversation_csv(&path).unwrap_err(); + assert!( + format!("{err:#}").contains("bad time_precision \"\""), + "{err:#}" + ); +} diff --git a/crates/libs/ir-format/src/read_csv.rs b/crates/libs/ir-format/src/read_csv.rs index bc24e55ff..a57d0dc6f 100644 --- a/crates/libs/ir-format/src/read_csv.rs +++ b/crates/libs/ir-format/src/read_csv.rs @@ -7,8 +7,8 @@ use message_csv::{AttachmentCell, ParticipantCell}; use message_ir::{ ConversationDocument, ConversationHeader, ConversationMeta, ConversationStats, EarlierVersion, ExportMeta, IrAttachment, IrConversationType, IrDirection, IrImessage, IrMessage, - IrMessageKind, IrParticipant, IrService, Reaction, ReplyTo, SCHEMA_VERSION, nonempty, - parse_android_type, + IrMessageKind, IrParticipant, IrService, Reaction, ReplyTo, SCHEMA_VERSION, TimePrecision, + nonempty, parse_android_type, }; use serde_json::Value; use std::collections::HashMap; @@ -116,6 +116,10 @@ fn message_from_record(cols: &HashMap<&str, usize>, row: &csv::StringRecord) -> let timestamp_unix_ms = get("timestamp_unix_ms") .parse::() .with_context(|| format!("bad timestamp_unix_ms {:?}", get("timestamp_unix_ms")))?; + // The flag, never the time, says whether a time has milliseconds, so a + // blank or unknown precision is refused rather than guessed. + let time_precision = TimePrecision::parse(get("time_precision")) + .with_context(|| format!("bad time_precision {:?}", get("time_precision")))?; let direction = match get("direction").to_ascii_lowercase().as_str() { "outgoing" => IrDirection::Outgoing, _ => IrDirection::Incoming, @@ -158,6 +162,7 @@ fn message_from_record(cols: &HashMap<&str, usize>, row: &csv::StringRecord) -> Ok(IrMessage { guid: get("guid").to_string(), timestamp_unix_ms, + time_precision, direction, service: IrService::parse(get("service")), message_kind: IrMessageKind::parse(get("message_kind")), diff --git a/crates/libs/ir-format/src/write.rs b/crates/libs/ir-format/src/write.rs index f7d6fbaae..3928e486b 100644 --- a/crates/libs/ir-format/src/write.rs +++ b/crates/libs/ir-format/src/write.rs @@ -26,6 +26,7 @@ pub const CSV_HEADERS: &[&str] = &[ "timestamp_utc", "timestamp_display", "timestamp_unix_ms", + "time_precision", "direction", "service", "sender_identity", @@ -381,7 +382,7 @@ fn csv_record<'a>( backup_taken_at: &'a str, msg: &'a IrMessage, cells: &'a MessageCells, -) -> [&'a str; 47] { +) -> [&'a str; 48] { let im = &cells.imessage; [ doc.conversation.chat_identifier.as_str(), @@ -393,6 +394,7 @@ fn csv_record<'a>( cells.ts_utc.as_str(), cells.ts_display.as_str(), cells.timestamp_unix_ms.as_str(), + msg.time_precision.as_str(), msg.direction.as_str(), msg.service.as_str(), msg.sender_identity.as_deref().unwrap_or(""), diff --git a/crates/libs/ir/src/identity.rs b/crates/libs/ir/src/identity.rs index 6f72fa799..259b73c95 100644 --- a/crates/libs/ir/src/identity.rs +++ b/crates/libs/ir/src/identity.rs @@ -11,11 +11,17 @@ //! so the id carries no counter of occurrences: a counter would depend on the //! order the copies are read in. +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::HashMap; -/// How finely a source recorded a message's time. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +/// How finely a source recorded a message's time: a message's +/// `time_precision` in the conversation file. +/// +/// The flag, never the value, says whether a time has milliseconds: a +/// millisecond time can end in `.000`, and a whole-second one always does. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum TimePrecision { /// Whole seconds: the milliseconds are zero because the source has none. Seconds, @@ -23,6 +29,25 @@ pub enum TimePrecision { Milliseconds, } +impl TimePrecision { + /// Both precisions. + pub const ALL: [Self; 2] = [Self::Seconds, Self::Milliseconds]; + + /// The name the conversation file, the database and the HTTP API use: + /// `seconds` or `milliseconds`. + pub fn as_str(self) -> &'static str { + match self { + Self::Seconds => "seconds", + Self::Milliseconds => "milliseconds", + } + } + + /// The precision [`Self::as_str`] names, or `None` for any other text. + pub fn parse(value: &str) -> Option { + Self::ALL.into_iter().find(|p| p.as_str() == value) + } +} + /// What a message's identity is made from. /// /// The chat is the caller's: an exporter passes the conversation's chat id, @@ -164,10 +189,10 @@ pub struct MessageCopy<'a> { /// which copy was read first. Two millisecond copies with different times /// are two messages. /// -/// Returns one entry per copy, in order: `Some(time)` for a copy that is -/// kept, with the time in milliseconds it keeps, and `None` for a copy that -/// repeats a kept one. -pub fn one_copy_per_message(copies: &[MessageCopy<'_>]) -> Vec> { +/// Returns one entry per copy, in order: `Some((time, precision))` for a +/// copy that is kept, with the time in milliseconds it keeps and how finely +/// that time was recorded, and `None` for a copy that repeats a kept one. +pub fn one_copy_per_message(copies: &[MessageCopy<'_>]) -> Vec> { struct Kept { index: usize, timestamp_unix_ms: i64, @@ -244,7 +269,12 @@ pub fn one_copy_per_message(copies: &[MessageCopy<'_>]) -> Vec> { } } for k in kept { - fate[k.index] = Some(k.timestamp_unix_ms); + let precision = if k.exact { + TimePrecision::Milliseconds + } else { + TimePrecision::Seconds + }; + fate[k.index] = Some((k.timestamp_unix_ms, precision)); } } fate @@ -353,7 +383,10 @@ mod tests { ]; assert_eq!( one_copy_per_message(&copies), - [Some(1_609_459_200_000), Some(1_609_459_200_000)] + [ + Some((1_609_459_200_000, SECS)), + Some((1_609_459_200_000, SECS)) + ] ); } @@ -365,7 +398,7 @@ mod tests { ]; assert_eq!( one_copy_per_message(&copies), - [Some(1_609_459_200_300), None] + [Some((1_609_459_200_300, MS)), None] ); } @@ -377,7 +410,7 @@ mod tests { ]; assert_eq!( one_copy_per_message(&copies), - [Some(1_609_459_200_100), Some(1_609_459_200_400)] + [Some((1_609_459_200_100, MS)), Some((1_609_459_200_400, MS))] ); } @@ -387,11 +420,11 @@ mod tests { let exact = copy("+15555550122", 1_609_459_200_876, MS, &[]); assert_eq!( one_copy_per_message(&[whole, exact]), - [None, Some(1_609_459_200_876)] + [None, Some((1_609_459_200_876, MS))] ); assert_eq!( one_copy_per_message(&[exact, whole]), - [Some(1_609_459_200_876), None] + [Some((1_609_459_200_876, MS)), None] ); } @@ -402,11 +435,11 @@ mod tests { let xml = copy("+15555550122", 1_609_459_200_250, MS, &[]); assert_eq!( one_copy_per_message(&[xml, pdu]), - [None, Some(1_609_459_200_250)] + [None, Some((1_609_459_200_250, MS))] ); assert_eq!( one_copy_per_message(&[pdu, xml]), - [Some(1_609_459_200_250), None] + [Some((1_609_459_200_250, MS)), None] ); } @@ -420,7 +453,10 @@ mod tests { ]; assert_eq!( one_copy_per_message(&copies), - [Some(1_609_459_200_000), Some(1_609_459_200_000)] + [ + Some((1_609_459_200_000, SECS)), + Some((1_609_459_200_000, SECS)) + ] ); } @@ -432,7 +468,7 @@ mod tests { b.vendor_key = Some("B2"); assert_eq!( one_copy_per_message(&[a, b]), - [Some(1_609_459_200_000), Some(1_609_459_200_000)] + [Some((1_609_459_200_000, MS)), Some((1_609_459_200_000, MS))] ); } @@ -444,7 +480,10 @@ mod tests { ]; assert_eq!( one_copy_per_message(&copies), - [Some(1_609_459_200_000), Some(1_609_459_201_000)] + [ + Some((1_609_459_200_000, SECS)), + Some((1_609_459_201_000, SECS)) + ] ); } } diff --git a/crates/libs/ir/src/lib.rs b/crates/libs/ir/src/lib.rs index ba8709bf6..9dd139cef 100644 --- a/crates/libs/ir/src/lib.rs +++ b/crates/libs/ir/src/lib.rs @@ -138,7 +138,7 @@ impl std::fmt::Display for UnknownDeletion { impl std::error::Error for UnknownDeletion {} /// Schema version written into every [`ConversationDocument`]. -pub const SCHEMA_VERSION: u32 = 11; +pub const SCHEMA_VERSION: u32 = 12; /// One exported chat: export metadata, conversation roster and stats, and messages. /// @@ -473,6 +473,12 @@ pub struct IrMessage { pub guid: String, /// Unix milliseconds; the chronological sort key. pub timestamp_unix_ms: i64, + /// Whether the source recorded [`Self::timestamp_unix_ms`] to the + /// millisecond or in whole seconds. Required: the flag, never the + /// value, says whether a time has milliseconds, since a millisecond + /// time can end in `.000`. The import shows a whole-second message once + /// when its source also holds it with milliseconds in the same second. + pub time_precision: TimePrecision, /// Incoming or outgoing. pub direction: IrDirection, /// Transport the message arrived on. diff --git a/crates/libs/ir/src/projection.rs b/crates/libs/ir/src/projection.rs index aee12212e..36f8cdc09 100644 --- a/crates/libs/ir/src/projection.rs +++ b/crates/libs/ir/src/projection.rs @@ -274,7 +274,7 @@ pub fn pending_to_document( let mut replies: Vec<(usize, PendingReply)> = Vec::new(); let mut guid_by_reply_key: HashMap> = HashMap::new(); for ((msg, p), kept) in convo.messages.iter().zip(&prepared).zip(kept) { - let Some(timestamp_unix_ms) = kept else { + let Some((timestamp_unix_ms, time_precision)) = kept else { tally.duplicates += 1; continue; }; @@ -322,6 +322,7 @@ pub fn pending_to_document( messages.push(IrMessage { guid, timestamp_unix_ms, + time_precision, direction: if outgoing { IrDirection::Outgoing } else { diff --git a/crates/libs/ir/src/schema_version.rs b/crates/libs/ir/src/schema_version.rs index 3b99d3f3a..d7905fea9 100644 --- a/crates/libs/ir/src/schema_version.rs +++ b/crates/libs/ir/src/schema_version.rs @@ -3,12 +3,12 @@ //! Every reader of a [`ConversationDocument`](crate::ConversationDocument) or //! its JSON Lines header — the format reader, the push client, the server's //! import — refuses a version other than [`SCHEMA_VERSION`] with the same -//! words, and refuses it before parsing the rest of the file: a version-10 -//! file is not expected to match version 11 (version 10 did not say when its -//! backup was made, so an import could not tell which of two backups of one -//! phone is the later one, and a file read as version 11 would claim it has -//! no date when it only never asked), and the person should read "schema -//! version 10", not a file that imports by other rules than its own. +//! words, and refuses it before parsing the rest of the file: a version-11 +//! file is not expected to match version 12 (version 11 did not say whether +//! a message's time has milliseconds, so a time ending in `.000` could be +//! either, and the import could not tell a whole-second copy of a message +//! from a millisecond one), and the person should read "schema version 11", +//! not a file that imports by other rules than its own. use crate::SCHEMA_VERSION; use serde::Deserialize; @@ -107,7 +107,22 @@ mod tests { .to_string(), format!("This file is schema version 10; Message Crate reads version {SCHEMA_VERSION}") ); - assert_eq!(SCHEMA_VERSION, 11); + } + + /// Version 11 had no `time_precision` on a message; version 12 says + /// whether each time has milliseconds, and the import shows a + /// whole-second message once when its source also holds it with + /// milliseconds. A version-11 file is refused by its version, never read + /// with its messages missing the field. + #[test] + fn refuses_a_version_11_file_by_name() { + assert_eq!( + check_schema_version_in_json(r#"{"schema_version":11,"export":{}}"#) + .unwrap_err() + .to_string(), + format!("This file is schema version 11; Message Crate reads version {SCHEMA_VERSION}") + ); + assert_eq!(SCHEMA_VERSION, 12); } #[test] diff --git a/crates/libs/ir/src/testutil.rs b/crates/libs/ir/src/testutil.rs index f394d34a9..5f7921f11 100644 --- a/crates/libs/ir/src/testutil.rs +++ b/crates/libs/ir/src/testutil.rs @@ -35,6 +35,7 @@ pub fn sample_document(text: &str) -> ConversationDocument { messages: vec![IrMessage { guid: "aabbccddeeff00112233445566778899".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: crate::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Sms, @@ -113,6 +114,7 @@ pub fn sample_imessage_document() -> ConversationDocument { IrMessage { guid: "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: crate::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::IMessage, message_kind: IrMessageKind::IMessage, @@ -157,6 +159,7 @@ pub fn sample_imessage_document() -> ConversationDocument { IrMessage { guid: "TAPBACK-GUID-0001".into(), timestamp_unix_ms: 1_400_773_262_000, + time_precision: crate::TimePrecision::Milliseconds, direction: IrDirection::Outgoing, service: IrService::IMessage, message_kind: IrMessageKind::Tapback, diff --git a/crates/libs/mail/src/headers.rs b/crates/libs/mail/src/headers.rs index 50702809b..478a66016 100644 --- a/crates/libs/mail/src/headers.rs +++ b/crates/libs/mail/src/headers.rs @@ -15,6 +15,9 @@ pub(crate) const SERVICE: &str = "X-ME-Service"; pub(crate) const MESSAGE_KIND: &str = "X-ME-Message-Kind"; /// Message timestamp in Unix milliseconds. pub(crate) const TIMESTAMP_UNIX_MS: &str = "X-ME-Timestamp-Unix-Ms"; +/// Whether the timestamp has milliseconds: `seconds` or `milliseconds`, as +/// `message_ir::TimePrecision` names them. +pub(crate) const TIME_PRECISION: &str = "X-ME-Time-Precision"; /// Message guid. pub(crate) const GUID: &str = "X-ME-Guid"; /// Export source id. diff --git a/crates/libs/mail/src/lib.rs b/crates/libs/mail/src/lib.rs index 7590e3a7b..5438e3aab 100644 --- a/crates/libs/mail/src/lib.rs +++ b/crates/libs/mail/src/lib.rs @@ -885,6 +885,7 @@ fn conversation_headers<'m>( (headers::SERVICE, msg.message.service.as_str()), (headers::MESSAGE_KIND, msg.message.message_kind.as_str()), (headers::TIMESTAMP_UNIX_MS, ×tamp), + (headers::TIME_PRECISION, msg.message.time_precision.as_str()), (headers::GUID, msg.message.guid.as_str()), (headers::EXPORT_SOURCE, msg.export_source.as_str()), (headers::EXPORT_TOOL, msg.export_tool.as_str()), diff --git a/crates/libs/mail/src/parse.rs b/crates/libs/mail/src/parse.rs index 17854ba1c..8317f8bbe 100644 --- a/crates/libs/mail/src/parse.rs +++ b/crates/libs/mail/src/parse.rs @@ -6,7 +6,7 @@ use anyhow::{Context, Result, bail}; use mailparse::{MailHeader, MailHeaderMap, ParsedMail}; use message_ir::{ Deletion, EarlierVersion, IrDirection, IrImessage, IrMessage, IrMessageKind, IrService, - IrSource, Reaction, ReplyTo, + IrSource, Reaction, ReplyTo, TimePrecision, }; use serde::Deserialize; use serde::de::DeserializeOwned; @@ -87,6 +87,7 @@ pub fn mail_message_from_eml_bytes(bytes: &[u8]) -> Result { .with_context(|| format!("missing required header {}", hn::TIMESTAMP_UNIX_MS))? .parse::() .context("parse X-ME-Timestamp-Unix-Ms")?; + let time_precision = parse_time_precision(headers)?; let direction = match typed_header(headers, hn::DIRECTION) .unwrap_or_default() .to_ascii_lowercase() @@ -166,6 +167,7 @@ pub fn mail_message_from_eml_bytes(bytes: &[u8]) -> Result { message: IrMessage { guid, timestamp_unix_ms, + time_precision, direction, service, message_kind, @@ -337,6 +339,21 @@ fn parse_backup_taken_at(headers: &[MailHeader<'_>]) -> Result> { .with_context(|| format!("This mail's {} header {raw:?}", hn::BACKUP_TAKEN_AT_UNIX_MS)) } +/// Whether the message's time has milliseconds, from the required +/// `X-ME-Time-Precision`. A missing header or a value other than `seconds` +/// or `milliseconds` is refused: the flag, never the time, says whether a +/// time ending in `.000` has milliseconds. +fn parse_time_precision(headers: &[MailHeader<'_>]) -> Result { + let raw = typed_header(headers, hn::TIME_PRECISION) + .with_context(|| format!("missing required header {}", hn::TIME_PRECISION))?; + TimePrecision::parse(&raw).with_context(|| { + format!( + "This mail's {} header {raw:?} is neither seconds nor milliseconds", + hn::TIME_PRECISION + ) + }) +} + /// The message's mark from `X-ME-Deletion`, or none when the header is /// absent. A value that names neither mark is refused rather than dropped. fn parse_deletion(headers: &[MailHeader<'_>]) -> Result> { @@ -488,6 +505,7 @@ mod tests { message: IrMessage { guid: "aabbccddeeff00112233445566778899".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Outgoing, service: IrService::Sms, message_kind: IrMessageKind::Sms, @@ -606,6 +624,7 @@ mod tests { message: IrMessage { guid: "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::IMessage, message_kind: IrMessageKind::IMessage, @@ -734,6 +753,7 @@ mod tests { message: IrMessage { guid: "11111111-2222-3333-4444-555555555555".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::IMessage, message_kind: IrMessageKind::IMessage, diff --git a/crates/libs/mail/src/tests.rs b/crates/libs/mail/src/tests.rs index f21893cc9..1624b3e92 100644 --- a/crates/libs/mail/src/tests.rs +++ b/crates/libs/mail/src/tests.rs @@ -20,6 +20,7 @@ fn base_sms() -> MailMessage { message: IrMessage { guid: "aabbccddeeff00112233445566778899".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: message_ir::IrService::Sms, message_kind: message_ir::IrMessageKind::Sms, @@ -489,7 +490,7 @@ fn writes_conversation_mboxrd() { a.message.timestamp_unix_ms = 1_400_773_261_000; let mut b = base_sms(); b.message.guid = "bbccddeeff00112233445566778899aa".into(); - b.message.text = "second".into(); + b.message.text = "the next one".into(); b.message.timestamp_unix_ms = 1_400_773_361_000; let tmp = tempfile::tempdir().unwrap(); @@ -501,7 +502,7 @@ fn writes_conversation_mboxrd() { assert!(text.contains(">From spoofed")); // Chronological: first then second let first_pos = text.find("first").unwrap(); - let second_pos = text.find("second").unwrap(); + let second_pos = text.find("the next one").unwrap(); assert!(first_pos < second_pos); assert_eq!(text.matches("\nFrom ").count(), 1); // one additional From_ between records assert!(text.contains("X-ME-Guid: aabbccddeeff00112233445566778899")); @@ -511,7 +512,7 @@ fn writes_conversation_mboxrd() { let parsed = mail_messages_from_mbox(&path).unwrap(); assert_eq!(parsed.len(), 2); assert_eq!(parsed[0].message.text, "From spoofed\nfirst\nlast"); - assert_eq!(parsed[1].message.text, "second"); + assert_eq!(parsed[1].message.text, "the next one"); } #[test] @@ -565,6 +566,7 @@ fn a_mail_that_names_addresses_handles_is_refused() { "X-ME-Chat-Identifier: sam@example.com\r\n", "X-ME-Guid: g1\r\n", "X-ME-Timestamp-Unix-Ms: 1400773261000\r\n", + "X-ME-Time-Precision: milliseconds\r\n", "X-ME-Participants: [{\"handle\":\"sam@example.com\"}]\r\n", "X-ME-Sender-Handle: sam@example.com\r\n", "\r\n", @@ -614,6 +616,7 @@ fn a_mail_that_keeps_reactions_in_x_me_tapbacks_is_refused() { "X-ME-Chat-Identifier: +15555550101\r\n", "X-ME-Guid: g1\r\n", "X-ME-Timestamp-Unix-Ms: 1400773261000\r\n", + "X-ME-Time-Precision: milliseconds\r\n", "X-ME-Tapbacks: [{\"part_index\":0,\"kind\":\"loved\",\"is_from_me\":false,\"reactor_identity\":\"+15555550101\"}]\r\n", "\r\n", "hello\r\n", @@ -635,6 +638,7 @@ fn a_mail_that_keeps_a_reply_link_in_x_me_thread_originator_guid_is_refused() { "X-ME-Chat-Identifier: +15555550101\r\n", "X-ME-Guid: g1\r\n", "X-ME-Timestamp-Unix-Ms: 1400773261000\r\n", + "X-ME-Time-Precision: milliseconds\r\n", "X-ME-Is-Reply: true\r\n", "X-ME-Thread-Originator-Guid: parent-guid\r\n", "\r\n", @@ -713,6 +717,7 @@ fn a_mail_that_keeps_the_deleted_mark_in_x_me_is_deleted_is_refused() { "X-ME-Chat-Identifier: +15555550101\r\n", "X-ME-Guid: g1\r\n", "X-ME-Timestamp-Unix-Ms: 1400773261000\r\n", + "X-ME-Time-Precision: milliseconds\r\n", "X-ME-Is-Deleted: true\r\n", "\r\n", "hello\r\n", @@ -734,6 +739,7 @@ fn a_mail_that_keeps_the_edit_history_in_x_me_edits_is_refused() { "X-ME-Chat-Identifier: +15555550101\r\n", "X-ME-Guid: g1\r\n", "X-ME-Timestamp-Unix-Ms: 1400773261000\r\n", + "X-ME-Time-Precision: milliseconds\r\n", "X-ME-Edits: [{\"part_index\":0,\"status\":\"edited\",\"text\":\"helo\"}]\r\n", "\r\n", "hello\r\n", @@ -767,6 +773,7 @@ fn assert_json_header_is_refused(header: &str, value: &str, what: &str) { "X-ME-Chat-Identifier: +15555550101\r\n\ X-ME-Guid: g1\r\n\ X-ME-Timestamp-Unix-Ms: 1400773261000\r\n\ + X-ME-Time-Precision: milliseconds\r\n\ X-ME-Service: imessage\r\n\ {header}: {value}\r\n\ \r\n\ @@ -817,6 +824,7 @@ fn a_mail_whose_deletion_names_no_mark_is_refused() { "X-ME-Chat-Identifier: +15555550101\r\n", "X-ME-Guid: g1\r\n", "X-ME-Timestamp-Unix-Ms: 1400773261000\r\n", + "X-ME-Time-Precision: milliseconds\r\n", "X-ME-Deletion: trashed\r\n", "\r\n", "hello\r\n", @@ -1019,6 +1027,7 @@ fn a_typed_header_that_ends_in_a_space_reads_as_its_value() { "X-ME-Conversation-Type: group \r\n", "X-ME-Guid: g1\r\n", "X-ME-Timestamp-Unix-Ms: 1400773261000 \r\n", + "X-ME-Time-Precision: seconds \r\n", "X-ME-Direction: outgoing \r\n", "X-ME-Service: imessage \r\n", "X-ME-Message-Kind: imessage \r\n", @@ -1036,6 +1045,10 @@ fn a_typed_header_that_ends_in_a_space_reads_as_its_value() { let msg = crate::mail_message_from_eml_bytes(eml.as_bytes()).unwrap(); assert_eq!(msg.conversation_type, "group"); assert_eq!(msg.message.timestamp_unix_ms, 1_400_773_261_000); + assert_eq!( + msg.message.time_precision, + message_ir::TimePrecision::Seconds + ); assert_eq!(msg.message.direction, IrDirection::Outgoing); assert_eq!(msg.message.service, message_ir::IrService::IMessage); assert_eq!( diff --git a/crates/libs/push/src/project.rs b/crates/libs/push/src/project.rs index d7e493e2a..cfbd2be7a 100644 --- a/crates/libs/push/src/project.rs +++ b/crates/libs/push/src/project.rs @@ -155,13 +155,14 @@ mod tests { packaging_stem_suffix: None, }; let header = String::from_utf8(document_header_line(&doc).unwrap()).unwrap(); - assert!(header.contains(r#""schema_version":11"#)); + assert!(header.contains(r#""schema_version":12"#)); assert!(header.contains(r#""sms-backup-restore""#)); assert!(!header.contains(r#""record":"conversation""#)); let msg = IrMessage { guid: "g1".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Sms, @@ -190,6 +191,7 @@ mod tests { let msg = IrMessage { guid: "g1".into(), timestamp_unix_ms: 1, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Sms, diff --git a/crates/libs/push/tests/push_mock.rs b/crates/libs/push/tests/push_mock.rs index 93d26f719..858636e12 100644 --- a/crates/libs/push/tests/push_mock.rs +++ b/crates/libs/push/tests/push_mock.rs @@ -50,6 +50,7 @@ fn sample_doc() -> ConversationDocument { messages: vec![IrMessage { guid: "guid-1".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Sms, diff --git a/crates/server/demo-seed/src/conversations.rs b/crates/server/demo-seed/src/conversations.rs index c5d810dbb..f9ab94971 100644 --- a/crates/server/demo-seed/src/conversations.rs +++ b/crates/server/demo-seed/src/conversations.rs @@ -13,7 +13,7 @@ use chrono::{Duration, Utc}; use message_ir::{ ConversationHeader, ConversationMeta, ConversationStats, Deletion, EarlierVersion, ExportMeta, IrAttachment, IrConversationType, IrDirection, IrImessage, IrMessage, IrMessageKind, - IrParticipant, IrService, Reaction, ReplyTo, SCHEMA_VERSION, orphaned_chat_id, + IrParticipant, IrService, Reaction, ReplyTo, SCHEMA_VERSION, TimePrecision, orphaned_chat_id, }; use rand::Rng; use rand::RngExt; @@ -317,6 +317,8 @@ impl SharedMessage { IrMessage { guid, timestamp_unix_ms: self.timestamp, + // Every source the demo imitates records milliseconds. + time_precision: TimePrecision::Milliseconds, direction: if self.from_me { IrDirection::Outgoing } else { @@ -1127,6 +1129,8 @@ impl Seeder<'_, R> { IrMessage { guid: guid.into(), timestamp_unix_ms, + // Every source the demo imitates records milliseconds. + time_precision: TimePrecision::Milliseconds, direction: if from_me { IrDirection::Outgoing } else { diff --git a/crates/server/server/src/cli.rs b/crates/server/server/src/cli.rs index 7d98db14e..e991907fc 100644 --- a/crates/server/server/src/cli.rs +++ b/crates/server/server/src/cli.rs @@ -39,7 +39,8 @@ pub enum Commands { /// Work on an account's Import Runs (`discard` clears a stranded one) Imports(ImportsArgs), - /// Soft-hide the same SMS when it appears under more than one import source + /// Soft-hide the same SMS when it appears under more than one import source, + /// or once in whole seconds beside its millisecond copy in one source DedupeCrossSource(DedupeArgs), /// Rebuild the Demo Account: generate Demo Data, clear the account, diff --git a/crates/server/server/src/db/conversation_messages.rs b/crates/server/server/src/db/conversation_messages.rs index 3ce6af2eb..9ebae9bb0 100644 --- a/crates/server/server/src/db/conversation_messages.rs +++ b/crates/server/server/src/db/conversation_messages.rs @@ -19,6 +19,7 @@ use sqlx::{Executor, Row}; pub use message_crate_api_types::{ Attachment, Deletion, EarlierVersion, Message, MessageConversation, ReplyTo, Tapback, + TimePrecision, }; use crate::db::conversations::is_group_type; @@ -55,6 +56,7 @@ struct RawRow { reply_count: i64, deletion: Option, backup_taken_at: Option, + time_precision: TimePrecision, chat_identifier: String, conversation_type: String, group_title: Option, @@ -475,7 +477,8 @@ fn message_page_sql( m.is_announcement, m.is_reply, m.reply_to_guid, m.reply_to_part, ({reply_count}) AS reply_count, hc.raw AS chat_identifier, c.conversation_type, c.group_title, - ho.raw AS owner, {label} AS label, m.deletion, m.backup_taken_at + ho.raw AS owner, {label} AS label, m.deletion, m.backup_taken_at, + m.time_precision {from_sql} WHERE {where_sql} ORDER BY {order_by} LIMIT ? OFFSET ?", @@ -529,6 +532,12 @@ async fn fetch_message_page( label: row.try_get(20)?, deletion: row.try_get(21)?, backup_taken_at: row.try_get(22)?, + time_precision: { + let stored: String = row.try_get(23)?; + TimePrecision::parse(&stored).ok_or_else(|| { + sqlx::Error::Decode(format!("time_precision {stored:?}").into()) + })? + }, }) }) .collect::, ApiError>>()?; @@ -553,6 +562,7 @@ async fn fetch_message_page( service: r.service, guid: r.guid, timestamp: r.timestamp, + time_precision: r.time_precision, sort_order: r.sort_order, is_from_me: r.is_from_me, sender: r.sender, diff --git a/crates/server/server/src/db/staging.rs b/crates/server/server/src/db/staging.rs index 424a6f9e9..c2229174a 100644 --- a/crates/server/server/src/db/staging.rs +++ b/crates/server/server/src/db/staging.rs @@ -203,6 +203,8 @@ pub struct StagingMessage<'a> { pub guid: &'a str, /// RFC 3339 UTC instant the message was sent. pub timestamp: &'a str, + /// Whether the source recorded `timestamp` to the millisecond. + pub time_precision: message_ir::TimePrecision, /// 1 when the account holder sent it. pub is_from_me: i64, /// Sender's handle id; `None` when unknown. @@ -344,7 +346,7 @@ const TAPBACK_COLUMNS: &[&str] = &[ ]; /// Bind counts, in lockstep with the `INSERT` column lists below. -const MESSAGE_BIND_COLUMNS: usize = 19; +const MESSAGE_BIND_COLUMNS: usize = 20; const ATTACHMENT_BIND_COLUMNS: usize = ATTACHMENT_COLUMNS.len(); const TAPBACK_BIND_COLUMNS: usize = TAPBACK_COLUMNS.len(); const EARLIER_VERSION_BIND_COLUMNS: usize = 4; @@ -369,7 +371,7 @@ pub async fn insert_messages( let sql = format!( r" INSERT INTO staging_messages ( - conversation_id, account_id, source, guid, timestamp, is_from_me, + conversation_id, account_id, source, guid, timestamp, time_precision, is_from_me, sender_handle_id, owner_handle_id, service, subject, body, is_announcement, is_reply, reply_to_guid, reply_to_part, deletion, sort_order, import_id, backup_taken_at ) VALUES {} @@ -386,6 +388,7 @@ pub async fn insert_messages( .bind(row.source) .bind(row.guid) .bind(row.timestamp) + .bind(row.time_precision.as_str()) .bind(row.is_from_me) .bind(row.sender_handle_id) .bind(row.owner_handle_id) @@ -1070,12 +1073,13 @@ pub async fn staged_message_id_bounds( /// the production ids follow it, which the id-map zip relies on. const INSERT_MESSAGES_FROM_STAGING: &str = r" INSERT INTO messages ( - conversation_id, account_id, source, guid, timestamp, is_from_me, + conversation_id, account_id, source, guid, timestamp, time_precision, is_from_me, sender_handle_id, owner_handle_id, service, subject, body, is_announcement, is_reply, reply_to_guid, reply_to_part, deletion, sort_order, import_id, backup_taken_at ) SELECT - cm.prod_id, sm.account_id, sm.source, sm.guid, sm.timestamp, sm.is_from_me, + cm.prod_id, sm.account_id, sm.source, sm.guid, sm.timestamp, sm.time_precision, + sm.is_from_me, sm.sender_handle_id, sm.owner_handle_id, sm.service, sm.subject, sm.body, sm.is_announcement, sm.is_reply, sm.reply_to_guid, sm.reply_to_part, sm.deletion, sm.sort_order, sm.import_id, sm.backup_taken_at @@ -1355,6 +1359,58 @@ pub async fn promote_backup_dates(conn: &mut SqliteConnection) -> Result { Ok(sqlx::query(&sql).execute(&mut *conn).await?.rows_affected()) } +/// Mark each stored message `milliseconds` when its staged row is and has +/// the same time: a whole-second copy and a millisecond copy whose time +/// ends in `.000` have one guid, so they are one message, and the source +/// did record its time to the millisecond, whichever copy was stored first +/// (#1923). A staged `seconds` row, or one at another time (a copy cut to +/// the second elsewhere that kept the guid), leaves the stored flag as it +/// is. +/// +/// # Errors +/// +/// Returns an error when the update fails. +pub async fn promote_time_precision(conn: &mut SqliteConnection) -> Result<()> { + sqlx::query( + r" + UPDATE messages + SET time_precision = sm.time_precision + FROM _promote_msg_map mm + JOIN staging_messages sm ON sm.id = mm.staging_id + WHERE messages.id = mm.prod_id + AND sm.time_precision = $1 + AND messages.time_precision != $1 + AND sm.timestamp = messages.timestamp + ", + ) + .bind(message_ir::TimePrecision::Milliseconds.as_str()) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// Mark the staged message `staged` `milliseconds` when it is at +/// `timestamp`, for another copy of it from the same import that is +/// `milliseconds` at that time: the staged-row form of +/// [`promote_time_precision`]. +/// +/// # Errors +/// +/// Returns an error when the update fails. +pub async fn add_staged_copy_milliseconds( + conn: &mut SqliteConnection, + staged: i64, + timestamp: &str, +) -> Result<()> { + sqlx::query("UPDATE staging_messages SET time_precision = $1 WHERE id = $2 AND timestamp = $3") + .bind(message_ir::TimePrecision::Milliseconds.as_str()) + .bind(staged) + .bind(timestamp) + .execute(&mut *conn) + .await?; + Ok(()) +} + /// Whether one copy of a message records a later edit than another, as an /// SQL expression over four SQL values: the copy's earlier-version count /// `n` and newest `edited_at` `newest`, and the other copy's `held_n` and diff --git a/crates/server/server/src/db/staging/tests.rs b/crates/server/server/src/db/staging/tests.rs index 90522030c..7e8a54ed2 100644 --- a/crates/server/server/src/db/staging/tests.rs +++ b/crates/server/server/src/db/staging/tests.rs @@ -31,6 +31,7 @@ async fn reset_for_account_leaves_other_accounts() { source: "sms", guid: "g1", timestamp: "2020-01-01T00:00:00.000Z", + time_precision: message_ir::TimePrecision::Milliseconds, is_from_me: 0, sender_handle_id: None, owner_handle_id: None, diff --git a/crates/server/server/src/db/trash/tests.rs b/crates/server/server/src/db/trash/tests.rs index 99227f4ce..da87e8970 100644 --- a/crates/server/server/src/db/trash/tests.rs +++ b/crates/server/server/src/db/trash/tests.rs @@ -607,8 +607,11 @@ async fn delete_reports_only_the_files_no_remaining_message_uses() { .unwrap(); let staging_message: i64 = sqlx::query_scalar( "INSERT INTO staging_messages ( - conversation_id, account_id, source, guid, timestamp, is_from_me, sort_order - ) VALUES ($1, $2, 'imessage', 'g-staged', '2020-01-01T00:00:00Z', 1, 0) RETURNING id", + conversation_id, account_id, source, guid, timestamp, time_precision, is_from_me, + sort_order + ) VALUES ( + $1, $2, 'imessage', 'g-staged', '2020-01-01T00:00:00.000Z', 'milliseconds', 1, 0 + ) RETURNING id", ) .bind(staging_conversation) .bind(ACCOUNT_A) diff --git a/crates/server/server/src/dedupe.rs b/crates/server/server/src/dedupe.rs index 3a64ba25b..87d499234 100644 --- a/crates/server/server/src/dedupe.rs +++ b/crates/server/server/src/dedupe.rs @@ -142,10 +142,12 @@ pub struct DedupeStats { /// Content keys written: those missing and those whose inputs changed /// (not a duplicate count). pub keys_filled: u64, - /// Groups of messages sharing one content key. + /// Groups of messages sharing one content key in which a message was + /// hidden. pub exact_groups: u64, /// Messages hidden as exact duplicates: all but as many per group as one - /// source holds. + /// source holds, and each whole-second message whose own source holds + /// it with milliseconds too. pub exact_flagged: u64, /// Messages flagged as near duplicates. pub near_flagged: u64, @@ -175,7 +177,8 @@ pub async fn source_priority_from_db( } /// Refresh the content keys, clear prior flags, then soft-hide cross-source -/// duplicates, all in one transaction. +/// duplicates and the whole-second twins of a message one source holds with +/// milliseconds too, all in one transaction. /// /// Survivor preference: most attachments, then the source imported first (min /// message id, then source name), then the lowest message id. Optional @@ -536,9 +539,17 @@ struct Cand { att_count: i64, } +/// One message of a content-key group in the exact pass: the candidate, +/// and whether its source recorded its time in whole seconds. +struct KeyedCand { + cand: Cand, + whole_seconds: bool, +} + /// Hide the messages that share a fingerprint with a preferred-source twin, -/// keeping as many as one source holds (see [`exact_group_flags`]). Returns -/// (groups, hidden). +/// keeping as many as one source holds (see [`exact_group_flags`]), and +/// each whole-second message whose own source holds it with milliseconds +/// too (see [`content_key_group_flags`]). Returns (groups, hidden). async fn flag_exact_content_key_dupes( tx: &mut WriteTx<'_>, account_id: i64, @@ -547,9 +558,9 @@ async fn flag_exact_content_key_dupes( let conn: &mut SqliteConnection = tx; // One scan of messages + one aggregated attachment pass, then group in Rust. // Avoids N round-trips (one SELECT + several UPDATEs per duplicate key). - let rows: Vec<(i64, String, String, i64)> = sqlx::query_as( + let rows: Vec<(i64, String, String, i64, String)> = sqlx::query_as( r" - SELECT m.id, m.source, m.content_key, COALESCE(ac.n, 0) + SELECT m.id, m.source, m.content_key, COALESCE(ac.n, 0), m.time_precision FROM messages m JOIN conversations c ON c.id = m.conversation_id LEFT JOIN ( @@ -569,24 +580,30 @@ async fn flag_exact_content_key_dupes( .fetch_all(&mut *conn) .await?; - let mut by_key: HashMap> = HashMap::new(); - for (id, source, content_key, att_count) in rows { - by_key.entry(content_key).or_default().push(Cand { - id, - source, - att_count, + let mut by_key: HashMap> = HashMap::new(); + for (id, source, content_key, att_count, time_precision) in rows { + by_key.entry(content_key).or_default().push(KeyedCand { + cand: Cand { + id, + source, + att_count, + }, + // The flag decides, never the time: a millisecond time can end + // in `.000`. + whole_seconds: message_ir::TimePrecision::parse(&time_precision) + .with_context(|| format!("message {id}: time_precision {time_precision:?}"))? + == message_ir::TimePrecision::Seconds, }); } let mut flags: Vec<(i64, i64)> = Vec::new(); // (loser_id, winner_id) let mut groups = 0u64; - for cands in by_key.values() { - let sources: HashSet<&str> = cands.iter().map(|c| c.source.as_str()).collect(); - if sources.len() < 2 { - continue; + for cands in by_key.into_values() { + let group_flags = content_key_group_flags(cands, prio); + if !group_flags.is_empty() { + groups += 1; } - groups += 1; - flags.extend(exact_group_flags(cands, prio)); + flags.extend(group_flags); } let flagged = flags.len() as u64; @@ -599,6 +616,43 @@ async fn flag_exact_content_key_dupes( Ok((groups, flagged)) } +/// The `(loser, winner)` pairs of the messages that share one content key. +/// +/// A whole-second message whose own source also holds a message of the same +/// key with milliseconds is first set aside as that message's twin: the key +/// is taken at whole seconds, so the two match in everything else and fall +/// in the same second, and the source recorded the message twice, once +/// without its milliseconds (an SMS Backup+ mail timed by its `Date` header +/// beside one timed by `X-smssync-date`). The rest are flagged by +/// [`exact_group_flags`] when two or more sources hold them, and each twin +/// is hidden under the rest's winner, which is always shown, so the message +/// is shown once. It keeps its milliseconds unless another source's copy +/// wins the cross-source comparison, as one whole-second source imported +/// first does. A source that holds the message only in whole seconds keeps +/// every copy, as one that holds it only with milliseconds does. +fn content_key_group_flags(cands: Vec, prio: &HashMap<&str, usize>) -> Vec<(i64, i64)> { + let with_milliseconds: HashSet = cands + .iter() + .filter(|c| !c.whole_seconds) + .map(|c| c.cand.source.clone()) + .collect(); + let (twins, rest): (Vec, Vec) = cands + .into_iter() + .partition(|c| c.whole_seconds && with_milliseconds.contains(c.cand.source.as_str())); + let rest: Vec = rest.into_iter().map(|c| c.cand).collect(); + let sources: HashSet<&str> = rest.iter().map(|c| c.source.as_str()).collect(); + let mut flags = if sources.len() < 2 { + Vec::new() + } else { + exact_group_flags(&rest, prio) + }; + if !twins.is_empty() { + let winner = pick_winner(&rest, prio); + flags.extend(twins.into_iter().map(|t| (t.cand.id, winner))); + } + flags +} + /// The `(loser, winner)` pairs of one group of copies that two or more /// sources hold: the messages of one content key in the exact pass, or one /// cluster of the near-time pass ([`cluster_near_dupes`]). diff --git a/crates/server/server/src/dedupe/tests.rs b/crates/server/server/src/dedupe/tests.rs index 7fe715a6a..84ccd9965 100644 --- a/crates/server/server/src/dedupe/tests.rs +++ b/crates/server/server/src/dedupe/tests.rs @@ -675,6 +675,135 @@ async fn identical_rows_from_one_source_are_both_kept() { } } +/// Insert one received SMS Backup+ message "On my way" at `timestamp`, +/// recorded with `precision`, and answer its id. +async fn sms_backup_plus_row( + conn: &mut SqliteConnection, + guid: &str, + timestamp: &'static str, + precision: message_ir::TimePrecision, +) -> i64 { + MessageRow { + source: "sms-backup-plus", + guid: Some(guid.into()), + timestamp, + time_precision: precision, + is_from_me: false, + body: Some("On my way"), + sort_order: 0, + ..MessageRow::new(TEST_ACCOUNT_ID, 1) + } + .insert(conn) + .await +} + +/// One source that holds a message once in whole seconds and once with +/// milliseconds in the same second shows it once, with the milliseconds: +/// the whole-second copy is the duplicate (#1923). +#[tokio::test] +async fn a_whole_second_message_is_the_duplicate_of_its_millisecond_twin_in_one_source() { + let (pool, _dir) = engine::test_pool().await; + let mut conn = pool.acquire().await.unwrap(); + setup_db(&mut conn).await; + let whole = sms_backup_plus_row( + &mut conn, + "g-whole", + "2015-03-12T18:04:22.000Z", + message_ir::TimePrecision::Seconds, + ) + .await; + let exact = sms_backup_plus_row( + &mut conn, + "g-exact", + "2015-03-12T18:04:22.250Z", + message_ir::TimePrecision::Milliseconds, + ) + .await; + + let stats = dedupe_cross_source(&mut conn, TEST_ACCOUNT_ID, None, 2) + .await + .unwrap(); + + assert_eq!((stats.exact_groups, stats.exact_flagged), (1, 1)); + assert_eq!(duplicate_of(&mut conn, whole).await, Some(exact)); + assert_eq!(duplicate_of(&mut conn, exact).await, None); +} + +/// A time whose source recorded milliseconds is never taken for a whole +/// second because it ends in `.000`: the flag decides, never the time. One +/// source holds the message at `.000` with milliseconds, at `.250` with +/// milliseconds, and once in whole seconds. The two millisecond copies are +/// two messages and both stay shown; only the whole-second copy is hidden. +#[tokio::test] +async fn a_millisecond_time_ending_in_000_is_not_whole_seconds() { + let (pool, _dir) = engine::test_pool().await; + let mut conn = pool.acquire().await.unwrap(); + setup_db(&mut conn).await; + let on_the_second = sms_backup_plus_row( + &mut conn, + "g-000", + "2015-03-12T18:04:22.000Z", + message_ir::TimePrecision::Milliseconds, + ) + .await; + let later = sms_backup_plus_row( + &mut conn, + "g-250", + "2015-03-12T18:04:22.250Z", + message_ir::TimePrecision::Milliseconds, + ) + .await; + let whole = sms_backup_plus_row( + &mut conn, + "g-whole", + "2015-03-12T18:04:22.000Z", + message_ir::TimePrecision::Seconds, + ) + .await; + + dedupe_cross_source(&mut conn, TEST_ACCOUNT_ID, None, 2) + .await + .unwrap(); + + assert_eq!(duplicate_of(&mut conn, on_the_second).await, None); + assert_eq!(duplicate_of(&mut conn, later).await, None); + assert!( + [Some(on_the_second), Some(later)].contains(&duplicate_of(&mut conn, whole).await), + "the whole-second copy is hidden under a millisecond copy" + ); +} + +/// A source that holds a message only in whole seconds keeps every copy, +/// as it did before: two whole-second messages in one second are two +/// messages. +#[tokio::test] +async fn two_whole_second_messages_from_one_source_are_both_kept() { + let (pool, _dir) = engine::test_pool().await; + let mut conn = pool.acquire().await.unwrap(); + setup_db(&mut conn).await; + let mut ids = Vec::new(); + for guid in ["g1", "g2"] { + ids.push( + sms_backup_plus_row( + &mut conn, + guid, + "2015-03-12T18:04:22.000Z", + message_ir::TimePrecision::Seconds, + ) + .await, + ); + } + + let stats = dedupe_cross_source(&mut conn, TEST_ACCOUNT_ID, None, 2) + .await + .unwrap(); + + assert_eq!((stats.exact_groups, stats.exact_flagged), (0, 0)); + for id in ids { + assert_eq!(duplicate_of(&mut conn, id).await, None); + } +} + /// One message held by three sources is one exact group with one survivor: /// the first-listed source's copy, with the other two pointing at it. #[tokio::test] diff --git a/crates/server/server/src/imports_api/promote.rs b/crates/server/server/src/imports_api/promote.rs index 7a7575f8f..7b7c52d0d 100644 --- a/crates/server/server/src/imports_api/promote.rs +++ b/crates/server/server/src/imports_api/promote.rs @@ -277,8 +277,11 @@ impl Promote<'_> { ), ); - let phase = Self::begin("Recording which backup each changed message came from…"); + let phase = Self::begin( + "Recording which backup each changed message came from, and whether its time has milliseconds…", + ); let dated = staging::promote_backup_dates(self.tx).await?; + staging::promote_time_precision(self.tx).await?; self.done( phase, words( diff --git a/crates/server/server/src/imports_api/staging.rs b/crates/server/server/src/imports_api/staging.rs index e89054f9d..e46d14d44 100644 --- a/crates/server/server/src/imports_api/staging.rs +++ b/crates/server/server/src/imports_api/staging.rs @@ -903,19 +903,20 @@ async fn flush_staging_message_chunk( Ok(()) } -/// Give the message staged under `row`'s guid what `row`, another copy of -/// it from the same import, adds, by the rules a later import of the copy -/// would follow (`db::staging::promote_deletion_marks`, +/// Give the message staged under `row`'s guid what `row`, another copy of it +/// from the same import, adds, by the rules a later import of the copy would +/// follow (`db::staging::promote_deletion_marks`, /// `db::staging::write_edit_map`): the attachments and reactions the staged /// message does not hold yet, and its mark and text as follows. When both /// backups have a date, a copy from a later backup gives its text, earlier /// versions and mark, mark or no mark, and one from an earlier backup gives /// neither (#1741, #1804). When either has no date, or the two dates are -/// equal ([`db_staging::later_backup`]), the copy gives its text and -/// earlier versions when it records a later edit, and its mark when it -/// carries one. One import of two backups then stores -/// what two separate imports of them store, in either file order (#1806, -/// #1837). +/// equal ([`db_staging::later_backup`]), the copy gives its text and earlier +/// versions when it records a later edit, and its mark when it carries one. +/// A copy at the staged message's time that has milliseconds marks it +/// `milliseconds` ([`db_staging::add_staged_copy_milliseconds`]). One import +/// of two backups then stores what two separate imports of them store, in +/// either file order (#1806, #1837). async fn add_staged_copy( tx: &mut SqliteConnection, stmts: &mut StagingInserts, @@ -929,6 +930,7 @@ async fn add_staged_copy( && row.msg.tapbacks.is_empty() && row.msg.deletion.is_none() && staged_source.backup_taken_at.is_none() + && row.msg.time_precision == message_ir::TimePrecision::Seconds { return Ok(()); } @@ -940,6 +942,9 @@ async fn add_staged_copy( let staged = db_staging::staged_message_id(tx, key) .await? .with_context(|| format!("no staged message holds the copy of {}", row.msg.guid))?; + if row.msg.time_precision == message_ir::TimePrecision::Milliseconds { + db_staging::add_staged_copy_milliseconds(tx, staged, &row.msg.timestamp).await?; + } let held_backup = db_staging::staged_backup_taken_at(tx, staged).await?; match db_staging::later_backup(staged_source.backup_taken_at, held_backup.as_deref()) { BackupOrder::Later(copy_backup) => { @@ -1001,6 +1006,7 @@ async fn insert_message_rows( source: staged_source.source, guid: &row.msg.guid, timestamp: &row.msg.timestamp, + time_precision: row.msg.time_precision, is_from_me: row.msg.is_from_me as i64, sender_handle_id: row.sender_handle_id, owner_handle_id: row.owner_handle_id, diff --git a/crates/server/server/src/imports_api/staging/tests.rs b/crates/server/server/src/imports_api/staging/tests.rs index 0e0c10ea6..4f0da36a3 100644 --- a/crates/server/server/src/imports_api/staging/tests.rs +++ b/crates/server/server/src/imports_api/staging/tests.rs @@ -188,7 +188,7 @@ async fn an_attachment_staging_refuses_is_a_rejection_naming_its_file() { let tmp = TempDir::new().unwrap(); let header = one_to_one_header(); // The path is the input under test, so the line stays written out. - let message = r#"{"guid":"g-escape","timestamp_unix_ms":1426183462000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550154","sender_display_name":null,"subject":null,"text":"hi","attachments":[{"path":"../escape.txt","original_name":null,"mime_type":null,"is_sticker":false,"transcription":null,"sticker_effect":null}],"imessage":null,"source":null} + let message = r#"{"guid":"g-escape","timestamp_unix_ms":1426183462000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550154","sender_display_name":null,"subject":null,"text":"hi","attachments":[{"path":"../escape.txt","original_name":null,"mime_type":null,"is_sticker":false,"transcription":null,"sticker_effect":null}],"imessage":null,"source":null} "#; let path = tmp.path().join("+15555550154.jsonl"); std::fs::write(&path, format!("{header}{message}")).unwrap(); diff --git a/crates/server/server/src/imports_api/tests.rs b/crates/server/server/src/imports_api/tests.rs index fb124b95f..55a998362 100644 --- a/crates/server/server/src/imports_api/tests.rs +++ b/crates/server/server/src/imports_api/tests.rs @@ -676,8 +676,8 @@ async fn staging_keeps_both_rows_when_guids_differ_only_by_whitespace() { let assets = tmp.path().join("assets"); let header = conversation_header("imessage", "+15555550123").participant("+15555550123", None); // The guids are the input under test, so these lines stay written out. - let first = r#"{"guid":"g-space","timestamp_unix_ms":1426183462000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550123","sender_display_name":null,"subject":null,"text":"trimmed","attachments":[{"path":"attachments/trim.bin","original_name":"trim.bin","mime_type":"application/octet-stream","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":12,"missing_reason":"not_found"}],"imessage":null,"source":null}"#; - let second = r#"{"guid":" g-space","timestamp_unix_ms":1426183463000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550123","sender_display_name":null,"subject":null,"text":"padded","attachments":[{"path":"attachments/pad.bin","original_name":"pad.bin","mime_type":"application/octet-stream","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":12,"missing_reason":"not_found"}],"imessage":null,"source":null}"#; + let first = r#"{"guid":"g-space","timestamp_unix_ms":1426183462000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550123","sender_display_name":null,"subject":null,"text":"trimmed","attachments":[{"path":"attachments/trim.bin","original_name":"trim.bin","mime_type":"application/octet-stream","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":12,"missing_reason":"not_found"}],"imessage":null,"source":null}"#; + let second = r#"{"guid":" g-space","timestamp_unix_ms":1426183463000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550123","sender_display_name":null,"subject":null,"text":"padded","attachments":[{"path":"attachments/pad.bin","original_name":"pad.bin","mime_type":"application/octet-stream","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":12,"missing_reason":"not_found"}],"imessage":null,"source":null}"#; let path = write_jsonl( tmp.path(), "guid-whitespace.jsonl", @@ -2357,7 +2357,7 @@ async fn rejects_attachment_path_traversal() { conversation_header("sms-backup-restore", "+15555550123") .participant("+15555550123", None), // The path is the input under test, so the line stays written out. - r#"{"guid":"g-trav","timestamp_unix_ms":1426183462000,"direction":"incoming","service":"sms","message_kind":"mms","sender_identity":"+15555550123","sender_display_name":null,"subject":null,"text":"x","attachments":[{"path":"../secret.txt","original_name":"secret.txt","mime_type":"text/plain","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":12,"missing_reason":null}],"imessage":null,"source":null} + r#"{"guid":"g-trav","timestamp_unix_ms":1426183462000,"time_precision":"milliseconds","direction":"incoming","service":"sms","message_kind":"mms","sender_identity":"+15555550123","sender_display_name":null,"subject":null,"text":"x","attachments":[{"path":"../secret.txt","original_name":"secret.txt","mime_type":"text/plain","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":12,"missing_reason":null}],"imessage":null,"source":null} "# ), ); @@ -2428,7 +2428,7 @@ async fn failed_replace_keeps_existing_messages() { conversation_header("sms-backup-restore", "+14075550107") .participant("+14075550107", None), // The path is the input under test, so the line stays written out. - r#"{"guid":"g-bad","timestamp_unix_ms":1426183462000,"direction":"incoming","service":"sms","message_kind":"mms","sender_identity":"+14075550107","sender_display_name":null,"subject":null,"text":"nope","attachments":[{"path":"../secret.txt","original_name":"secret.txt","mime_type":"text/plain","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":1,"missing_reason":null}],"imessage":null,"source":null} + r#"{"guid":"g-bad","timestamp_unix_ms":1426183462000,"time_precision":"milliseconds","direction":"incoming","service":"sms","message_kind":"mms","sender_identity":"+14075550107","sender_display_name":null,"subject":null,"text":"nope","attachments":[{"path":"../secret.txt","original_name":"secret.txt","mime_type":"text/plain","digest_sha256":null,"is_sticker":false,"transcription":null,"sticker_effect":null,"size_bytes":1,"missing_reason":null}],"imessage":null,"source":null} "# ), ); @@ -2789,7 +2789,7 @@ fn one_attachment_batch(path: &str, sha: Option<&str>) -> String { let digest = sha.map_or("null".to_string(), |sha| format!(r#""{sha}""#)); format!( r#"{header} -{{"guid":"g-att","timestamp_unix_ms":1700000000000,"direction":"incoming","service":"whatsapp","message_kind":"sms","sender_identity":"+15555550151","sender_display_name":null,"subject":null,"text":"x","attachments":[{{"path":"{path}","original_name":"a.bin","mime_type":"application/octet-stream","digest_sha256":{digest},"is_sticker":false,"transcription":null,"sticker_effect":null}}],"imessage":null,"source":null}} +{{"guid":"g-att","timestamp_unix_ms":1700000000000,"time_precision":"milliseconds","direction":"incoming","service":"whatsapp","message_kind":"sms","sender_identity":"+15555550151","sender_display_name":null,"subject":null,"text":"x","attachments":[{{"path":"{path}","original_name":"a.bin","mime_type":"application/octet-stream","digest_sha256":{digest},"is_sticker":false,"transcription":null,"sticker_effect":null}}],"imessage":null,"source":null}} "# ) } @@ -5810,3 +5810,4 @@ async fn a_page_of_import_runs_is_read_without_a_statement_per_row() { } mod backup_dates; +mod time_precision; diff --git a/crates/server/server/src/imports_api/tests/time_precision.rs b/crates/server/server/src/imports_api/tests/time_precision.rs new file mode 100644 index 000000000..813222828 --- /dev/null +++ b/crates/server/server/src/imports_api/tests/time_precision.rs @@ -0,0 +1,220 @@ +//! A message's `time_precision` through import, the API and an Export Run, +//! and the whole-second twin of a message one source holds with +//! milliseconds too (#1923). + +use super::*; + +/// 2015-03-12T18:04:22Z, the second every message here falls in. +const SECOND: i64 = 1_426_183_462_000; + +/// Create an Import Run for `source` with dedupe on, post `body` as its one +/// batch, and complete it. +async fn import_with_dedupe( + state: &crate::server::AppState, + token: &str, + source: &str, + body: String, +) { + let (_, created): (String, serde_json::Value) = post_created_json( + state, + "/v1/imports", + token, + serde_json::json!({ "source": source, "mode": "append", "dedupe": true }), + ) + .await; + let id = created["id"].as_i64().unwrap(); + let (status, text) = crate::test_support::post_raw( + state, + &format!("/v1/imports/{id}/batches"), + token, + "application/jsonl", + body, + ) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "{text}"); + let _: serde_json::Value = post_json( + state, + &format!("/v1/imports/{id}/complete"), + token, + serde_json::json!({ "status": "completed" }), + ) + .await; +} + +/// An SMS Backup+ file holding `line` as its one message. +fn sms_backup_plus_file(line: MessageLine) -> String { + let header = + conversation_header("sms-backup-plus", "+15555550123").participant("+15555550123", None); + format!("{header}\n{}\n", line.sms().sender("+15555550123")) +} + +/// The same SMS Backup+ message imported once from a file timed in whole +/// seconds and once from a file timed in milliseconds, in either order, is +/// shown once, with the millisecond time. +#[tokio::test] +async fn a_whole_second_copy_and_a_millisecond_copy_are_shown_once_with_the_milliseconds() { + let whole = || { + sms_backup_plus_file( + message_line("g-whole", "On my way") + .at(SECOND) + .whole_seconds(), + ) + }; + let exact = || sms_backup_plus_file(message_line("g-exact", "On my way").at(SECOND + 250)); + for (label, files) in [ + ("whole second first", [whole(), exact()]), + ("milliseconds first", [exact(), whole()]), + ] { + let (state, _fixture, token) = importer().await; + for file in files { + import_with_dedupe(&state, &token, "sms-backup-plus", file).await; + } + let page: serde_json::Value = get_json(&state, "/v1/messages", &token).await; + let items = page["items"].as_array().unwrap(); + assert_eq!(items.len(), 1, "{label}: {page}"); + assert_eq!(items[0]["timestamp"], "2015-03-12T18:04:22.250Z", "{label}"); + assert_eq!(items[0]["time_precision"], "milliseconds", "{label}"); + } +} + +/// A copy cut to the second elsewhere that kept the guid of a message whose +/// time has other milliseconds stays `seconds`: its time is not the +/// millisecond copy's, so nothing says the source recorded it. +#[tokio::test] +async fn a_whole_second_copy_at_another_time_keeps_seconds() { + let header = + conversation_header("sms-backup-plus", "+15555550123").participant("+15555550123", None); + let whole = message_line("g-same", "On my way") + .at(SECOND) + .whole_seconds() + .sms() + .sender("+15555550123"); + let exact = message_line("g-same", "On my way") + .at(SECOND + 678) + .sms() + .sender("+15555550123"); + for (label, imports) in [ + ( + "two imports", + vec![ + format!("{header}\n{whole}\n"), + format!("{header}\n{exact}\n"), + ], + ), + ("one import", vec![format!("{header}\n{whole}\n{exact}\n")]), + ] { + let (state, _fixture, token) = importer().await; + for body in imports { + import_with_dedupe(&state, &token, "sms-backup-plus", body).await; + } + let page: serde_json::Value = get_json(&state, "/v1/messages", &token).await; + let items = page["items"].as_array().unwrap(); + assert_eq!(items.len(), 1, "{label}: {page}"); + assert_eq!(items[0]["timestamp"], "2015-03-12T18:04:22.000Z", "{label}"); + assert_eq!(items[0]["time_precision"], "seconds", "{label}"); + } +} + +/// A message whose source recorded whole seconds and one whose source +/// recorded milliseconds that end in `.000` keep their precision through +/// the import, the API and an Export Run, and neither hides the other: they +/// are two messages, and the flag, never the time, says which has +/// milliseconds. +#[tokio::test] +async fn each_precision_is_kept_through_import_the_api_and_an_export_run() { + let (state, _fixture, token) = importer().await; + let header = + conversation_header("sms-backup-plus", "+15555550123").participant("+15555550123", None); + let whole = message_line("g-whole", "whole second") + .at(SECOND) + .whole_seconds() + .sms() + .sender("+15555550123"); + let exact = message_line("g-exact", "milliseconds that end in .000") + .at(SECOND + 1000) + .sms() + .sender("+15555550123"); + import_with_dedupe( + &state, + &token, + "sms-backup-plus", + format!("{header}\n{whole}\n{exact}\n"), + ) + .await; + + let precisions = |page: &serde_json::Value| -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = page["items"] + .as_array() + .unwrap() + .iter() + .map(|m| { + ( + m["guid"].as_str().unwrap().to_string(), + m["time_precision"].as_str().unwrap().to_string(), + ) + }) + .collect(); + out.sort(); + out + }; + let expected = vec![ + ("g-exact".to_string(), "milliseconds".to_string()), + ("g-whole".to_string(), "seconds".to_string()), + ]; + + let listed: serde_json::Value = get_json(&state, "/v1/messages", &token).await; + assert_eq!(precisions(&listed), expected, "{listed}"); + + let (_, run): (String, serde_json::Value) = post_created_json( + &state, + "/v1/exports", + &token, + serde_json::json!({ "scope": { "kind": "everything" } }), + ) + .await; + let exported: serde_json::Value = get_json( + &state, + &format!("/v1/exports/{}/messages", run["id"]), + &token, + ) + .await; + assert_eq!(precisions(&exported), expected, "{exported}"); +} + +/// A whole-second copy and a millisecond copy that ends in `.000` have one +/// guid, so they are one stored message. It says `milliseconds` whichever +/// copy came first, in two imports or in one: the source did record the +/// time to the millisecond. +#[tokio::test] +async fn one_message_held_at_whole_seconds_and_at_000_milliseconds_says_milliseconds() { + let line = |whole: bool| { + let line = message_line("g-same", "On my way").at(SECOND); + let line = if whole { line.whole_seconds() } else { line }; + line.sms().sender("+15555550123") + }; + let header = + conversation_header("sms-backup-plus", "+15555550123").participant("+15555550123", None); + let file = |whole: bool| format!("{header}\n{}\n", line(whole)); + let both = |first: bool| format!("{header}\n{}\n{}\n", line(first), line(!first)); + for (label, imports) in [ + ( + "whole second first, two imports", + vec![file(true), file(false)], + ), + ( + "milliseconds first, two imports", + vec![file(false), file(true)], + ), + ("whole second first, one import", vec![both(true)]), + ("milliseconds first, one import", vec![both(false)]), + ] { + let (state, _fixture, token) = importer().await; + for body in imports { + import_with_dedupe(&state, &token, "sms-backup-plus", body).await; + } + let page: serde_json::Value = get_json(&state, "/v1/messages", &token).await; + let items = page["items"].as_array().unwrap(); + assert_eq!(items.len(), 1, "{label}: {page}"); + assert_eq!(items[0]["time_precision"], "milliseconds", "{label}"); + } +} diff --git a/crates/server/server/src/models.rs b/crates/server/server/src/models.rs index 9c76eb3b3..404871244 100644 --- a/crates/server/server/src/models.rs +++ b/crates/server/server/src/models.rs @@ -6,7 +6,7 @@ use anyhow::{Context, Result}; use chrono::{DateTime, TimeZone, Utc}; use message_ir::{ ConversationHeader, Deletion, EarlierVersion, HandleService, HandleType, IrAttachment, - IrDirection, IrMessage, IrMessageKind, IrParticipant, Reaction, ReplyTo, + IrDirection, IrMessage, IrMessageKind, IrParticipant, Reaction, ReplyTo, TimePrecision, check_schema_version_in_json, nonempty, trimmed, }; use phone::Handle; @@ -99,6 +99,9 @@ pub struct MessageRecord { /// with three fractional digits and a `Z` suffix /// (`2015-03-12T18:04:22.250Z`). pub timestamp: String, + /// Whether the source recorded `timestamp` to the millisecond or in + /// whole seconds, as the conversation file says. + pub time_precision: TimePrecision, /// True for messages sent by the account owner. pub is_from_me: bool, /// Sender handle for incoming messages: the address, or the name when the @@ -396,6 +399,7 @@ fn message_from_ir( guid: msg.guid.clone(), line, timestamp, + time_precision: msg.time_precision, is_from_me, sender: sender.as_ref().map(|(value, _)| value.clone()), sender_handle_type: sender.and_then(|(_, kind)| kind), @@ -826,7 +830,7 @@ mod tests { .participant("+15555550101", Some("Sam")) .to_string(); // The timestamp is the input under test, so the line stays written out. - let msg = r#"{"guid":"g1","timestamp_unix_ms":9223372036854775807,"direction":"incoming","service":"sms","message_kind":"sms","sender_identity":"+15555550101","sender_display_name":"Sam","subject":null,"text":"hello","attachments":[],"imessage":null,"source":null}"#; + let msg = r#"{"guid":"g1","timestamp_unix_ms":9223372036854775807,"time_precision":"milliseconds","direction":"incoming","service":"sms","message_kind":"sms","sender_identity":"+15555550101","sender_display_name":"Sam","subject":null,"text":"hello","attachments":[],"imessage":null,"source":null}"#; let failure = parse_ir_lines([header, msg.to_string()]).unwrap_err(); match failure { ImportFailure::Invalid { line, .. } => assert_eq!(line, 2), @@ -845,7 +849,7 @@ mod tests { // The guids are the input under test, so the lines stay written out. let msg = |guid: &str| { format!( - r#"{{"guid":"{guid}","timestamp_unix_ms":1400773261000,"direction":"incoming","service":"sms","message_kind":"sms","sender_identity":"+15555550101","sender_display_name":"Sam","subject":null,"text":"hello","attachments":[],"imessage":null,"source":null}}"# + r#"{{"guid":"{guid}","timestamp_unix_ms":1400773261000,"time_precision":"milliseconds","direction":"incoming","service":"sms","message_kind":"sms","sender_identity":"+15555550101","sender_display_name":"Sam","subject":null,"text":"hello","attachments":[],"imessage":null,"source":null}}"# ) }; let lines = [header.to_string(), msg("g1"), msg(""), msg(" ")]; diff --git a/crates/server/server/src/test_support.rs b/crates/server/server/src/test_support.rs index cb84904d0..f3c37fb71 100644 --- a/crates/server/server/src/test_support.rs +++ b/crates/server/server/src/test_support.rs @@ -837,6 +837,8 @@ pub struct MessageRow<'a> { pub guid: Option, /// RFC 3339 in UTC to the millisecond, as the importer writes it. pub timestamp: &'a str, + /// `messages.time_precision`. + pub time_precision: message_ir::TimePrecision, /// Whether the account sent it. pub is_from_me: bool, /// `messages.sender_handle_id`. @@ -880,6 +882,7 @@ impl MessageRow<'_> { source: "imessage", guid: Some(unique_guid()), timestamp: "2020-01-01T00:00:00.000Z", + time_precision: message_ir::TimePrecision::Milliseconds, is_from_me: false, sender_handle_id: None, owner_handle_id: None, @@ -925,10 +928,11 @@ impl MessageRow<'_> { id, conversation_id, account_id, source, guid, timestamp, is_from_me, sender_handle_id, owner_handle_id, service, subject, body, is_announcement, is_reply, reply_to_guid, reply_to_part, - deletion, sort_order, content_key, duplicate_of, import_id + deletion, sort_order, content_key, duplicate_of, import_id, + time_precision ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, - $13, $14, $15, $16, $17, $18, $19, $20, $21 + $13, $14, $15, $16, $17, $18, $19, $20, $21, $22 ) RETURNING id", ) .bind(self.id) @@ -952,6 +956,7 @@ impl MessageRow<'_> { .bind(self.content_key) .bind(self.duplicate_of) .bind(self.import_id) + .bind(self.time_precision.as_str()) .fetch_one(&mut **tx) .await } diff --git a/crates/server/server/src/test_support/lines.rs b/crates/server/server/src/test_support/lines.rs index 4a95458b5..0c025d99e 100644 --- a/crates/server/server/src/test_support/lines.rs +++ b/crates/server/server/src/test_support/lines.rs @@ -149,6 +149,7 @@ pub fn message_line(guid: &str, text: &str) -> MessageLine { MessageLine(message_ir::IrMessage { guid: guid.to_string(), timestamp_unix_ms: 1_426_183_462_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: message_ir::IrDirection::Incoming, service: message_ir::IrService::IMessage, message_kind: message_ir::IrMessageKind::IMessage, @@ -174,6 +175,13 @@ impl MessageLine { self } + /// From a source that records whole seconds: `time_precision` is + /// `seconds`. The time stays as it was. + pub fn whole_seconds(mut self) -> Self { + self.0.time_precision = message_ir::TimePrecision::Seconds; + self + } + /// Sent by the account holder. pub fn outgoing(mut self) -> Self { self.0.direction = message_ir::IrDirection::Outgoing; diff --git a/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl b/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl index 1e9fb24e9..0d76945a4 100644 --- a/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl +++ b/crates/server/server/tests/fixtures/apple-messages-deletions.jsonl @@ -1,4 +1,4 @@ -{"schema_version":11,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578308040000,"last_timestamp_unix_ms":1578308160000}}} -{"guid":"guid-16","timestamp_unix_ms":1578308040000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Delete me","attachments":[],"deletion":"deleted_in_source_app","imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Delete me"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} -{"guid":"guid-17","timestamp_unix_ms":1578308100000,"direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"","attachments":[],"deletion":"unsent","imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} -{"guid":"guid-18","timestamp_unix_ms":1578308160000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Still here","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Still here"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} +{"schema_version":12,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578308040000,"last_timestamp_unix_ms":1578308160000}}} +{"guid":"guid-16","timestamp_unix_ms":1578308040000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Delete me","attachments":[],"deletion":"deleted_in_source_app","imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Delete me"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} +{"guid":"guid-17","timestamp_unix_ms":1578308100000,"time_precision":"milliseconds","direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"","attachments":[],"deletion":"unsent","imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} +{"guid":"guid-18","timestamp_unix_ms":1578308160000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Still here","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Still here"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} diff --git a/crates/server/server/tests/fixtures/apple-messages-edits.jsonl b/crates/server/server/tests/fixtures/apple-messages-edits.jsonl index 2342100ba..70311c1dc 100644 --- a/crates/server/server/tests/fixtures/apple-messages-edits.jsonl +++ b/crates/server/server/tests/fixtures/apple-messages-edits.jsonl @@ -1,4 +1,4 @@ -{"schema_version":11,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578309000000,"last_timestamp_unix_ms":1578309120000}}} -{"guid":"guid-edited-twice","timestamp_unix_ms":1578309000000,"direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Meet at the bakery","attachments":[],"edits":[{"part_index":0,"text":"Meet at the library","edited_at_unix_ms":1578309000000},{"part_index":0,"text":"Meet at the museum","edited_at_unix_ms":1578309030000}],"imessage":null,"source":null} -{"guid":"guid-edited-final-match","timestamp_unix_ms":1578309060000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"The library opens at nine","attachments":[],"edits":[{"part_index":0,"text":"The library opens at eight","edited_at_unix_ms":1578309060000}],"imessage":null,"source":null} -{"guid":"guid-never-edited","timestamp_unix_ms":1578309120000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Nothing changed here","attachments":[],"imessage":null,"source":null} +{"schema_version":12,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578309000000,"last_timestamp_unix_ms":1578309120000}}} +{"guid":"guid-edited-twice","timestamp_unix_ms":1578309000000,"time_precision":"milliseconds","direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Meet at the bakery","attachments":[],"edits":[{"part_index":0,"text":"Meet at the library","edited_at_unix_ms":1578309000000},{"part_index":0,"text":"Meet at the museum","edited_at_unix_ms":1578309030000}],"imessage":null,"source":null} +{"guid":"guid-edited-final-match","timestamp_unix_ms":1578309060000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"The library opens at nine","attachments":[],"edits":[{"part_index":0,"text":"The library opens at eight","edited_at_unix_ms":1578309060000}],"imessage":null,"source":null} +{"guid":"guid-never-edited","timestamp_unix_ms":1578309120000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Nothing changed here","attachments":[],"imessage":null,"source":null} diff --git a/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl b/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl index 3f2cc0f3d..7c0219e75 100644 --- a/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl +++ b/crates/server/server/tests/fixtures/apple-messages-reactions.jsonl @@ -1,4 +1,4 @@ -{"schema_version":11,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"chat100","conversation_type":"group","group_title":"Weekend plans","participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"},{"identity":"friend@example.com","display_name":null,"identity_type":"email"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578307860000,"last_timestamp_unix_ms":1578307980000}}} -{"guid":"00000000-0000-4000-8000-000000000013","timestamp_unix_ms":1578307860000,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"friend@example.com","sender_display_name":"friend@example.com","owner_identity":"+15555550106","subject":null,"text":"Pizza?","attachments":[],"reactions":[{"part_index":0,"kind":"loved","is_from_me":false,"reactor_identity":"+15555550107","reactor_display_name":"+15555550107"},{"part_index":0,"kind":"emoji","emoji":"🔥","is_from_me":true,"reactor_display_name":"+15555550106"}],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Pizza?"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} -{"guid":"guid-14","timestamp_unix_ms":1578307920000,"direction":"incoming","service":"imessage","message_kind":"tapback","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Loved a message","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":"00000000-0000-4000-8000-000000000013","associated_part":0,"tapback_kind":"loved","tapback_emoji":null,"tapback_action":"add"},"source":null} -{"guid":"guid-15","timestamp_unix_ms":1578307980000,"direction":"outgoing","service":"imessage","message_kind":"tapback","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"🔥 reacted","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":"00000000-0000-4000-8000-000000000013","associated_part":0,"tapback_kind":"emoji","tapback_emoji":"🔥","tapback_action":"add"},"source":null} +{"schema_version":12,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"chat100","conversation_type":"group","group_title":"Weekend plans","participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"},{"identity":"friend@example.com","display_name":null,"identity_type":"email"}],"stats":{"message_count":3,"attachment_count":0,"first_timestamp_unix_ms":1578307860000,"last_timestamp_unix_ms":1578307980000}}} +{"guid":"00000000-0000-4000-8000-000000000013","timestamp_unix_ms":1578307860000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"friend@example.com","sender_display_name":"friend@example.com","owner_identity":"+15555550106","subject":null,"text":"Pizza?","attachments":[],"reactions":[{"part_index":0,"kind":"loved","is_from_me":false,"reactor_identity":"+15555550107","reactor_display_name":"+15555550107"},{"part_index":0,"kind":"emoji","emoji":"🔥","is_from_me":true,"reactor_display_name":"+15555550106"}],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":[{"index":0,"kind":"run","text":"Pizza?"}],"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":null,"associated_part":null,"tapback_kind":null,"tapback_emoji":null,"tapback_action":null},"source":null} +{"guid":"guid-14","timestamp_unix_ms":1578307920000,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"tapback","sender_identity":"+15555550107","sender_display_name":"+15555550107","owner_identity":"+15555550106","subject":null,"text":"Loved a message","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":"00000000-0000-4000-8000-000000000013","associated_part":0,"tapback_kind":"loved","tapback_emoji":null,"tapback_action":"add"},"source":null} +{"guid":"guid-15","timestamp_unix_ms":1578307980000,"time_precision":"milliseconds","direction":"outgoing","service":"imessage","message_kind":"tapback","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"🔥 reacted","attachments":[],"imessage":{"send_effect":null,"shared_location":null,"announcement":null,"read_receipt_rfc3339":null,"parts":null,"app":null,"balloon_bundle_id":null,"balloon_kind":null,"associated_guid":"00000000-0000-4000-8000-000000000013","associated_part":0,"tapback_kind":"emoji","tapback_emoji":"🔥","tapback_action":"add"},"source":null} diff --git a/crates/server/server/tests/fixtures/apple-messages-sub-second-times.jsonl b/crates/server/server/tests/fixtures/apple-messages-sub-second-times.jsonl index 04a008e8d..e1f2db387 100644 --- a/crates/server/server/tests/fixtures/apple-messages-sub-second-times.jsonl +++ b/crates/server/server/tests/fixtures/apple-messages-sub-second-times.jsonl @@ -1,3 +1,3 @@ -{"schema_version":10,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":2,"attachment_count":0,"first_timestamp_unix_ms":1578309000250,"last_timestamp_unix_ms":1578309000550}}} -{"guid":"guid-300-ms-later","timestamp_unix_ms":1578309000550,"direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"On my way","attachments":[],"imessage":null,"source":null} -{"guid":"guid-first","timestamp_unix_ms":1578309000250,"direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Meet at the bakery","attachments":[],"edits":[{"part_index":0,"text":"Meet at the library","edited_at_unix_ms":1578309000125}],"imessage":null,"source":null} +{"schema_version":12,"export":{"source":"imessage","tool":"imessage-ir-exporter","tool_version":"0.1.0","owner_identity":"+15555550106","owner_display_name":null},"conversation":{"chat_identifier":"+15555550107","conversation_type":"individual","group_title":null,"participants":[{"identity":"+15555550107","display_name":null,"identity_type":"phone"}],"stats":{"message_count":2,"attachment_count":0,"first_timestamp_unix_ms":1578309000250,"last_timestamp_unix_ms":1578309000550}}} +{"guid":"guid-300-ms-later","timestamp_unix_ms":1578309000550,"time_precision":"milliseconds","direction":"incoming","service":"imessage","message_kind":"imessage","sender_identity":"+15555550107","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"On my way","attachments":[],"imessage":null,"source":null} +{"guid":"guid-first","timestamp_unix_ms":1578309000250,"time_precision":"milliseconds","direction":"outgoing","service":"imessage","message_kind":"imessage","sender_identity":"+15555550106","sender_display_name":null,"owner_identity":"+15555550106","subject":null,"text":"Meet at the bakery","attachments":[],"edits":[{"part_index":0,"text":"Meet at the library","edited_at_unix_ms":1578309000125}],"imessage":null,"source":null} diff --git a/docs/architecture/contacts-identities-and-messages.md b/docs/architecture/contacts-identities-and-messages.md index bf002ae3a..0bcae4e7c 100644 --- a/docs/architecture/contacts-identities-and-messages.md +++ b/docs/architecture/contacts-identities-and-messages.md @@ -443,6 +443,32 @@ was written, not when a part was unsent, so they cannot tell with dedupe off still leaves a changed message's duplicate flag until the next dedupe ([#1805](https://github.com/messagecrate/message-crate/issues/1805)). +**Within one source, a whole-second message is the duplicate of its +millisecond twin.** Each message says whether its source recorded its time to +the millisecond or in whole seconds (`time_precision` in the conversation +file, `messages.time_precision`). The dedupe hides a whole-second message as +the duplicate of a message from the same source that matches it in everything +else and has milliseconds in the same second, so the message is shown once. It +is shown with its milliseconds unless another source holds it too and that +source's copy wins the cross-source comparison (`pick_winner`, which ranks by +attachments and then import order, not precision). The content key stays at +whole seconds, so the two share a key, and the exact pass sets the +whole-second copy aside before it compares sources (`content_key_group_flags` +in `dedupe.rs`). A source that holds a message only in whole seconds, or only +with milliseconds, keeps every copy, as before. The flag decides, never the +time: a millisecond time that ends in `.000` is not a whole second. A +whole-second copy and a millisecond copy whose time ends in `.000` have one +guid, so they are one stored message, and it says `milliseconds` whichever +copy came first, in one import or across several (`promote_time_precision` and +`add_staged_copy_milliseconds` in `db/staging.rs`). Why: one source can record +one message twice, once without its milliseconds (an SMS Backup+ mail timed by +its `Date` header beside one timed by `X-smssync-date`), and the two copies +have different guids, which are made at milliseconds, so both reach the +database; without the flag, a copy that landed on `.000` could not be told +from a copy that never had milliseconds +([#1096](https://github.com/messagecrate/message-crate/issues/1096), +[#1923](https://github.com/messagecrate/message-crate/issues/1923)). + **A participant's display name has one rule.** The contact's name, else what that backup called them in that conversation, else the identity. One loader applies it for the conversation list, the message pane, and Export. diff --git a/docs/src/assets/openapi.json b/docs/src/assets/openapi.json index d31bd5f69..52ce88eba 100644 --- a/docs/src/assets/openapi.json +++ b/docs/src/assets/openapi.json @@ -15769,6 +15769,7 @@ "source", "guid", "timestamp", + "time_precision", "sort_order", "is_from_me", "is_announcement", @@ -15912,6 +15913,10 @@ ], "description": "Body text, when present." }, + "time_precision": { + "$ref": "#/components/schemas/TimePrecision", + "description": "Whether the source recorded `timestamp` to the millisecond or in\nwhole seconds. A `timestamp` ending in `.000` is a whole second\nonly when this says `seconds`." + }, "timestamp": { "type": "string", "description": "The instant the message was sent, to the millisecond: RFC 3339\nin UTC with three fractional digits and a `Z` suffix\n(`2015-03-12T18:04:22.250Z`; `.000` when the source records\nwhole seconds). Messages are listed in the order of this time. A\ncaller shows it in the account's time zone\n(`Account.time_zone`); the database stores nothing\nabout where the phone was." @@ -17833,6 +17838,7 @@ "source", "guid", "timestamp", + "time_precision", "sort_order", "is_from_me", "is_announcement", @@ -17976,6 +17982,10 @@ ], "description": "Body text, when present." }, + "time_precision": { + "$ref": "#/components/schemas/TimePrecision", + "description": "Whether the source recorded `timestamp` to the millisecond or in\nwhole seconds. A `timestamp` ending in `.000` is a whole second\nonly when this says `seconds`." + }, "timestamp": { "type": "string", "description": "The instant the message was sent, to the millisecond: RFC 3339\nin UTC with three fractional digits and a `Z` suffix\n(`2015-03-12T18:04:22.250Z`; `.000` when the source records\nwhole seconds). Messages are listed in the order of this time. A\ncaller shows it in the account's time zone\n(`Account.time_zone`); the database stores nothing\nabout where the phone was." @@ -18971,6 +18981,14 @@ } } }, + "TimePrecision": { + "type": "string", + "description": "How finely the source recorded a message's time.", + "enum": [ + "seconds", + "milliseconds" + ] + }, "TopAttachment": { "type": "object", "description": "One of an account's largest attachments by byte size.", diff --git a/docs/src/content/docs/docs/developer/architecture/common-message.md b/docs/src/content/docs/docs/developer/architecture/common-message.md index 802dbaca5..ebda41ed6 100644 --- a/docs/src/content/docs/docs/developer/architecture/common-message.md +++ b/docs/src/content/docs/docs/developer/architecture/common-message.md @@ -30,13 +30,13 @@ Pipeline: `backup → common message → FormatSink → user-picked format`. - **Common-message path** (`ConversationDocument` → `message_ir_format::FormatSink`, one of json/jsonl/csv/eml/mbox/xml): all exporters, including iMessage (`imessage-ir-exporter`). Per-chat formats also accept `write_format`; XML uses a single `smses.xml` via the sink. - **Media + obfuscate** run inside `FormatSink::finish` for every format (`message_crate_core::ExportTransforms`: none / copy / convert / compress, plus optional obfuscate). When obfuscate is on, exporters skip staging real attachment bytes and convert/compress is not run — only placeholder files are written. Exporters pass transforms from `ExporterConfig.media` / `.obfuscate`; there is no CSV-only post-step. EML / MBOX / XML embed media and drop the staged `attachments/` directory afterward. -- **Schema version 11 only** (breaking). Version 11 says when the backup was made, in `export.backup_taken_at_unix_ms` (see [When the backup was made](#when-the-backup-was-made)), which version 10 did not, so an import could not tell which of two backups of one phone is the later one. Version 10 had kept the message a reply quotes in the message's own `reply_to`, for every source (see [Replies](#replies)), where version 9 kept the Apple Messages reply link in `imessage.is_reply` and `imessage.in_reply_to_guid`, and a reply count in `imessage.num_replies`. Version 9 had given orphaned messages conversations of type `orphaned` (see [Orphaned messages](#orphaned-messages)), where version 8 put them all in one `individual` conversation named `orphaned`. Version 8 had kept an edited message's earlier versions in its own `edits`, for every source, where version 7 kept the Apple Messages edit history as a JSON value in `imessage.edits`. Version 7 had moved a message's mark, Deleted in the source app or Unsent, in its own `deletion`, for every source, where version 6 kept the Apple Messages deleted mark in `imessage.is_deleted`. Version 6 had moved a message's reactions into its own `reactions` list, one shape for every source, where version 5 kept Apple Messages reactions as a JSON value in `imessage.tapbacks`. Version 5 had named every address an identity (`identity`, `identity_type`, `owner_identity`, `sender_identity`, `reactor_identity`) where version 4 said `handle`. Version 10 and older are refused, never upgraded. Typed enums/bags, filled outgoing identity, conversation stats, stable null/`[]` keys. Older common-message JSON is not read — regenerate exports after schema changes. +- **Schema version 12 only** (breaking). Version 12 says whether each message's time has milliseconds, in its required `time_precision` (see [Time precision](#time-precision)), which version 11 did not, so a time ending in `.000` could be a whole second or a millisecond time, and the import could not tell a whole-second copy of a message from a millisecond one. Version 11 had said when the backup was made, in `export.backup_taken_at_unix_ms` (see [When the backup was made](#when-the-backup-was-made)), which version 10 did not, so an import could not tell which of two backups of one phone is the later one. Version 10 had kept the message a reply quotes in the message's own `reply_to`, for every source (see [Replies](#replies)), where version 9 kept the Apple Messages reply link in `imessage.is_reply` and `imessage.in_reply_to_guid`, and a reply count in `imessage.num_replies`. Version 9 had given orphaned messages conversations of type `orphaned` (see [Orphaned messages](#orphaned-messages)), where version 8 put them all in one `individual` conversation named `orphaned`. Version 8 had kept an edited message's earlier versions in its own `edits`, for every source, where version 7 kept the Apple Messages edit history as a JSON value in `imessage.edits`. Version 7 had moved a message's mark, Deleted in the source app or Unsent, in its own `deletion`, for every source, where version 6 kept the Apple Messages deleted mark in `imessage.is_deleted`. Version 6 had moved a message's reactions into its own `reactions` list, one shape for every source, where version 5 kept Apple Messages reactions as a JSON value in `imessage.tapbacks`. Version 5 had named every address an identity (`identity`, `identity_type`, `owner_identity`, `sender_identity`, `reactor_identity`) where version 4 said `handle`. Version 11 and older are refused, never upgraded. Typed enums/bags, filled outgoing identity, conversation stats, stable null/`[]` keys. Older common-message JSON is not read — regenerate exports after schema changes. -## Document schema (`schema_version: 11`) +## Document schema (`schema_version: 12`) ```json { - "schema_version": 11, + "schema_version": 12, "export": { "source": "sms-backup-restore", "tool": "SMS Backup & Restore", @@ -63,6 +63,7 @@ Pipeline: `backup → common message → FormatSink → user-picked format`. { "guid": "…", "timestamp_unix_ms": 1400773261000, + "time_precision": "milliseconds", "direction": "outgoing", "service": "sms", "message_kind": "sms", @@ -96,6 +97,27 @@ Pipeline: `backup → common message → FormatSink → user-picked format`. - `guid` is Apple's own id for Apple Messages. Every other source's `guid` is a `MessageGuid`: SHA-256 of the chat id, the direction, the sender of an incoming message, the UTC instant in milliseconds, the text with whitespace collapsed, the sorted attachment digests, and the source's own key where it has one (WhatsApp's `key_id`). It reads no time zone and no display format, so one backup gives the same ids on any computer. The server refuses a message whose `guid` is empty. - Two records a backup cannot tell apart are one message, and the exporter keeps one (`message_ir::one_copy_per_message`). The server's content key, which matches one message across sources, is the same identity at whole seconds. +### Time precision + +Every message has a required `time_precision`: `milliseconds` when the source recorded the time below the second, `seconds` when it recorded whole seconds and `timestamp_unix_ms` ends in `000` because the source has nothing finer. The flag, never the time, says which: a millisecond time can end in `000` too, and it is still `milliseconds`. Each exporter writes what its source records: + +| Source | Precision | +|--------|-----------| +| Apple Messages | `milliseconds` | +| WhatsApp | `milliseconds` | +| SMS Backup & Restore | `milliseconds`, from the `date` attribute | +| GO SMS Pro | `milliseconds` for a message from the XML backup; `seconds` for one read only from a PDU file, whose name records the second | +| SMS Backup+ | `milliseconds` from `X-smssync-date`; `seconds` for a mail without it, timed by its `Date` header | +| iMazing | `seconds` | +| OpenExtract | `seconds` | +| An Export Run of the server | The precision the server stored | + +When an exporter keeps one copy of a message that its source recorded twice ([Identity](#identity)), a whole-second copy that takes a millisecond copy's time takes its precision too. + +The server keeps the flag and answers it as `time_precision` on a message. Within one source, it shows a whole-second message once when the source also holds it with milliseconds in the same second: the whole-second copy is hidden as the duplicate, and the message is shown with its milliseconds, unless another source holds it too and that source's copy is the one shown. An SMS Backup+ message imported once from a mail timed by `Date` and once from one timed by `X-smssync-date` is one message. + +CSV carries it in the `time_precision` column, and EML and MBOX in the `X-ME-Time-Precision` header. A blank or unknown value is refused rather than guessed. SMS Backup & Restore XML has no place for it: an Export Run that writes XML leaves it out, and the XML reads back as `milliseconds`, the precision its `date` attribute holds, whatever the server stored. + ### When the backup was made `export.backup_taken_at_unix_ms` is when the backup the file was read from was made, in Unix milliseconds, or `null` when nothing says. Each exporter reads it from its source: @@ -204,7 +226,7 @@ Attachment **bytes** are never stored in JSON/JSONL (`#[serde(skip)]`). Paths + ## JSONL layout ```text -{"schema_version":11,"export":{…},"conversation":{…}} +{"schema_version":12,"export":{…},"conversation":{…}} {"guid":"…","timestamp_unix_ms":…, …} … ``` diff --git a/docs/src/content/docs/docs/developer/formats/index.md b/docs/src/content/docs/docs/developer/formats/index.md index f439d6d44..2820a89c7 100644 --- a/docs/src/content/docs/docs/developer/formats/index.md +++ b/docs/src/content/docs/docs/developer/formats/index.md @@ -9,7 +9,7 @@ What each converter writes (and where it falls short). Marks: **yes** / **partia ## Shared model -All converters build a **common message** per conversation (`ConversationDocument`, schema version 10 in [`message-ir`](https://github.com/messagecrate/message-crate/tree/main/crates/libs/ir)), then project the user-picked format via `FormatSink` in [`message-ir-format`](https://github.com/messagecrate/message-crate/tree/main/crates/libs/ir-format) (default **JSON**). When packaging is CSV, columns follow [`CSV_HEADERS`](https://github.com/messagecrate/message-crate/blob/main/crates/libs/ir-format/src/write.rs). Across the board: +All converters build a **common message** per conversation (`ConversationDocument`, schema version 12 in [`message-ir`](https://github.com/messagecrate/message-crate/tree/main/crates/libs/ir)), then project the user-picked format via `FormatSink` in [`message-ir-format`](https://github.com/messagecrate/message-crate/tree/main/crates/libs/ir-format) (default **JSON**). When packaging is CSV, columns follow [`CSV_HEADERS`](https://github.com/messagecrate/message-crate/blob/main/crates/libs/ir-format/src/write.rs). Across the board: - The peer is `chat_identifier` — there is **no** dedicated receiver-phone column - Every participant is a **typed identity**: `identity_type` (`phone` / `email` / `username` / `other`) on each JSON/JSONL participant and inside `participants_json`; the CSV `identity_type` column carries the sender's type, inferred from the identity when the source doesn't supply it diff --git a/docs/src/content/docs/docs/developer/formats/mail-archive.md b/docs/src/content/docs/docs/developer/formats/mail-archive.md index 50229f3aa..c184f51e7 100644 --- a/docs/src/content/docs/docs/developer/formats/mail-archive.md +++ b/docs/src/content/docs/docs/developer/formats/mail-archive.md @@ -144,6 +144,7 @@ A mail an earlier Message Crate wrote names its addresses with `X-ME-Sender-Hand | `X-ME-Service` | lowercase common-message vocabulary preferred (`sms` / `imessage` / …) | Older exports may use `SMS` / `iMessage` | | `X-ME-Message-Kind` | see taxonomy below | | | `X-ME-Timestamp-Unix-Ms` | integer string | Authoritative epoch ms (UTC) | +| `X-ME-Time-Precision` | `seconds` / `milliseconds` | Whether the source recorded the time below the second; required. The header, never the time, decides: a millisecond time can end in `000`. A missing header or another value is refused | | `X-ME-Subject` | string | When distinct from mail `Subject` | | `X-ME-Guid` | hex / guid string | Matches CSV `guid` when possible | | `X-ME-Export-Source` | string | e.g. `sms-backup-restore` | @@ -340,6 +341,7 @@ Normal sticker sends: image MIME part + `X-ME-Attachment-Meta` (`is_sticker`, `s | `group_title` | `X-ME-Group-Title` | | `guid` | `X-ME-Guid` + `Message-ID` | | `timestamp` / `timestamp_utc` / `timestamp_unix_ms` | `Date` + `X-ME-Timestamp-Unix-Ms` | +| `time_precision` | `X-ME-Time-Precision` (`seconds`/`milliseconds`; required) | | `direction` | `X-ME-Direction` | | `service` | `X-ME-Service` | | `sender_identity` / `sender_display_name` | headers + `From` phrase | diff --git a/docs/src/content/docs/docs/developer/message-transfer.md b/docs/src/content/docs/docs/developer/message-transfer.md index ea18e9124..09a6c6191 100644 --- a/docs/src/content/docs/docs/developer/message-transfer.md +++ b/docs/src/content/docs/docs/developer/message-transfer.md @@ -41,11 +41,11 @@ Each conversation is one text file whose name ends in `.jsonl`. JSON Lines means Pictures and other media sit next to those files in `attachments/`. ```jsonl title="One conversation file" -{"schema_version":11,"export":{"source":"sms-backup-restore","tool":"SMS Backup & Restore","owner_identity":"+15555550100","owner_display_name":"Me","backup_taken_at_unix_ms":1400800000000},"conversation":{"chat_identifier":"+15555550101","conversation_type":"individual","participants":[{"identity":"+15555550101","display_name":"Sam"}]}} -{"guid":"msg-1","timestamp_unix_ms":1400773261000,"direction":"outgoing","service":"sms","text":"Hello"} +{"schema_version":12,"export":{"source":"sms-backup-restore","tool":"SMS Backup & Restore","owner_identity":"+15555550100","owner_display_name":"Me","backup_taken_at_unix_ms":1400800000000},"conversation":{"chat_identifier":"+15555550101","conversation_type":"individual","participants":[{"identity":"+15555550101","display_name":"Sam"}]}} +{"guid":"msg-1","timestamp_unix_ms":1400773261000,"time_precision":"milliseconds","direction":"outgoing","service":"sms","text":"Hello"} ``` -The server only reads this current layout (schema version 10). A version-9 file is refused by name, never upgraded. The full field list is on [Export structure](/docs/developer/reference/export-structure/). +The server only reads this current layout (schema version 12). A version-11 file is refused by name, never upgraded. The full field list is on [Export structure](/docs/developer/reference/export-structure/). ## Converters for full backups diff --git a/docs/src/content/docs/docs/developer/reference/csv-columns.md b/docs/src/content/docs/docs/developer/reference/csv-columns.md index 5d4d77615..a316a0d7e 100644 --- a/docs/src/content/docs/docs/developer/reference/csv-columns.md +++ b/docs/src/content/docs/docs/developer/reference/csv-columns.md @@ -24,6 +24,7 @@ CSV output contains one row per message. Conversation and export identity are re | `timestamp_utc` | UTC RFC 3339 time. | | `timestamp_display` | Human-readable time. | | `timestamp_unix_ms` | Unix time in milliseconds. | +| `time_precision` | `milliseconds` when the source recorded the time below the second, `seconds` when it recorded whole seconds. A millisecond time can end in `000` and is still `milliseconds`. A file with a blank or any other value is refused. | | `direction` | `incoming` or `outgoing`. | | `service` | `sms`, `imessage`, `whatsapp`, `rcs`, `discord`, `signal`, `telegram`, `slack`, or `unknown`. | | `sender_identity` | Sender phone number, email, or other identity. Outgoing rows use the export owner when known. | diff --git a/docs/src/content/docs/docs/developer/reference/database.md b/docs/src/content/docs/docs/developer/reference/database.md index e81c4abb0..1386607a0 100644 --- a/docs/src/content/docs/docs/developer/reference/database.md +++ b/docs/src/content/docs/docs/developer/reference/database.md @@ -54,7 +54,8 @@ handle is on in `contact_handles`. ### `messages` -One row = one message (`source`, `guid`, timestamps, `is_from_me`, optional +One row = one message (`source`, `guid`, timestamps, `time_precision`, +`is_from_me`, optional `service` for per-message transport such as `sms` / `imessage` / `rcs` / `whatsapp`, `body`, `content_key`, optional `sender_handle_id` → `handles`, optional `duplicate_of`). diff --git a/docs/src/content/docs/docs/developer/reference/export-structure.md b/docs/src/content/docs/docs/developer/reference/export-structure.md index 2b1e64d5a..1e9619e48 100644 --- a/docs/src/content/docs/docs/developer/reference/export-structure.md +++ b/docs/src/content/docs/docs/developer/reference/export-structure.md @@ -1,9 +1,9 @@ --- title: Export structure -description: The JSONL format Message Crate imports — schema version 10, one file per conversation. +description: The JSONL format Message Crate imports — schema version 12, one file per conversation. --- -Message Crate imports JSONL (JSON Lines) exports at schema version 10. Version 9 and older are refused, never upgraded. This page describes the format for CLI users and tool authors. +Message Crate imports JSONL (JSON Lines) exports at schema version 12. Version 11 and older are refused, never upgraded. This page describes the format for CLI users and tool authors. ## Happy path @@ -16,7 +16,9 @@ The JSONL files are plain text — one JSON object per line. The format is the s One `*.jsonl` per conversation, plus media files under `attachments/`: 1. **Line 1** — Conversation header (`schema_version`, `export`, `conversation`) -2. **Following lines** — One message per line (`timestamp_unix_ms`, `direction`, `service`, `text`, `attachments`, and optional fields) +2. **Following lines** — One message per line (`timestamp_unix_ms`, `time_precision`, `direction`, `service`, `text`, `attachments`, and optional fields) + +`time_precision` is required: `milliseconds` when the source recorded the time below the second, `seconds` when it recorded whole seconds. A millisecond time can end in `000` and is still `milliseconds`. Within one source, the server shows a whole-second message once when the source also holds it with milliseconds in the same second, with the milliseconds, unless another source holds it too and that source's copy is the one shown. `service` is the channel (`sms`, `imessage`, `rcs`, …). A message's reactions are its `reactions` list, one shape for every source: each names the part reacted to (`part_index`), what the reaction is (`kind`, and `emoji` for an emoji reaction), whether the owner reacted (`is_from_me`), and who did (`reactor_identity`, `reactor_display_name`). The server stores each reaction under the person who reacted. A message deleted in the source app before the backup carries `"deletion": "deleted_in_source_app"`, and one its sender unsent carries `"deletion": "unsent"`; a message with neither leaves `deletion` out. The server stores the mark and imports the message like any other. An edited message carries its earlier versions in `edits`, oldest first within each part, each with its `part_index`, its `text` and the time it was written (`edited_at_unix_ms`); `text` is the final version, and a message never edited leaves `edits` out. The server stores each version, and search finds the message by any of them. A reply carries `reply_to`, naming the quoted message's `guid` when the source names it and the part replied to (`part_index`) when the source records one; a reply whose source names no quoted message has a `null` `guid`, and a message that is not a reply leaves `reply_to` out. A named message can still be missing from the export, such as an Apple Messages thread's first message deleted before the backup was made, which the backup does not hold. The server counts each message's replies when it reads it. Apple-specific fields such as message effects live under an optional `imessage` object. @@ -29,7 +31,7 @@ Attachment records may include `digest_sha256` so clients can upload by hash (`P ## Schema compatibility -The server reads one schema version, currently 10. Version 10 keeps the message a reply quotes in the message's own `reply_to`, for every source, where version 9 kept the Apple Messages reply link in `imessage.is_reply` and `imessage.in_reply_to_guid`. Version 9 had given orphaned messages, ones the backup holds without recording which conversation they were said in, conversations of type `orphaned`: one for each sender, keyed `orphaned:` and the sender's address, and one with no participants, keyed `orphaned:`, for the ones the account holder sent; version 8 put them all in one `individual` conversation named `orphaned`. Version 8 had kept an edited message's earlier versions in its own `edits`, where version 7 kept the Apple Messages edit history in `imessage.edits`. Version 7 had moved a message's mark, Deleted in the source app or Unsent, in its own `deletion`, where version 6 kept the Apple Messages deleted mark in `imessage.is_deleted`. Version 6 had moved a message's reactions into its own `reactions` list, where version 5 kept Apple Messages reactions in `imessage.tapbacks`. Version 5 had called every address an identity (`identity`, `identity_type`, `owner_identity`, `sender_identity`, `reactor_identity`) where version 4 said `handle`. A file written at any other version is refused, with an error naming both the file's version and the version the server expects. To import an older export, re-export it with the current desktop app. +The server reads one schema version, currently 12. Version 12 says whether each message's time has milliseconds, in its `time_precision`, where version 11 did not. Version 11 had said when the backup was made, in `export.backup_taken_at_unix_ms`, where version 10 did not. Version 10 had kept the message a reply quotes in the message's own `reply_to`, for every source, where version 9 kept the Apple Messages reply link in `imessage.is_reply` and `imessage.in_reply_to_guid`. Version 9 had given orphaned messages, ones the backup holds without recording which conversation they were said in, conversations of type `orphaned`: one for each sender, keyed `orphaned:` and the sender's address, and one with no participants, keyed `orphaned:`, for the ones the account holder sent; version 8 put them all in one `individual` conversation named `orphaned`. Version 8 had kept an edited message's earlier versions in its own `edits`, where version 7 kept the Apple Messages edit history in `imessage.edits`. Version 7 had moved a message's mark, Deleted in the source app or Unsent, in its own `deletion`, where version 6 kept the Apple Messages deleted mark in `imessage.is_deleted`. Version 6 had moved a message's reactions into its own `reactions` list, where version 5 kept Apple Messages reactions in `imessage.tapbacks`. Version 5 had called every address an identity (`identity`, `identity_type`, `owner_identity`, `sender_identity`, `reactor_identity`) where version 4 said `handle`. A file written at any other version is refused, with an error naming both the file's version and the version the server expects. To import an older export, re-export it with the current desktop app. ## Related diff --git a/docs/src/content/docs/docs/developer/reference/server-cli.md b/docs/src/content/docs/docs/developer/reference/server-cli.md index e36411e45..2dd4634af 100644 --- a/docs/src/content/docs/docs/developer/reference/server-cli.md +++ b/docs/src/content/docs/docs/developer/reference/server-cli.md @@ -37,7 +37,7 @@ Import and view messages in SQLite * `import` — Import a message-ir JSONL directory, one Import Run per source (source from export.source unless --source) * `imports` — Work on an account's Import Runs (`discard` clears a stranded one) -* `dedupe-cross-source` — Soft-hide the same SMS when it appears under more than one import source +* `dedupe-cross-source` — Soft-hide the same SMS when it appears under more than one import source, or once in whole seconds beside its millisecond copy in one source * `reset-demo` — Rebuild the Demo Account: generate Demo Data, clear the account, import, and process assets. Adds the account when it is not there * `create-database` — Create an empty database, with no Demo Account. `serve` adds the Demo Account only to a database that does not exist yet, so this is how a Message Crate starts empty * `serve` — Run the HTTP API. A database that does not exist yet is created with the Demo Account before the server listens @@ -109,7 +109,7 @@ Discard the account's running Import Run, if it has one. A killed `import` leave ## `message-crate-server dedupe-cross-source` -Soft-hide the same SMS when it appears under more than one import source +Soft-hide the same SMS when it appears under more than one import source, or once in whole seconds beside its millisecond copy in one source **Usage:** `message-crate-server dedupe-cross-source [OPTIONS] --account ` diff --git a/docs/src/content/docs/docs/developer/release.md b/docs/src/content/docs/docs/developer/release.md index 4ce3c4459..a7345f0df 100644 --- a/docs/src/content/docs/docs/developer/release.md +++ b/docs/src/content/docs/docs/developer/release.md @@ -12,7 +12,7 @@ The same tag publishes this documentation site to messagecrate.app, so the site Nothing is published to npm or PyPI. Pushing the git tag `v` is what runs the release jobs. A merge to `main` does not ship. -The JSONL schema version 10 is independent of the product version. Version 9 and older are refused, never upgraded. Leave other `Cargo.toml` files at `0.1.0`, and don't bump `web-next/` for a product release. +The JSONL schema version 12 is independent of the product version. Version 11 and older are refused, never upgraded. Leave other `Cargo.toml` files at `0.1.0`, and don't bump `web-next/` for a product release. ## Before tagging diff --git a/docs/src/content/docs/docs/user/features/messages/import.md b/docs/src/content/docs/docs/user/features/messages/import.md index 025f5e030..c4bdbc869 100644 --- a/docs/src/content/docs/docs/user/features/messages/import.md +++ b/docs/src/content/docs/docs/user/features/messages/import.md @@ -390,3 +390,6 @@ Which backup is newer comes from the backup's own date: the date an iPhone backu **Settings → Storage → Import history** shows it beside the backup each run read. A backup that records no date at all keeps a mark once given, and takes a newer text only when its edits are newer. It also keeps the attachments and reactions each backup holds of a message, each one once, so one import of two backups stores what two separate imports of them store. + +Some backups record a message's time to the second and others to the millisecond, and an SMS Backup+ backup can hold both for one message. +When the Message Crate hides duplicates (the server's `import` and `dedupe-cross-source` commands do), a message one backup app holds once to the second and once to the millisecond is shown once, at its time to the millisecond. When another backup app holds the same message too, the Message Crate may show that app's copy instead, which can have only whole seconds. diff --git a/schema/sql/messages.sql b/schema/sql/messages.sql index 063323782..250564af6 100644 --- a/schema/sql/messages.sql +++ b/schema/sql/messages.sql @@ -62,6 +62,13 @@ CREATE TABLE IF NOT EXISTS messages ( -- text sorts in time order and lists order by it. Shown, searched and -- filed by day and year in the account's time zone (accounts.time_zone). timestamp TEXT NOT NULL, + -- Whether the source recorded timestamp to the millisecond or in whole + -- seconds, as the conversation file's time_precision says. The flag, never + -- the time, decides: a millisecond time can end in .000. Within one + -- source, a 'seconds' message is the duplicate of one that matches it in + -- everything else with 'milliseconds' in the same second + -- (docs/architecture/contacts-identities-and-messages.md). + time_precision TEXT NOT NULL CHECK (time_precision IN ('seconds', 'milliseconds')), -- 1 = sent by the account holder; 0 = received from someone else. is_from_me INTEGER NOT NULL, -- Sender identity (`handles.id`); NULL when unknown. diff --git a/schema/sql/staging.sql b/schema/sql/staging.sql index c7246c47f..b78a4e1f9 100644 --- a/schema/sql/staging.sql +++ b/schema/sql/staging.sql @@ -49,6 +49,9 @@ CREATE TABLE IF NOT EXISTS staging_messages ( -- The instant the message was sent, in the form messages.timestamp holds: -- RFC 3339 in UTC to the millisecond (2015-03-12T18:04:22.250Z). timestamp TEXT NOT NULL, + -- Whether the source recorded timestamp to the millisecond or in whole + -- seconds, as messages.time_precision. + time_precision TEXT NOT NULL CHECK (time_precision IN ('seconds', 'milliseconds')), -- 1 = sent by the account holder; 0 = received from someone else. is_from_me INTEGER NOT NULL, -- Sender identity handle id; NULL when unknown. diff --git a/src-tauri/src/commands/upload.rs b/src-tauri/src/commands/upload.rs index 85ab7fc06..1641e0bf8 100644 --- a/src-tauri/src/commands/upload.rs +++ b/src-tauri/src/commands/upload.rs @@ -394,6 +394,7 @@ mod tests { let message = json!(IrMessage { guid: "guid-1".into(), timestamp_unix_ms: 1_400_773_261_000, + time_precision: message_ir::TimePrecision::Milliseconds, direction: IrDirection::Incoming, service: IrService::Sms, message_kind: IrMessageKind::Sms, diff --git a/web/src/lib/serverApi.types.ts b/web/src/lib/serverApi.types.ts index bab9af5dc..63c0ce985 100644 --- a/web/src/lib/serverApi.types.ts +++ b/web/src/lib/serverApi.types.ts @@ -3418,6 +3418,12 @@ export interface components { tapbacks: components["schemas"]["Tapback"][]; /** @description Body text, when present. */ text: string | null; + /** + * @description Whether the source recorded `timestamp` to the millisecond or in + * whole seconds. A `timestamp` ending in `.000` is a whole second + * only when this says `seconds`. + */ + time_precision: components["schemas"]["TimePrecision"]; /** * @description The instant the message was sent, to the millisecond: RFC 3339 * in UTC with three fractional digits and a `Z` suffix @@ -4503,6 +4509,12 @@ export interface components { tapbacks: components["schemas"]["Tapback"][]; /** @description Body text, when present. */ text: string | null; + /** + * @description Whether the source recorded `timestamp` to the millisecond or in + * whole seconds. A `timestamp` ending in `.000` is a whole second + * only when this says `seconds`. + */ + time_precision: components["schemas"]["TimePrecision"]; /** * @description The instant the message was sent, to the millisecond: RFC 3339 * in UTC with three fractional digits and a `Z` suffix @@ -5096,6 +5108,11 @@ export interface components { /** @description The identity that reacted, for incoming reactions. */ sender: string | null; }; + /** + * @description How finely the source recorded a message's time. + * @enum {string} + */ + TimePrecision: "seconds" | "milliseconds"; /** @description One of an account's largest attachments by byte size. */ TopAttachment: { /** @description Raw text of the identity that keys the conversation. */ diff --git a/web/src/test/apiShapes.ts b/web/src/test/apiShapes.ts index 4e5b321ad..e4f04d856 100644 --- a/web/src/test/apiShapes.ts +++ b/web/src/test/apiShapes.ts @@ -45,6 +45,7 @@ export function message(fields: Partial = {}): Schema["Messag service: null, guid: "g1", timestamp: "2026-08-11T15:04:00Z", + time_precision: "milliseconds", sort_order: 0, is_from_me: false, is_announcement: false,