From 2ccc606feefbcdfe79d97060a473bf4c6e68fc7d Mon Sep 17 00:00:00 2001 From: adhi Date: Mon, 24 Aug 2026 16:13:17 -0500 Subject: [PATCH 1/2] make stuff very fast --- Cargo.lock | 1 + crates/flowstate-collab/src/crdt_runtime.rs | 45 ++-- .../tests/vendored_patch_guards.rs | 42 ++++ crates/flowstate-document/src/loro_import.rs | 151 +++++++++++-- crates/flowstate-document/src/loro_schema.rs | 9 +- vendor/loro-internal/Cargo.lock | 1 + vendor/loro-internal/Cargo.toml | 3 + vendor/loro-internal/Cargo.toml.orig | 1 + vendor/loro-internal/src/handler.rs | 25 ++- vendor/loro-internal/src/lib.rs | 1 + vendor/loro-internal/src/mergeable_marker.rs | 201 ++++++++++++++++++ vendor/loro-internal/src/state.rs | 9 +- vendor/loro-internal/src/state/map_state.rs | 2 +- vendor/loro-internal/src/state/mergeable.rs | 8 +- vendor/loro-internal/src/txn.rs | 6 +- 15 files changed, 450 insertions(+), 55 deletions(-) create mode 100644 vendor/loro-internal/src/mergeable_marker.rs diff --git a/Cargo.lock b/Cargo.lock index 5627a842..cdbe8e8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6253,6 +6253,7 @@ dependencies = [ "append-only-bytes", "arref", "bytes", + "crc32fast", "either", "ensure-cov", "enum-as-inner 0.6.1", diff --git a/crates/flowstate-collab/src/crdt_runtime.rs b/crates/flowstate-collab/src/crdt_runtime.rs index 2b3d42d0..94a101ac 100644 --- a/crates/flowstate-collab/src/crdt_runtime.rs +++ b/crates/flowstate-collab/src/crdt_runtime.rs @@ -11,8 +11,8 @@ use std::{ use anyhow::{Context as _, Result}; use flowstate_document::{ AssetId, AssetRecord, BLOCKS_BY_ID, Block, BlockId, CellId, ColumnId, DEFAULT_UPDATE_SEGMENT_COMPACTION_THRESHOLD, DocumentPackage, - DocumentProjection, FLOW_ATTRS_KEY, FLOW_ID_KEY, FLOW_KIND_KEY, FLOW_TEXT_KEY, FLOWS_BY_ID, ImportedLoroDocument, InputBlock, - InputBlockAlignment, InputEquationDisplay, InputImageSizing, InputParagraph, InputTableBlock, InputTableCell, InputTableCellBlock, + DocumentProjection, FLOW_ID_KEY, FLOW_KIND_KEY, FLOW_TEXT_KEY, FLOWS_BY_ID, ImportedLoroDocument, InputBlock, InputBlockAlignment, + InputEquationDisplay, InputEquationSyntax, InputImageSizing, InputParagraph, InputTableBlock, InputTableCell, InputTableCellBlock, InputTableColumnWidth, MAIN_BODY_BLOCK_ID, MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, MARK_PARAGRAPH_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, OBJECT_REPLACEMENT, PARAGRAPHS_BY_ID, Paragraph, ParagraphId, ParagraphStyle, ProjectionDefect, ProjectionPatch, ProjectionStructuralBlock, ROOT, ROOT_BODY_FLOW_ID, ROOT_FIRST_PARAGRAPH_ID, RowId, RunSemanticStyle, RunStyles, SENTINEL_NEWLINE, SectionId, @@ -5973,9 +5973,34 @@ fn replace_equation_block_from_input(doc: &LoroDoc, block: &LoroMap, equation: & block.insert("source_flow_id", source_flow_id.as_str())?; let source_flow = ensure_flow(doc, &source_flow_id, "equation_source")?; replace_text(&source_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, &equation.source)?; - let attrs = block.ensure_mergeable_map("attrs")?; - attrs.insert("syntax", "latex")?; - attrs.insert("display", equation_display_name(equation.display))?; + write_equation_attrs(block, equation.syntax, equation.display)?; + Ok(()) +} + +fn write_equation_attrs(block: &LoroMap, syntax: InputEquationSyntax, display: InputEquationDisplay) -> Result<()> { + // Match the import writer's sparse canonical form: Latex + display are + // projection defaults, so only the non-default inline display needs storage. + match syntax { + InputEquationSyntax::Latex => {}, + } + let attrs = match display { + InputEquationDisplay::Display => child_map(block, "attrs"), + InputEquationDisplay::InlineLikeParagraph => Some(block.ensure_mergeable_map("attrs")?), + }; + let Some(attrs) = attrs else { + return Ok(()); + }; + if attrs.get("syntax").is_some() { + attrs.delete("syntax")?; + } + match display { + InputEquationDisplay::Display => { + if attrs.get("display").is_some() { + attrs.delete("display")?; + } + }, + InputEquationDisplay::InlineLikeParagraph => attrs.insert("display", equation_display_name(display))?, + } Ok(()) } @@ -6667,7 +6692,6 @@ fn ensure_paragraph_metadata_at_boundary_with_keys( paragraph.insert("start_cursor", encoded.clone())?; paragraph.insert("boundary_cursor", encoded.clone())?; } - let _paragraph_attrs = paragraph.ensure_mergeable_map("attrs")?; let block_id = forced_block_id .or_else(|| paragraph_block_key_at_boundary(doc, body, &blocks, boundary)) @@ -6680,8 +6704,6 @@ fn ensure_paragraph_metadata_at_boundary_with_keys( if let Some(encoded) = boundary_cursor { block.insert("anchor_cursor", encoded)?; } - let _block_attrs = block.ensure_mergeable_map("attrs")?; - let _nested_refs = block.ensure_mergeable_map("nested_refs")?; Ok(()) } @@ -7382,9 +7404,7 @@ fn insert_equation_block(doc: &LoroDoc, unicode_index: usize, source: &str, disp block.insert("source_flow_id", source_flow_id.as_str())?; let source_flow = ensure_flow(doc, &source_flow_id, "equation_source")?; replace_text(&source_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, source)?; - let attrs = block.ensure_mergeable_map("attrs")?; - attrs.insert("syntax", "latex")?; - attrs.insert("display", equation_display_name(display))?; + write_equation_attrs(&block, InputEquationSyntax::Latex, display)?; Ok(()) } @@ -7493,7 +7513,6 @@ fn ensure_flow(doc: &LoroDoc, flow_id: &str, kind: &str) -> loro::LoroResult (std::time::Duration, std::time::Dura mod tests { use super::*; + #[test] + fn mergeable_marker_fast_path_round_trips_all_enabled_container_kinds() -> Result<()> { + // Marker creation still uses loro-common; reading after a snapshot import + // routes through the vendored allocation-free parser. Non-ASCII and marker + // separator/escape characters guard the exact length-prefixed wire encoding. + let doc = loro::LoroDoc::new(); + let root = doc.get_map("marker/root"); + root.ensure_mergeable_map(r"map/é>\child")?; + root.ensure_mergeable_list(r"list/é>\child")?; + root.ensure_mergeable_movable_list(r"movable/é>\child")?; + root.ensure_mergeable_text(r"text/é>\child")?; + root.ensure_mergeable_tree(r"tree/é>\child")?; + let nested = root.ensure_mergeable_map(r"nested/é>\parent")?; + nested.ensure_mergeable_map(r"child/🤝>\map")?; + doc.commit(); + + let snapshot = doc.export(loro::ExportMode::Snapshot)?; + let reopened = loro::LoroDoc::new(); + reopened.import(&snapshot)?; + let root = reopened.get_map("marker/root"); + macro_rules! assert_container_kind { + ($parent:expr, $key:expr, $conversion:ident) => { + let Some(ValueOrContainer::Container(container)) = $parent.get($key) else { + panic!("{} marker did not translate into a container", $key); + }; + assert!(container.$conversion().is_ok(), "{} translated to the wrong container kind", $key); + }; + } + assert_container_kind!(root, r"map/é>\child", into_map); + assert_container_kind!(root, r"list/é>\child", into_list); + assert_container_kind!(root, r"movable/é>\child", into_movable_list); + assert_container_kind!(root, r"text/é>\child", into_text); + assert_container_kind!(root, r"tree/é>\child", into_tree); + let Some(ValueOrContainer::Container(nested)) = root.get(r"nested/é>\parent") else { + panic!("nested marker did not translate into a container"); + }; + let nested = nested.into_map().expect("nested marker kind"); + assert_container_kind!(nested, r"child/🤝>\map", into_map); + Ok(()) + } + /// §stylemap pass (2026-07-09, follow-on to act-eleven A11.5): pins the /// LAYERED `StyleValue` representation in the vendored `style_range_map.rs` /// (persistent `im::OrdSet` base + small owned overlay + one-side-reuse diff --git a/crates/flowstate-document/src/loro_import.rs b/crates/flowstate-document/src/loro_import.rs index 3ee42219..d0162dc0 100644 --- a/crates/flowstate-document/src/loro_import.rs +++ b/crates/flowstate-document/src/loro_import.rs @@ -11,9 +11,9 @@ use rustc_hash::FxHashMap; use uuid::Uuid; use crate::{ - AssetChunk, BLOCKS_BY_ID, BODY_FLOW_ID, FLOW_ATTRS_KEY, FLOW_ID_KEY, FLOW_KIND_KEY, FLOW_TEXT_KEY, FLOWS_BY_ID, MARK_DIRECT_UNDERLINE, - MARK_HIGHLIGHT_STYLE, MARK_PARAGRAPH_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, MARK_VERT_ALIGN, OBJECT_REPLACEMENT, - PARAGRAPHS_BY_ID, ROOT, ROOT_BODY_FLOW_ID, SECTIONS_BY_ID, + AssetChunk, BLOCKS_BY_ID, BODY_FLOW_ID, FLOW_ID_KEY, FLOW_KIND_KEY, FLOW_TEXT_KEY, FLOWS_BY_ID, MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, + MARK_PARAGRAPH_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, MARK_VERT_ALIGN, OBJECT_REPLACEMENT, PARAGRAPHS_BY_ID, ROOT, + ROOT_BODY_FLOW_ID, SECTIONS_BY_ID, loro_schema::{ ASSETS_BY_ID, REVISIONS, SectionPageAttrs, TABLE_CELLS_BY_ID, TABLE_COLUMN_ORDER, TABLE_COLUMNS_BY_ID, TABLE_KEY, TABLE_ROW_ORDER, TABLE_ROWS_BY_ID, cell_flow_loro_id, cell_loro_id, column_loro_id, row_loro_id, write_section_page_attrs, @@ -80,7 +80,11 @@ fn fidelity_report_import(doc: &LoroDoc, document: &DocumentProjection) { /// `ensure_mergeable_map`, this never creates the container, so a fidelity /// invariant cannot fabricate the very structure it is meant to verify. fn read_root_child_map(doc: &LoroDoc, key: &str) -> Option { - match doc.get_map(ROOT).get(key)? { + read_child_map(&doc.get_map(ROOT), key) +} + +fn read_child_map(parent: &LoroMap, key: &str) -> Option { + match parent.get(key)? { ValueOrContainer::Container(container) => container.into_map().ok(), ValueOrContainer::Value(_) => None, } @@ -181,9 +185,7 @@ pub(crate) fn replace_body_from_document(doc: &LoroDoc, document: &DocumentProje block.insert("source_flow_id", source_flow_id.as_str())?; let source_flow = ensure_flow(&flows, &source_flow_id, "equation_source")?; replace_text(&source_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, equation.source.as_ref())?; - let attrs = block.ensure_mergeable_map("attrs")?; - attrs.insert("syntax", equation_syntax_name(equation.syntax))?; - attrs.insert("display", equation_display_name(equation.display))?; + write_equation_attrs(&block, equation.syntax, equation.display)?; }, (Block::Table(table), FlowBlockPosition::Object { anchor_pos }) => { let durable_block_id = projection_block_id(document, block_ix, "table"); @@ -435,13 +437,9 @@ fn import_paragraph_record( paragraph_map.insert("boundary_cursor", cursor.encode())?; } }); - // §perf-heaven T8.2: the `attrs` container is kept (the projection reads it and - // the invalidation whitelist expects it), but the `*_container_id` MIRROR - // values are dropped. They duplicated `map.id()` — write-only (no reader in the - // projection or collab), yet each was a long flattened-cid `String` stored as a - // map value AND minted as an op, ×5 per paragraph ×N paragraphs (a large slice - // of the 38.7 KB/record). Derivable from the container itself if ever needed. - paragraph_map.ensure_mergeable_map("attrs")?; + // §perf: optional child maps are materialized only when they contain a value. + // Paragraph attributes currently live in text marks, so an eager empty `attrs` + // map adds a container/op per paragraph without encoding document semantics. ensure_block(blocks, block_id, "paragraph", flow_id, body_text, text_op_base, boundary_pos)?; Ok(()) } @@ -901,7 +899,6 @@ fn ensure_flow(flows: &LoroMap, flow_id: &str, kind: &str) -> LoroResult &'static str { } } -fn equation_syntax_name(syntax: EquationSyntax) -> &'static str { +fn write_equation_attrs(block: &LoroMap, syntax: EquationSyntax, display: EquationDisplay) -> LoroResult<()> { + // Latex + display are the projection's documented defaults. Encoding those + // defaults created an otherwise-unneeded map plus two values per equation. + // Existing maps are retained (they may contain future/unknown fields), but + // legacy default keys are removed when this writer revisits the block. match syntax { - EquationSyntax::Latex => "latex", + EquationSyntax::Latex => {}, + } + let attrs = match display { + EquationDisplay::Display => read_child_map(block, "attrs"), + EquationDisplay::InlineLikeParagraph => Some(block.ensure_mergeable_map("attrs")?), + }; + let Some(attrs) = attrs else { + return Ok(()); + }; + if attrs.get("syntax").is_some() { + attrs.delete("syntax")?; } + match display { + EquationDisplay::Display => { + if attrs.get("display").is_some() { + attrs.delete("display")?; + } + }, + EquationDisplay::InlineLikeParagraph => attrs.insert("display", equation_display_name(display))?, + } + Ok(()) } fn equation_display_name(display: EquationDisplay) -> &'static str { @@ -1037,6 +1056,96 @@ mod tests { Ok(()) } + #[test] + fn canonical_import_omits_empty_optional_maps_and_default_equation_attrs() -> io::Result<()> { + for (display, expected_display_attr) in [ + (gpui_flowtext::InputEquationDisplay::Display, None), + (gpui_flowtext::InputEquationDisplay::InlineLikeParagraph, Some("inline_like_paragraph")), + ] { + let source = gpui_flowtext::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![ + gpui_flowtext::InputBlock::Paragraph(gpui_flowtext::InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![gpui_flowtext::InputRun { + text: "before".to_string(), + styles: RunStyles::default(), + }], + }), + gpui_flowtext::InputBlock::Equation(gpui_flowtext::InputEquationBlock { + source: "x^2".to_string(), + syntax: gpui_flowtext::InputEquationSyntax::Latex, + display, + }), + ], + ); + let paragraph_key = projection_block_id(&source, 0, "paragraph_block"); + let equation_key = projection_block_id(&source, 1, "equation"); + let paragraph_id = projection_paragraph_id(&source, 0); + let equation_flow_id = nested_flow_id("equation_source", &equation_key); + let doc = document_to_loro(&source, "Sparse defaults")?; + + let blocks = read_root_child_map(&doc, BLOCKS_BY_ID).expect("blocks map"); + let paragraph_block = read_child_map(&blocks, ¶graph_key).expect("paragraph block"); + assert!(paragraph_block.get("attrs").is_none()); + assert!(paragraph_block.get("nested_refs").is_none()); + let equation_block = read_child_map(&blocks, &equation_key).expect("equation block"); + assert!(equation_block.get("nested_refs").is_none()); + let equation_attrs = read_child_map(&equation_block, "attrs"); + assert_eq!( + equation_attrs + .as_ref() + .and_then(|attrs| attrs.get("display")) + .and_then(|value| match value { + ValueOrContainer::Value(LoroValue::String(value)) => Some(value.to_string()), + _ => None, + }) + .as_deref(), + expected_display_attr + ); + assert!( + equation_attrs + .as_ref() + .is_none_or(|attrs| attrs.get("syntax").is_none()) + ); + + let paragraphs = read_root_child_map(&doc, PARAGRAPHS_BY_ID).expect("paragraphs map"); + assert!( + read_child_map(¶graphs, ¶graph_id) + .expect("paragraph record") + .get("attrs") + .is_none() + ); + let flows = read_root_child_map(&doc, FLOWS_BY_ID).expect("flows map"); + assert!( + read_child_map(&flows, ROOT_BODY_FLOW_ID) + .expect("body flow") + .get("attrs") + .is_none() + ); + assert!( + read_child_map(&flows, &equation_flow_id) + .expect("equation flow") + .get("attrs") + .is_none() + ); + + let projected = crate::document_from_loro(&doc)?; + let Block::Equation(equation) = &projected.blocks[1] else { + panic!("second block should remain an equation"); + }; + assert_eq!(equation.source.as_ref(), "x^2"); + assert_eq!( + equation.display, + match display { + gpui_flowtext::InputEquationDisplay::Display => EquationDisplay::Display, + gpui_flowtext::InputEquationDisplay::InlineLikeParagraph => EquationDisplay::InlineLikeParagraph, + } + ); + } + Ok(()) + } + #[test] fn custom_paragraph_style_slot_zero_round_trips() -> io::Result<()> { let source = gpui_flowtext::document_from_input_blocks( diff --git a/crates/flowstate-document/src/loro_schema.rs b/crates/flowstate-document/src/loro_schema.rs index 7f4c188a..079f56d8 100644 --- a/crates/flowstate-document/src/loro_schema.rs +++ b/crates/flowstate-document/src/loro_schema.rs @@ -22,6 +22,8 @@ pub const COMMENTS_BY_ID: &str = "comments_by_id"; pub const REPLICAS_BY_ID: &str = "replicas_by_id"; pub const FLOW_TEXT_KEY: &str = "text"; +/// Optional child map. Canonical writers leave it absent while all attributes +/// have their projection defaults; the first non-default attribute creates it. pub const FLOW_ATTRS_KEY: &str = "attrs"; pub const FLOW_KIND_KEY: &str = "kind"; pub const FLOW_ID_KEY: &str = "id"; @@ -338,7 +340,7 @@ fn init_loro_document_structure(doc: &LoroDoc, title: &str, include_initial_para meta.insert("revisions_container_id", revisions.id().to_string())?; meta.insert("users_container_id", users.id().to_string())?; meta.insert("replicas_container_id", replicas.id().to_string())?; - // `ensure_flow` establishes the body flow's text + attrs containers for both + // `ensure_flow` establishes the body flow's required text container for both // the seeded and the import (`include_initial_paragraph == false`) paths. ensure_flow(&flows, ROOT_BODY_FLOW_ID, "body")?; if include_initial_paragraph { @@ -374,7 +376,6 @@ pub fn seed_document_body(doc: &LoroDoc) -> LoroResult<()> { let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID)?; let body_flow = ensure_flow(&flows, ROOT_BODY_FLOW_ID, "body")?; let body_text = body_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; - body_flow.ensure_mergeable_map(FLOW_ATTRS_KEY)?; ensure_sentinel(&body_text)?; ensure_initial_paragraph(¶graphs, &blocks, &body_text)?; // §P2a fidelity: the canonical seed establishes exactly the sentinel + @@ -705,7 +706,6 @@ fn ensure_flow(flows: &LoroMap, flow_id: &str, kind: &str) -> LoroResult>() @@ -4647,7 +4650,11 @@ impl MapHandler { MaybeDetached::Attached(inner) => { let value = inner .with_doc_state(|state| state.get_map_value_by_key(inner.container_idx, key))?; - Some(loro_common::translate_mergeable_marker_value(&inner.id, key, value)) + Some(crate::mergeable_marker::translate_mergeable_marker_value( + &inner.id, + key, + value, + )) } } } @@ -4662,7 +4669,8 @@ impl MapHandler { MaybeDetached::Attached(inner) => { let value = inner .with_doc_state(|state| state.get_map_value_by_key(inner.container_idx, key))?; - let value = loro_common::translate_mergeable_marker_value(&inner.id, key, value); + let value = + crate::mergeable_marker::translate_mergeable_marker_value(&inner.id, key, value); Some(value_to_value_or_handler(inner, value)) } } @@ -4728,7 +4736,7 @@ impl MapHandler { // accepted as an existing mergeable occupant. if let Some(existing) = &existing_raw { if !matches!(existing, LoroValue::Null) - && loro_common::parse_mergeable_marker(&parent.id, key, existing).is_none() + && crate::mergeable_marker::parse_mergeable_marker(&parent.id, key, existing).is_none() { return Err(LoroError::ArgErr( format!( @@ -4872,8 +4880,11 @@ impl MapHandler { .get_map_entries(a.container_idx) .into_iter() .map(|(key, value)| { - let translated = - loro_common::translate_mergeable_marker_value(&a.id, key.as_ref(), value); + let translated = crate::mergeable_marker::translate_mergeable_marker_value( + &a.id, + key.as_ref(), + value, + ); value_to_value_or_handler(a, translated) }) .collect() diff --git a/vendor/loro-internal/src/lib.rs b/vendor/loro-internal/src/lib.rs index 256b7b96..f8d24988 100644 --- a/vendor/loro-internal/src/lib.rs +++ b/vendor/loro-internal/src/lib.rs @@ -66,6 +66,7 @@ pub mod version; mod error; #[cfg(feature = "test_utils")] pub mod fuzz; +mod mergeable_marker; mod parent; pub mod pre_commit; mod span; diff --git a/vendor/loro-internal/src/mergeable_marker.rs b/vendor/loro-internal/src/mergeable_marker.rs new file mode 100644 index 00000000..d76e154e --- /dev/null +++ b/vendor/loro-internal/src/mergeable_marker.rs @@ -0,0 +1,201 @@ +//! Allocation-free validation of mergeable-container map markers. +//! +//! `loro_common::parse_mergeable_marker` assembles a temporary byte vector and +//! evaluates IEEE CRC-32 one bit at a time. Map reads invoke marker translation +//! for every mergeable child, making that validation a projection hot path. +//! Keep the wire-compatible validation here, but stream the exact encoded +//! segments into `crc32fast` without allocating. + +use crc32fast::Hasher; +use loro_common::{ + ContainerID, ContainerType, LoroValue, MERGEABLE_MARKER_MAGIC, +}; + +const MARKER_DIGEST_LEN: usize = 3; +const MARKER_LEN: usize = MERGEABLE_MARKER_MAGIC.len() + 1 + MARKER_DIGEST_LEN; +const CRC_DOMAIN: &[u8] = b"loro.mergeable.marker.v1"; + +pub(crate) fn parse_mergeable_marker( + parent: &ContainerID, + key: &str, + value: &LoroValue, +) -> Option { + let LoroValue::Binary(bytes) = value else { + return None; + }; + if bytes.len() != MARKER_LEN || !bytes.starts_with(&MERGEABLE_MARKER_MAGIC) { + return None; + } + + let kind = ContainerType::try_from_u8(bytes[MERGEABLE_MARKER_MAGIC.len()]).ok()?; + if matches!(kind, ContainerType::Unknown(_)) { + return None; + } + + let digest_start = MERGEABLE_MARKER_MAGIC.len() + 1; + let expected = mergeable_marker_crc24(parent, key, kind); + (&bytes[digest_start..] == expected.as_slice()).then_some(kind) +} + +pub(crate) fn translate_mergeable_marker_value( + parent: &ContainerID, + key: &str, + value: LoroValue, +) -> LoroValue { + match parse_mergeable_marker(parent, key, &value) { + Some(kind) => LoroValue::Container(ContainerID::new_mergeable(parent, key, kind)), + None => value, + } +} + +fn mergeable_marker_crc24( + parent: &ContainerID, + key: &str, + kind: ContainerType, +) -> [u8; MARKER_DIGEST_LEN] { + let mut hasher = Hasher::new(); + hasher.update(CRC_DOMAIN); + update_len_prefixed_parent(&mut hasher, parent); + update_unsigned(&mut hasher, key.len()); + hasher.update(key.as_bytes()); + hasher.update(&[kind.to_u8()]); + + let crc = hasher.finalize() & 0x00ff_ffff; + [ + ((crc >> 16) & 0xff) as u8, + ((crc >> 8) & 0xff) as u8, + (crc & 0xff) as u8, + ] +} + +/// Feed the bytes produced by `ContainerID::encode`, preceded by their unsigned +/// LEB128 length, without materializing `ContainerID::to_bytes()`. +fn update_len_prefixed_parent(hasher: &mut Hasher, parent: &ContainerID) { + let encoded_len = match parent { + ContainerID::Root { name, .. } => { + 1 + unsigned_len(name.len()) + name.len() + } + ContainerID::Normal { .. } => 13, + }; + update_unsigned(hasher, encoded_len); + + match parent { + ContainerID::Root { + name, + container_type, + } => { + hasher.update(&[container_type.to_u8() | 0b1000_0000]); + update_unsigned(hasher, name.len()); + hasher.update(name.as_bytes()); + } + ContainerID::Normal { + peer, + counter, + container_type, + } => { + hasher.update(&[container_type.to_u8()]); + hasher.update(&peer.to_le_bytes()); + hasher.update(&counter.to_le_bytes()); + } + } +} + +fn unsigned_len(mut value: usize) -> usize { + let mut len = 1; + while value >= 0x80 { + value >>= 7; + len += 1; + } + len +} + +fn update_unsigned(hasher: &mut Hasher, mut value: usize) { + let mut bytes = [0_u8; 10]; + let mut len = 0; + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + bytes[len] = byte; + len += 1; + if value == 0 { + break; + } + } + hasher.update(&bytes[..len]); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fast_marker_parser_matches_loro_common() { + let root = ContainerID::Root { + name: "root".into(), + container_type: ContainerType::Map, + }; + let normal = ContainerID::Normal { + peer: u64::MAX - 7, + counter: -42, + container_type: ContainerType::Map, + }; + let nested = ContainerID::new_mergeable( + &root, + "nested/key>with\\escapes", + ContainerType::Map, + ); + + for parent in [&root, &normal, &nested] { + for key in ["plain", "slash/key", "angle>key", r"back\slash", "émoji-🤝"] { + for kind in ContainerType::ALL_TYPES { + let marker = loro_common::mergeable_marker(parent, key, kind); + assert_eq!( + parse_mergeable_marker(parent, key, &marker), + loro_common::parse_mergeable_marker(parent, key, &marker), + ); + assert_eq!( + translate_mergeable_marker_value(parent, key, marker.clone()), + loro_common::translate_mergeable_marker_value(parent, key, marker), + ); + } + } + } + } + + #[test] + fn malformed_values_remain_scalars() { + let parent = ContainerID::Root { + name: "root".into(), + container_type: ContainerType::Map, + }; + let key = "child"; + let mut corrupt = loro_common::mergeable_marker( + &parent, + key, + ContainerType::Map, + ); + let LoroValue::Binary(bytes) = &mut corrupt else { + unreachable!(); + }; + let mut bytes = bytes.to_vec(); + bytes[MARKER_LEN - 1] ^= 0xff; + corrupt = LoroValue::Binary(bytes.into()); + + assert_eq!(parse_mergeable_marker(&parent, key, &corrupt), None); + assert_eq!( + translate_mergeable_marker_value(&parent, key, corrupt.clone()), + corrupt, + ); + assert_eq!( + translate_mergeable_marker_value( + &parent, + key, + LoroValue::String("not a marker".into()), + ), + LoroValue::String("not a marker".into()), + ); + } +} diff --git a/vendor/loro-internal/src/state.rs b/vendor/loro-internal/src/state.rs index 7cd015cd..325e4e58 100644 --- a/vendor/loro-internal/src/state.rs +++ b/vendor/loro-internal/src/state.rs @@ -2119,7 +2119,7 @@ impl DocState { LoroValue::Container(c) => c.clone(), value => { let parent_id = parent_id?; - let kind = loro_common::parse_mergeable_marker( + let kind = crate::mergeable_marker::parse_mergeable_marker( &parent_id, key, value, )?; ContainerID::new_mergeable(&parent_id, key, kind) @@ -2208,8 +2208,11 @@ impl DocState { if let Some(key) = index.as_key() { let value = m.get(key).cloned()?; if let Some(parent_id) = &parent_id { - if let Some(kind) = - loro_common::parse_mergeable_marker(parent_id, key, &value) + if let Some(kind) = crate::mergeable_marker::parse_mergeable_marker( + parent_id, + key, + &value, + ) { let cid = ContainerID::new_mergeable(parent_id, key, kind); LoroValue::Container(cid) diff --git a/vendor/loro-internal/src/state/map_state.rs b/vendor/loro-internal/src/state/map_state.rs index 947eb6bc..d691563f 100644 --- a/vendor/loro-internal/src/state/map_state.rs +++ b/vendor/loro-internal/src/state/map_state.rs @@ -30,7 +30,7 @@ fn translate_with_parent( ) -> LoroValue { match parent_id { Some(parent) => { - loro_common::translate_mergeable_marker_value(parent, key.as_ref(), value) + crate::mergeable_marker::translate_mergeable_marker_value(parent, key.as_ref(), value) } None => value, } diff --git a/vendor/loro-internal/src/state/mergeable.rs b/vendor/loro-internal/src/state/mergeable.rs index f41080c2..a3e91d1b 100644 --- a/vendor/loro-internal/src/state/mergeable.rs +++ b/vendor/loro-internal/src/state/mergeable.rs @@ -62,7 +62,9 @@ impl DocState { } let value = self.store.map_get(parent_idx, key)?; - if loro_common::parse_mergeable_marker(&actual_parent_id, key, &value) == Some(kind) { + if crate::mergeable_marker::parse_mergeable_marker(&actual_parent_id, key, &value) + == Some(kind) + { Some(Index::Key(key.into())) } else { None @@ -82,7 +84,9 @@ impl DocState { ) -> Vec<(InternalString, ContainerID)> { let mut ans = Vec::new(); for (key, value) in map_value.iter() { - if let Some(kind) = loro_common::parse_mergeable_marker(parent_id, key, value) { + if let Some(kind) = + crate::mergeable_marker::parse_mergeable_marker(parent_id, key, value) + { let key_istr: InternalString = key.as_str().into(); let cid = ContainerID::new_mergeable(parent_id, key, kind); ans.push((key_istr, cid)); diff --git a/vendor/loro-internal/src/txn.rs b/vendor/loro-internal/src/txn.rs index 3d8ee4ad..ced1225a 100644 --- a/vendor/loro-internal/src/txn.rs +++ b/vendor/loro-internal/src/txn.rs @@ -1323,7 +1323,11 @@ fn change_to_diff( let parent_id = doc.arena.idx_to_id(container_idx); let value = value.map(|v| match parent_id.as_ref() { Some(pid) => { - loro_common::translate_mergeable_marker_value(pid, key.as_ref(), v) + crate::mergeable_marker::translate_mergeable_marker_value( + pid, + key.as_ref(), + v, + ) } None => v, }); From 8d0a35e9d635a3045b8fb74a7300637af6a5b207 Mon Sep 17 00:00:00 2001 From: adhi Date: Mon, 24 Aug 2026 17:25:27 -0500 Subject: [PATCH 2/2] stuff --- .cargo/audit.toml | 14 +++++++++++--- Cargo.lock | 4 ++-- deny.toml | 20 ++++++++++++++++++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 80d23156..20a84fe1 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -4,7 +4,15 @@ # quick-xml 0.41; the remaining old copies in Cargo.lock are pinned by the # LATEST releases of third-party crates (xcb 0.30, docx-rs 0.36, # rdocx-opc/oxml 0.37, citationberg-via-typst 0.38, wayland-scanner 0.39) and -# cannot be bumped without vendoring them. Mirrors the deny.toml ignore list — -# keep the two in sync. Re-check on dependency upgrades. +# cannot be bumped without vendoring them. +# +# rkyv 0.7 is pinned by mitex 0.2.4 and only reads mitex's generated +# specification data. The affected 0.7 series has no patched release; mitex +# must migrate to rkyv >= 0.8.17. Mirrors the vulnerability entries in +# deny.toml; keep them in sync and re-check on dependency upgrades. [advisories] -ignore = ["RUSTSEC-2026-0194", "RUSTSEC-2026-0195"] +ignore = [ + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", + "RUSTSEC-2026-0235", +] diff --git a/Cargo.lock b/Cargo.lock index cdbe8e8f..23193f76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4241,9 +4241,9 @@ checksum = "12101ecc8225ea6d675bc70263074eab6169079621c2186fe0c66590b2df9681" [[package]] name = "h2" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", diff --git a/deny.toml b/deny.toml index a551eafb..e918b54b 100644 --- a/deny.toml +++ b/deny.toml @@ -40,8 +40,24 @@ yanked = "warn" # `quick-xml-oxml` rename in flowstate-docx (Reader types must unify) # 0.38 citationberg (even 0.7.0 pins ^0.38; via typst-library/hayagriva) # 0.39 wayland-scanner (proc-macro, build-time trusted input) -# Re-check on dependency upgrades and drop these when upstreams move. -ignore = ["RUSTSEC-2026-0194", "RUSTSEC-2026-0195"] +# rkyv 0.7 archive-validation advisory. This copy is pinned by mitex 0.2.4 and +# only reads mitex's generated specification data; rkyv 0.7 has no patched +# release, while the fix requires mitex to migrate to rkyv >= 0.8.17. +# +# im advisories. im has no fixed release (the project is archived), and both +# Loro 1.13's vendored internals and gpui-flowtext depend on its persistent +# collections. Replacing the app's direct dependency alone would leave Loro's +# affected copy in the graph. Migrate both consumers to imbl when compatible. +# +# Re-check on dependency upgrades and drop these when upstreams move. Keep the +# vulnerability entries in sync with .cargo/audit.toml. +ignore = [ + "RUSTSEC-2023-0126", + "RUSTSEC-2026-0194", + "RUSTSEC-2026-0195", + "RUSTSEC-2026-0235", + "RUSTSEC-2026-0248", +] # --------------------------------------------------------------------------- # License policy. Unlisted licenses are denied under version 2.